@cogitator-ai/rag 0.1.9 → 0.2.0

package/dist/chunkers/semantic-chunker.js

@@ -1,10 +1,13 @@
 import { nanoid } from 'nanoid';
 function cosineSimilarity(a, b) {
-    const len = Math.min(a.length, b.length);
+    if (a.length !== b.length) {
+        throw new Error(`Embedding dimension mismatch: ${a.length} vs ${b.length}. ` +
+            'Ensure all embeddings come from the same model.');
+    }
     let dot = 0;
     let normA = 0;
     let normB = 0;
-    for (let i = 0; i < len; i++) {
+    for (let i = 0; i < a.length; i++) {
         dot += a[i] * b[i];
         normA += a[i] * a[i];
         normB += b[i] * b[i];
@@ -12,7 +15,7 @@ function cosineSimilarity(a, b) {
     const denom = Math.sqrt(normA) * Math.sqrt(normB);
     return denom === 0 ? 0 : dot / denom;
 }
-const SENTENCE_SPLIT_RE = /(?<=[.?!])\s+/;
+const SENTENCE_SPLIT_RE = /(?<=[.?!])\s+|\n\s*\n/;
 function splitSentences(text) {
     return text
         .split(SENTENCE_SPLIT_RE)
@@ -25,10 +28,19 @@ export class SemanticChunker {
     minChunkSize;
     maxChunkSize;
     constructor(options) {
+        if (options.breakpointThreshold !== undefined &&
+            (options.breakpointThreshold < 0 || options.breakpointThreshold > 1)) {
+            throw new Error('breakpointThreshold must be between 0 and 1');
+        }
+        const maxChunkSize = options.maxChunkSize ?? 2000;
+        const minChunkSize = options.minChunkSize ?? Math.min(100, maxChunkSize);
+        if (minChunkSize > maxChunkSize) {
+            throw new Error('minChunkSize must not exceed maxChunkSize');
+        }
         this.embeddingService = options.embeddingService;
         this.breakpointThreshold = options.breakpointThreshold ?? 0.5;
-        this.minChunkSize = options.minChunkSize ?? 100;
-        this.maxChunkSize = options.maxChunkSize ?? 2000;
+        this.minChunkSize = minChunkSize;
+        this.maxChunkSize = maxChunkSize;
     }
     async chunk(text, documentId) {
         if (!text || text.trim().length === 0)
@@ -40,6 +52,9 @@ export class SemanticChunker {
             return this.buildChunks([sentences], text, documentId);
         }
         const embeddings = await this.embeddingService.embedBatch(sentences);
+        if (embeddings.length !== sentences.length) {
+            throw new Error(`Embedding count mismatch: got ${embeddings.length} vectors for ${sentences.length} sentences`);
+        }
         const similarities = [];
         for (let i = 0; i < embeddings.length - 1; i++) {
             similarities.push(cosineSimilarity(embeddings[i], embeddings[i + 1]));
@@ -94,21 +109,25 @@ export class SemanticChunker {
         }
         return result;
     }
+    findSentenceOffset(sentence, originalText, searchFrom) {
+        const idx = originalText.indexOf(sentence, searchFrom);
+        return idx >= 0 ? idx : searchFrom;
+    }
     buildChunks(groups, originalText, documentId) {
         const chunks = [];
         let searchFrom = 0;
         for (const group of groups) {
-            const content = group.join(' ');
+            const startOffset = this.findSentenceOffset(group[0], originalText, searchFrom);
+            const lastSentence = group[group.length - 1];
+            const lastStart = this.findSentenceOffset(lastSentence, originalText, startOffset);
+            const endOffset = lastStart + lastSentence.length;
+            const content = originalText.slice(startOffset, endOffset);
             if (content.length <= this.maxChunkSize) {
-                const startOffset = originalText.indexOf(group[0], searchFrom);
-                const lastSentence = group[group.length - 1];
-                const lastStart = originalText.indexOf(lastSentence, startOffset >= 0 ? startOffset : 0);
-                const endOffset = lastStart >= 0 ? lastStart + lastSentence.length : startOffset + content.length;
                 chunks.push({
                     id: nanoid(),
                     documentId,
                     content,
-                    startOffset: Math.max(0, startOffset),
+                    startOffset,
                     endOffset,
                     order: chunks.length,
                 });
@@ -131,22 +150,35 @@ export class SemanticChunker {
         let order = startOrder;
         let offset = searchFrom;
         for (const sentence of sentences) {
+            if (sentence.length > this.maxChunkSize) {
+                if (currentSentences.length > 0) {
+                    const chunk = this.emitSentenceGroup(currentSentences, originalText, documentId, offset, order++);
+                    chunks.push(chunk);
+                    offset = chunk.endOffset;
+                    currentSentences = [];
+                    currentLength = 0;
+                }
+                const sentenceStart = this.findSentenceOffset(sentence, originalText, offset);
+                for (let pos = 0; pos < sentence.length; pos += this.maxChunkSize) {
+                    const sliceEnd = Math.min(pos + this.maxChunkSize, sentence.length);
+                    const slice = sentence.slice(pos, sliceEnd);
+                    chunks.push({
+                        id: nanoid(),
+                        documentId,
+                        content: slice,
+                        startOffset: sentenceStart + pos,
+                        endOffset: sentenceStart + sliceEnd,
+                        order: order++,
+                    });
+                }
+                offset = sentenceStart + sentence.length;
+                continue;
+            }
             const newLength = currentLength + (currentSentences.length > 0 ? 1 : 0) + sentence.length;
             if (newLength > this.maxChunkSize && currentSentences.length > 0) {
-                const content = currentSentences.join(' ');
-                const startOffset = originalText.indexOf(currentSentences[0], offset);
-                const lastSentence = currentSentences[currentSentences.length - 1];
-                const lastStart = originalText.indexOf(lastSentence, startOffset >= 0 ? startOffset : 0);
-                const endOffset = lastStart >= 0 ? lastStart + lastSentence.length : startOffset + content.length;
-                chunks.push({
-                    id: nanoid(),
-                    documentId,
-                    content,
-                    startOffset: Math.max(0, startOffset),
-                    endOffset,
-                    order: order++,
-                });
-                offset = endOffset;
+                const chunk = this.emitSentenceGroup(currentSentences, originalText, documentId, offset, order++);
+                chunks.push(chunk);
+                offset = chunk.endOffset;
                 currentSentences = [sentence];
                 currentLength = sentence.length;
             }
@@ -156,21 +188,23 @@ export class SemanticChunker {
             }
         }
         if (currentSentences.length > 0) {
-            const content = currentSentences.join(' ');
-            const startOffset = originalText.indexOf(currentSentences[0], offset);
-            const lastSentence = currentSentences[currentSentences.length - 1];
-            const lastStart = originalText.indexOf(lastSentence, startOffset >= 0 ? startOffset : 0);
-            const endOffset = lastStart >= 0 ? lastStart + lastSentence.length : startOffset + content.length;
-            chunks.push({
-                id: nanoid(),
-                documentId,
-                content,
-                startOffset: Math.max(0, startOffset),
-                endOffset,
-                order: order++,
-            });
+            chunks.push(this.emitSentenceGroup(currentSentences, originalText, documentId, offset, order++));
         }
         return chunks;
     }
+    emitSentenceGroup(sentences, originalText, documentId, searchFrom, order) {
+        const startOffset = this.findSentenceOffset(sentences[0], originalText, searchFrom);
+        const lastSentence = sentences[sentences.length - 1];
+        const lastStart = this.findSentenceOffset(lastSentence, originalText, startOffset);
+        const endOffset = lastStart + lastSentence.length;
+        return {
+            id: nanoid(),
+            documentId,
+            content: originalText.slice(startOffset, endOffset),
+            startOffset,
+            endOffset,
+            order,
+        };
+    }
 }
 //# sourceMappingURL=semantic-chunker.js.map

package/dist/chunkers/semantic-chunker.js.map

@@ -1 +1 @@
-{"version":3,"file":"semantic-chunker.js","sourceRoot":"","sources":["../../src/chunkers/semantic-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAUhC,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IAChD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;IACzC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC;AACvC,CAAC;AAED,MAAM,iBAAiB,GAAG,eAAe,CAAC;AAE1C,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,IAAI;SACR,KAAK,CAAC,iBAAiB,CAAC;SACxB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,OAAO,eAAe;IACT,gBAAgB,CAAmB;IACnC,mBAAmB,CAAS;IAC5B,YAAY,CAAS;IACrB,YAAY,CAAS;IAEtC,YAAY,OAA+B;QACzC,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;QACjD,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,IAAI,GAAG,CAAC;QAC9D,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,GAAG,CAAC;QAChD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,UAAkB;QAC1C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjD,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEtC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAErE,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,YAAY,CAAC,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,MAAM,GAAe,EAAE,CAAC;QAC9B,IAAI,YAAY,GAAa,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QAE5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,IAAI,YAAY,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAC/C,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAC1B,YAAY,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAE1B,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAC7C,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IACpD,CAAC;IAEO,gBAAgB,CAAC,MAAkB;QACzC,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC;YAAE,OAAO,MAAM,CAAC;QAEtC,MAAM,MAAM,GAAe,EAAE,CAAC;QAC9B,IAAI,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrC,MAAM,cAAc,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;YAEhE,IAAI,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,cAAc,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClF,OAAO,GAAG,CAAC,GAAG,OAAO,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YACvC,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACrB,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,IAAI,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAChC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClE,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,CAAC;gBACpD,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvB,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,WAAW,CACjB,MAAkB,EAClB,YAAoB,EACpB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAEhC,IAAI,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACxC,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;gBAC/D,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAC7C,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,YAAY,EAAE,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACzF,MAAM,SAAS,GACb,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;gBAElF,MAAM,CAAC,IAAI,CAAC;oBACV,EAAE,EAAE,MAAM,EAAE;oBACZ,UAAU;oBACV,OAAO;oBACP,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC;oBACrC,SAAS;oBACT,KAAK,EAAE,MAAM,CAAC,MAAM;iBACrB,CAAC,CAAC;gBAEH,UAAU,GAAG,SAAS,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CACpC,KAAK,EACL,YAAY,EACZ,UAAU,EACV,UAAU,EACV,MAAM,CAAC,MAAM,CACd,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC1B,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzB,UAAU,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;gBACzD,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,eAAe,CACrB,SAAmB,EACnB,YAAoB,EACpB,UAAkB,EAClB,UAAkB,EAClB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,gBAAgB,GAAa,EAAE,CAAC;QACpC,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,KAAK,GAAG,UAAU,CAAC;QACvB,IAAI,MAAM,GAAG,UAAU,CAAC;QAExB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,SAAS,GAAG,aAAa,GAAG,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;YAE1F,IAAI,SAAS,GAAG,IAAI,CAAC,YAAY,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACjE,MAAM,OAAO,GAAG,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC3C,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;gBACtE,MAAM,YAAY,GAAG,gBAAgB,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACnE,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,YAAY,EAAE,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACzF,MAAM,SAAS,GACb,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;gBAElF,MAAM,CAAC,IAAI,CAAC;oBACV,EAAE,EAAE,MAAM,EAAE;oBACZ,UAAU;oBACV,OAAO;oBACP,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC;oBACrC,SAAS;oBACT,KAAK,EAAE,KAAK,EAAE;iBACf,CAAC,CAAC;gBAEH,MAAM,GAAG,SAAS,CAAC;gBACnB,gBAAgB,GAAG,CAAC,QAAQ,CAAC,CAAC;gBAC9B,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACN,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAChC,aAAa,GAAG,SAAS,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,OAAO,GAAG,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC3C,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;YACtE,MAAM,YAAY,GAAG,gBAAgB,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACnE,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,YAAY,EAAE,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACzF,MAAM,SAAS,GACb,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;YAElF,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,MAAM,EAAE;gBACZ,UAAU;gBACV,OAAO;gBACP,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC;gBACrC,SAAS;gBACT,KAAK,EAAE,KAAK,EAAE;aACf,CAAC,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}
+{"version":3,"file":"semantic-chunker.js","sourceRoot":"","sources":["../../src/chunkers/semantic-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAUhC,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IAChD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,iCAAiC,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,MAAM,IAAI;YAC1D,iDAAiD,CACpD,CAAC;IACJ,CAAC;IACD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC;AACvC,CAAC;AAED,MAAM,iBAAiB,GAAG,uBAAuB,CAAC;AAElD,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,IAAI;SACR,KAAK,CAAC,iBAAiB,CAAC;SACxB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,OAAO,eAAe;IACT,gBAAgB,CAAmB;IACnC,mBAAmB,CAAS;IAC5B,YAAY,CAAS;IACrB,YAAY,CAAS;IAEtC,YAAY,OAA+B;QACzC,IACE,OAAO,CAAC,mBAAmB,KAAK,SAAS;YACzC,CAAC,OAAO,CAAC,mBAAmB,GAAG,CAAC,IAAI,OAAO,CAAC,mBAAmB,GAAG,CAAC,CAAC,EACpE,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;QACjE,CAAC;QAED,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC;QAClD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QAEzE,IAAI,YAAY,GAAG,YAAY,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;QACjD,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,IAAI,GAAG,CAAC;QAC9D,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,UAAkB;QAC1C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjD,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEtC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QACrE,IAAI,UAAU,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,EAAE,CAAC;YAC3C,MAAM,IAAI,KAAK,CACb,iCAAiC,UAAU,CAAC,MAAM,gBAAgB,SAAS,CAAC,MAAM,YAAY,CAC/F,CAAC;QACJ,CAAC;QAED,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,YAAY,CAAC,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,MAAM,GAAe,EAAE,CAAC;QAC9B,IAAI,YAAY,GAAa,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QAE5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,IAAI,YAAY,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAC/C,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAC1B,YAAY,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAE1B,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAC7C,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IACpD,CAAC;IAEO,gBAAgB,CAAC,MAAkB;QACzC,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC;YAAE,OAAO,MAAM,CAAC;QAEtC,MAAM,MAAM,GAAe,EAAE,CAAC;QAC9B,IAAI,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrC,MAAM,cAAc,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;YAEhE,IAAI,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,cAAc,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClF,OAAO,GAAG,CAAC,GAAG,OAAO,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YACvC,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACrB,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,IAAI,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAChC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClE,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,CAAC;gBACpD,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvB,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,kBAAkB,CAAC,QAAgB,EAAE,YAAoB,EAAE,UAAkB;QACnF,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QACvD,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC;IACrC,CAAC;IAEO,WAAW,CACjB,MAAkB,EAClB,YAAoB,EACpB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC;YAChF,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;YACnF,MAAM,SAAS,GAAG,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC;YAElD,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;YAE3D,IAAI,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACxC,MAAM,CAAC,IAAI,CAAC;oBACV,EAAE,EAAE,MAAM,EAAE;oBACZ,UAAU;oBACV,OAAO;oBACP,WAAW;oBACX,SAAS;oBACT,KAAK,EAAE,MAAM,CAAC,MAAM;iBACrB,CAAC,CAAC;gBACH,UAAU,GAAG,SAAS,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CACpC,KAAK,EACL,YAAY,EACZ,UAAU,EACV,UAAU,EACV,MAAM,CAAC,MAAM,CACd,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC1B,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzB,UAAU,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;gBACzD,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,eAAe,CACrB,SAAmB,EACnB,YAAoB,EACpB,UAAkB,EAClB,UAAkB,EAClB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,gBAAgB,GAAa,EAAE,CAAC;QACpC,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,KAAK,GAAG,UAAU,CAAC;QACvB,IAAI,MAAM,GAAG,UAAU,CAAC;QAExB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,IAAI,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;gBACxC,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAChC,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAClC,gBAAgB,EAChB,YAAY,EACZ,UAAU,EACV,MAAM,EACN,KAAK,EAAE,CACR,CAAC;oBACF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACnB,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC;oBACzB,gBAAgB,GAAG,EAAE,CAAC;oBACtB,aAAa,GAAG,CAAC,CAAC;gBACpB,CAAC;gBAED,MAAM,aAAa,GAAG,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;gBAC9E,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,QAAQ,CAAC,MAAM,EAAE,GAAG,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;oBACpE,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;oBAC5C,MAAM,CAAC,IAAI,CAAC;wBACV,EAAE,EAAE,MAAM,EAAE;wBACZ,UAAU;wBACV,OAAO,EAAE,KAAK;wBACd,WAAW,EAAE,aAAa,GAAG,GAAG;wBAChC,SAAS,EAAE,aAAa,GAAG,QAAQ;wBACnC,KAAK,EAAE,KAAK,EAAE;qBACf,CAAC,CAAC;gBACL,CAAC;gBACD,MAAM,GAAG,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC;gBACzC,SAAS;YACX,CAAC;YAED,MAAM,SAAS,GAAG,aAAa,GAAG,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;YAE1F,IAAI,SAAS,GAAG,IAAI,CAAC,YAAY,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACjE,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAClC,gBAAgB,EAChB,YAAY,EACZ,UAAU,EACV,MAAM,EACN,KAAK,EAAE,CACR,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACnB,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC;gBACzB,gBAAgB,GAAG,CAAC,QAAQ,CAAC,CAAC;gBAC9B,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACN,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAChC,aAAa,GAAG,SAAS,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,CAAC,IAAI,CACT,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CACpF,CAAC;QACJ,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,iBAAiB,CACvB,SAAmB,EACnB,YAAoB,EACpB,UAAkB,EAClB,UAAkB,EAClB,KAAa;QAEb,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC;QACpF,MAAM,YAAY,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACrD,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;QACnF,MAAM,SAAS,GAAG,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC;QAElD,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,UAAU;YACV,OAAO,EAAE,YAAY,CAAC,KAAK,CAAC,WAAW,EAAE,SAAS,CAAC;YACnD,WAAW;YACX,SAAS;YACT,KAAK;SACN,CAAC;IACJ,CAAC;CACF"}

package/dist/index.d.ts

@@ -1,4 +1,4 @@
-export declare const VERSION = "0.1.0";
+export declare const VERSION: string;
 export { RAGPipeline, type RAGPipelineDeps } from './rag-pipeline.js';
 export { RAGPipelineBuilder } from './rag-builder.js';
 export * from './loaders/index.js';
@@ -6,6 +6,6 @@ export * from './chunkers/index.js';
 export * from './retrievers/index.js';
 export * from './rerankers/index.js';
 export * from './schema.js';
-export { createSearchTool, createIngestTool, ragTools, type RAGTool } from './tools.js';
+export { createSearchTool, createIngestTool, ragTools, type RAGTool, type IngestToolOptions, } from './tools.js';
 export type { RAGDocument, DocumentChunk, DocumentLoader, Chunker, AsyncChunker, ChunkingConfig, ChunkingStrategy, Retriever, RetrievalConfig, RetrievalStrategy, RetrievalResult, Reranker, RAGPipelineConfig, EmbeddingService, EmbeddingAdapter, } from '@cogitator-ai/types';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,OAAO,UAAU,CAAC;AAE/B,OAAO,EAAE,WAAW,EAAE,KAAK,eAAe,EAAE,MAAM,gBAAgB,CAAC;AACnE,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAEnD,cAAc,WAAW,CAAC;AAC1B,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAE5B,cAAc,UAAU,CAAC;AAEzB,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,QAAQ,EAAE,KAAK,OAAO,EAAE,MAAM,SAAS,CAAC;AAErF,YAAY,EACV,WAAW,EACX,aAAa,EACb,cAAc,EACd,OAAO,EACP,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,SAAS,EACT,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,QAAQ,EACR,iBAAiB,EACjB,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,qBAAqB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,OAAO,EAAE,MAA4B,CAAC;AAEnD,OAAO,EAAE,WAAW,EAAE,KAAK,eAAe,EAAE,MAAM,mBAAmB,CAAC;AACtE,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEtD,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC;AAErC,cAAc,aAAa,CAAC;AAE5B,OAAO,EACL,gBAAgB,EAChB,gBAAgB,EAChB,QAAQ,EACR,KAAK,OAAO,EACZ,KAAK,iBAAiB,GACvB,MAAM,YAAY,CAAC;AAEpB,YAAY,EACV,WAAW,EACX,aAAa,EACb,cAAc,EACd,OAAO,EACP,YAAY,EACZ,cAAc,EACd,gBAAgB,EAChB,SAAS,EACT,eAAe,EACf,iBAAiB,EACjB,eAAe,EACf,QAAQ,EACR,iBAAiB,EACjB,gBAAgB,EAChB,gBAAgB,GACjB,MAAM,qBAAqB,CAAC"}

package/dist/index.js

@@ -1,4 +1,6 @@
-export const VERSION = '0.1.0';
+import { createRequire } from 'node:module';
+const packageJson = createRequire(import.meta.url)('../package.json');
+export const VERSION = packageJson.version;
 export { RAGPipeline } from './rag-pipeline.js';
 export { RAGPipelineBuilder } from './rag-builder.js';
 export * from './loaders/index.js';
@@ -6,5 +8,5 @@ export * from './chunkers/index.js';
 export * from './retrievers/index.js';
 export * from './rerankers/index.js';
 export * from './schema.js';
-export { createSearchTool, createIngestTool, ragTools } from './tools.js';
+export { createSearchTool, createIngestTool, ragTools, } from './tools.js';
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,OAAO,GAAG,OAAO,CAAC;AAE/B,OAAO,EAAE,WAAW,EAAwB,MAAM,gBAAgB,CAAC;AACnE,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAEnD,cAAc,WAAW,CAAC;AAC1B,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAE5B,cAAc,UAAU,CAAC;AAEzB,OAAO,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,QAAQ,EAAgB,MAAM,SAAS,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAE5C,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,iBAAiB,CAAwB,CAAC;AAE7F,MAAM,CAAC,MAAM,OAAO,GAAW,WAAW,CAAC,OAAO,CAAC;AAEnD,OAAO,EAAE,WAAW,EAAwB,MAAM,mBAAmB,CAAC;AACtE,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAEtD,cAAc,oBAAoB,CAAC;AACnC,cAAc,qBAAqB,CAAC;AACpC,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC;AAErC,cAAc,aAAa,CAAC;AAE5B,OAAO,EACL,gBAAgB,EAChB,gBAAgB,EAChB,QAAQ,GAGT,MAAM,YAAY,CAAC"}

package/dist/loaders/csv-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"csv-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/csv-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAUvE,MAAM,WAAW,gBAAgB;IAC/B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,SAAU,YAAW,cAAc;IAC9C,QAAQ,CAAC,cAAc,WAAW;IAClC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAW;IAC5C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAS;gBAExB,OAAO,CAAC,EAAE,gBAAgB;IAMhC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IA4BlD,OAAO,CAAC,eAAe;CAWxB"}
+{"version":3,"file":"csv-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/csv-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAevE,MAAM,WAAW,gBAAgB;IAC/B,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,SAAU,YAAW,cAAc;IAC9C,QAAQ,CAAC,cAAc,WAAW;IAClC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAW;IAC5C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAS;gBAExB,OAAO,CAAC,EAAE,gBAAgB;IAMhC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAmClD,OAAO,CAAC,eAAe;CAWxB"}

package/dist/loaders/csv-loader.js

@@ -2,12 +2,15 @@ import { readFile } from 'node:fs/promises';
 import { resolve } from 'node:path';
 import { nanoid } from 'nanoid';
 async function loadPapaparse() {
+    let mod;
     try {
-        return await import('papaparse');
+        mod = (await import('papaparse'));
     }
     catch {
         throw new Error('papaparse is required for CSVLoader. Install it: pnpm add papaparse');
     }
+    const interop = mod.default;
+    return interop && typeof interop.parse === 'function' ? interop : mod;
 }
 export class CSVLoader {
     supportedTypes = ['csv'];
@@ -30,15 +33,20 @@ export class CSVLoader {
         });
         const headers = result.meta.fields ?? [];
         const contentCol = this.contentColumn ?? headers[0];
-        return result.data.map((row) => {
+        if (contentCol === undefined)
+            return [];
+        if (this.contentColumn && !headers.includes(this.contentColumn)) {
+            throw new Error(`CSVLoader: contentColumn "${this.contentColumn}" not found in CSV headers: ${headers.join(', ')}`);
+        }
+        return result.data.map((row, index) => {
             const content = row[contentCol] ?? '';
-            const metadata = this.extractMetadata(row);
+            const metadata = { ...this.extractMetadata(row), row: index + 1 };
             return {
                 id: nanoid(),
                 content,
                 source: filePath,
                 sourceType: 'csv',
-                ...(metadata && Object.keys(metadata).length > 0 && { metadata }),
+                metadata,
             };
         });
     }

package/dist/loaders/csv-loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"csv-loader.js","sourceRoot":"","sources":["../../src/loaders/csv-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,KAAK,UAAU,aAAa;IAC1B,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;IACnC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;IACzF,CAAC;AACH,CAAC;AAQD,MAAM,OAAO,SAAS;IACX,cAAc,GAAG,CAAC,KAAK,CAAC,CAAC;IACjB,aAAa,CAAU;IACvB,eAAe,CAAY;IAC3B,SAAS,CAAU;IAEpC,YAAY,OAA0B;QACpC,IAAI,CAAC,aAAa,GAAG,OAAO,EAAE,aAAa,CAAC;QAC5C,IAAI,CAAC,eAAe,GAAG,OAAO,EAAE,eAAe,CAAC;QAChD,IAAI,CAAC,SAAS,GAAG,OAAO,EAAE,SAAS,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,IAAI,GAAG,MAAM,aAAa,EAAE,CAAC;QACnC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAE9C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAyB,GAAG,EAAE;YACrD,MAAM,EAAE,IAAI;YACZ,cAAc,EAAE,IAAI;YACpB,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;QACzC,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;QAEpD,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;YAC7B,MAAM,OAAO,GAAG,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;YACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;YAE3C,OAAO;gBACL,EAAE,EAAE,MAAM,EAAE;gBACZ,OAAO;gBACP,MAAM,EAAE,QAAQ;gBAChB,UAAU,EAAE,KAAc;gBAC1B,GAAG,CAAC,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC;aAClE,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,eAAe,CAAC,GAA2B;QACjD,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,OAAO,SAAS,CAAC;QAE5C,MAAM,QAAQ,GAA4B,EAAE,CAAC;QAC7C,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACvC,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;gBACf,QAAQ,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF"}
+{"version":3,"file":"csv-loader.js","sourceRoot":"","sources":["../../src/loaders/csv-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAKhC,KAAK,UAAU,aAAa;IAC1B,IAAI,GAA8B,CAAC;IACnC,IAAI,CAAC;QACH,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,WAAW,CAAC,CAA8B,CAAC;IACjE,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;IACzF,CAAC;IACD,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;IAC5B,OAAO,OAAO,IAAI,OAAO,OAAO,CAAC,KAAK,KAAK,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACxE,CAAC;AAQD,MAAM,OAAO,SAAS;IACX,cAAc,GAAG,CAAC,KAAK,CAAC,CAAC;IACjB,aAAa,CAAU;IACvB,eAAe,CAAY;IAC3B,SAAS,CAAU;IAEpC,YAAY,OAA0B;QACpC,IAAI,CAAC,aAAa,GAAG,OAAO,EAAE,aAAa,CAAC;QAC5C,IAAI,CAAC,eAAe,GAAG,OAAO,EAAE,eAAe,CAAC;QAChD,IAAI,CAAC,SAAS,GAAG,OAAO,EAAE,SAAS,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,IAAI,GAAG,MAAM,aAAa,EAAE,CAAC;QACnC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAE9C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAyB,GAAG,EAAE;YACrD,MAAM,EAAE,IAAI;YACZ,cAAc,EAAE,IAAI;YACpB,SAAS,EAAE,IAAI,CAAC,SAAS;SAC1B,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;QACzC,MAAM,UAAU,GAAG,IAAI,CAAC,aAAa,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC;QACpD,IAAI,UAAU,KAAK,SAAS;YAAE,OAAO,EAAE,CAAC;QAExC,IAAI,IAAI,CAAC,aAAa,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,aAAa,CAAC,EAAE,CAAC;YAChE,MAAM,IAAI,KAAK,CACb,6BAA6B,IAAI,CAAC,aAAa,+BAA+B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACnG,CAAC;QACJ,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;YACpC,MAAM,OAAO,GAAG,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC;YACtC,MAAM,QAAQ,GAAG,EAAE,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,EAAE,GAAG,EAAE,KAAK,GAAG,CAAC,EAAE,CAAC;YAElE,OAAO;gBACL,EAAE,EAAE,MAAM,EAAE;gBACZ,OAAO;gBACP,MAAM,EAAE,QAAQ;gBAChB,UAAU,EAAE,KAAc;gBAC1B,QAAQ;aACT,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,eAAe,CAAC,GAA2B;QACjD,IAAI,CAAC,IAAI,CAAC,eAAe;YAAE,OAAO,SAAS,CAAC;QAE5C,MAAM,QAAQ,GAA4B,EAAE,CAAC;QAC7C,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACvC,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC;gBACf,QAAQ,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF"}

package/dist/loaders/html-loader.d.ts

@@ -2,6 +2,10 @@ import type { DocumentLoader, RAGDocument } from '@cogitator-ai/types';
 export interface HTMLLoaderOptions {
     selector?: string;
 }
+/**
+ * Extracts readable text from HTML: scripts, styles and other non-content elements are
+ * dropped, block elements become line breaks and whitespace is normalized.
+ */
 export declare class HTMLLoader implements DocumentLoader {
     readonly supportedTypes: string[];
     readonly selector: string;

package/dist/loaders/html-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"html-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/html-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAUvE,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,qBAAa,UAAW,YAAW,cAAc;IAC/C,QAAQ,CAAC,cAAc,WAAmB;IAC1C,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;gBAEd,OAAO,CAAC,EAAE,iBAAiB;IAIjC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAM5C,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,KAAK,GAAG,OAAO,CAAC,WAAW,CAAC;CAkBhG"}
+{"version":3,"file":"html-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/html-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAUvE,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAgBD;;;GAGG;AACH,qBAAa,UAAW,YAAW,cAAc;IAC/C,QAAQ,CAAC,cAAc,WAAmB;IAC1C,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;gBAEd,OAAO,CAAC,EAAE,iBAAiB;IAIjC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAM5C,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,KAAK,GAAG,OAAO,CAAC,WAAW,CAAC;CA+BhG"}

package/dist/loaders/html-loader.js

@@ -9,6 +9,21 @@ async function loadCheerio() {
         throw new Error('cheerio is required for HTMLLoader. Install it: pnpm add cheerio');
     }
 }
+const NON_CONTENT_SELECTOR = 'script, style, noscript, template, svg, iframe, object, head';
+const BLOCK_SELECTOR = 'address, article, aside, blockquote, dd, div, dl, dt, fieldset, figcaption, figure, footer, ' +
+    'form, h1, h2, h3, h4, h5, h6, header, hr, li, main, nav, ol, p, pre, section, table, tr, ul';
+function normalizeWhitespace(text) {
+    return text
+        .replace(/\r\n?/g, '\n')
+        .replace(/[^\S\n]+/g, ' ')
+        .replace(/ *\n */g, '\n')
+        .replace(/\n{3,}/g, '\n\n')
+        .trim();
+}
+/**
+ * Extracts readable text from HTML: scripts, styles and other non-content elements are
+ * dropped, block elements become line breaks and whitespace is normalized.
+ */
 export class HTMLLoader {
     supportedTypes = ['html', 'htm'];
     selector;
@@ -23,8 +38,17 @@ export class HTMLLoader {
     async parseHTML(html, source, sourceType) {
         const cheerio = await loadCheerio();
         const $ = cheerio.load(html);
-        const content = $(this.selector).text().trim();
-        const title = $('title').text().trim() || undefined;
+        const title = $('title').first().text().trim() || undefined;
+        $(NON_CONTENT_SELECTOR).remove();
+        $('br').replaceWith('\n');
+        $(BLOCK_SELECTOR).each((_, element) => {
+            $(element).before('\n').after('\n');
+        });
+        $('td, th').after(' ');
+        const content = normalizeWhitespace($(this.selector)
+            .map((_, element) => $(element).text())
+            .get()
+            .join('\n\n'));
         const metadata = {};
         if (title)
             metadata.title = title;

package/dist/loaders/html-loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"html-loader.js","sourceRoot":"","sources":["../../src/loaders/html-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,KAAK,UAAU,WAAW;IACxB,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;IACjC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;IACtF,CAAC;AACH,CAAC;AAMD,MAAM,OAAO,UAAU;IACZ,cAAc,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACjC,QAAQ,CAAS;IAE1B,YAAY,OAA2B;QACrC,IAAI,CAAC,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,MAAM,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC/C,OAAO,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,IAAY,EAAE,MAAc,EAAE,UAA0B;QACtE,MAAM,OAAO,GAAG,MAAM,WAAW,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE7B,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAC/C,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC;QAEpD,MAAM,QAAQ,GAA4B,EAAE,CAAC;QAC7C,IAAI,KAAK;YAAE,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;QAElC,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM;YACN,UAAU;YACV,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC;SACtD,CAAC;IACJ,CAAC;CACF"}
+{"version":3,"file":"html-loader.js","sourceRoot":"","sources":["../../src/loaders/html-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,KAAK,UAAU,WAAW;IACxB,IAAI,CAAC;QACH,OAAO,MAAM,MAAM,CAAC,SAAS,CAAC,CAAC;IACjC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,kEAAkE,CAAC,CAAC;IACtF,CAAC;AACH,CAAC;AAMD,MAAM,oBAAoB,GAAG,8DAA8D,CAAC;AAC5F,MAAM,cAAc,GAClB,8FAA8F;IAC9F,6FAA6F,CAAC;AAEhG,SAAS,mBAAmB,CAAC,IAAY;IACvC,OAAO,IAAI;SACR,OAAO,CAAC,QAAQ,EAAE,IAAI,CAAC;SACvB,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC;SACzB,OAAO,CAAC,SAAS,EAAE,IAAI,CAAC;SACxB,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC;SAC1B,IAAI,EAAE,CAAC;AACZ,CAAC;AAED;;;GAGG;AACH,MAAM,OAAO,UAAU;IACZ,cAAc,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACjC,QAAQ,CAAS;IAE1B,YAAY,OAA2B;QACrC,IAAI,CAAC,QAAQ,GAAG,OAAO,EAAE,QAAQ,IAAI,MAAM,CAAC;IAC9C,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC/C,OAAO,CAAC,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,IAAY,EAAE,MAAc,EAAE,UAA0B;QACtE,MAAM,OAAO,GAAG,MAAM,WAAW,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAE7B,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC;QAE5D,CAAC,CAAC,oBAAoB,CAAC,CAAC,MAAM,EAAE,CAAC;QACjC,CAAC,CAAC,IAAI,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;QAC1B,CAAC,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE;YACpC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACtC,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,QAAQ,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEvB,MAAM,OAAO,GAAG,mBAAmB,CACjC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC;aACb,GAAG,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;aACtC,GAAG,EAAE;aACL,IAAI,CAAC,MAAM,CAAC,CAChB,CAAC;QAEF,MAAM,QAAQ,GAA4B,EAAE,CAAC;QAC7C,IAAI,KAAK;YAAE,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;QAElC,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM;YACN,UAAU;YACV,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC;SACtD,CAAC;IACJ,CAAC;CACF"}

package/dist/loaders/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/loaders/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AACnD,YAAY,EAAE,qBAAqB,EAAE,MAAM,mBAAmB,CAAC;AAC/D,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,YAAY,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACvD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,YAAY,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,YAAY,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AACvD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,YAAY,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC;AACrD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,YAAY,EAAE,gBAAgB,EAAE,MAAM,cAAc,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/loaders/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,YAAY,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAClE,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,YAAY,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,YAAY,EAAE,iBAAiB,EAAE,MAAM,kBAAkB,CAAC;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACxD,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,YAAY,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC"}

package/dist/loaders/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/loaders/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAC3C,OAAO,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAEnD,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAE3C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAE3C,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAEzC,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/loaders/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAEtD,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAE9C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAE5C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC"}

package/dist/loaders/json-loader.d.ts

@@ -11,6 +11,7 @@ export declare class JSONLoader implements DocumentLoader {
     load(source: string): Promise<RAGDocument[]>;
     private itemToDocument;
     private extractContent;
+    private valueToString;
     private extractMetadata;
 }
 //# sourceMappingURL=json-loader.d.ts.map

package/dist/loaders/json-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"json-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/json-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAIvE,MAAM,WAAW,iBAAiB;IAChC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;CAC3B;AAED,qBAAa,UAAW,YAAW,cAAc;IAC/C,QAAQ,CAAC,cAAc,WAAY;IACnC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAW;gBAE/B,OAAO,CAAC,EAAE,iBAAiB;IAKjC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IASlD,OAAO,CAAC,cAAc;IAsBtB,OAAO,CAAC,cAAc;IActB,OAAO,CAAC,eAAe;CAWxB"}
+{"version":3,"file":"json-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/json-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAIvE,MAAM,WAAW,iBAAiB;IAChC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;CAC3B;AAED,qBAAa,UAAW,YAAW,cAAc;IAC/C,QAAQ,CAAC,cAAc,WAAY;IACnC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAW;gBAE/B,OAAO,CAAC,EAAE,iBAAiB;IAKjC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAgBlD,OAAO,CAAC,cAAc;IAsBtB,OAAO,CAAC,cAAc;IActB,OAAO,CAAC,aAAa;IAKrB,OAAO,CAAC,eAAe;CAWxB"}

package/dist/loaders/json-loader.js

@@ -13,7 +13,14 @@ export class JSONLoader {
     async load(source) {
         const filePath = resolve(source);
         const raw = await readFile(filePath, 'utf-8');
-        const data = JSON.parse(raw);
+        let data;
+        try {
+            data = JSON.parse(raw);
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            throw new Error(`JSONLoader: failed to parse "${filePath}": ${message}`);
+        }
         const items = Array.isArray(data) ? data : [data];
         return items.map((item) => this.itemToDocument(item, filePath));
     }
@@ -39,15 +46,20 @@ export class JSONLoader {
     }
     extractContent(obj) {
         if (this.contentField && this.contentField in obj) {
-            return String(obj[this.contentField]);
+            return this.valueToString(obj[this.contentField]);
         }
         for (const field of CONTENT_FIELDS) {
             if (field in obj) {
-                return String(obj[field]);
+                return this.valueToString(obj[field]);
             }
         }
         return JSON.stringify(obj, null, 2);
     }
+    valueToString(value) {
+        if (typeof value === 'string')
+            return value;
+        return JSON.stringify(value, null, 2);
+    }
     extractMetadata(obj) {
         if (!this.metadataFields)
             return undefined;

package/dist/loaders/json-loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"json-loader.js","sourceRoot":"","sources":["../../src/loaders/json-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,cAAc,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAOnD,MAAM,OAAO,UAAU;IACZ,cAAc,GAAG,CAAC,MAAM,CAAC,CAAC;IAClB,YAAY,CAAU;IACtB,cAAc,CAAY;IAE3C,YAAY,OAA2B;QACrC,IAAI,CAAC,YAAY,GAAG,OAAO,EAAE,YAAY,CAAC;QAC1C,IAAI,CAAC,cAAc,GAAG,OAAO,EAAE,cAAc,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC9C,MAAM,IAAI,GAAY,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEtC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAClD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;IAClE,CAAC;IAEO,cAAc,CAAC,IAAa,EAAE,MAAc;QAClD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACrE,OAAO;gBACL,EAAE,EAAE,MAAM,EAAE;gBACZ,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC;gBACrB,MAAM;gBACN,UAAU,EAAE,MAAM;aACnB,CAAC;QACJ,CAAC;QACD,MAAM,GAAG,GAAG,IAA+B,CAAC;QAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QAE3C,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM;YACN,UAAU,EAAE,MAAM;YAClB,GAAG,CAAC,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC;SAClE,CAAC;IACJ,CAAC;IAEO,cAAc,CAAC,GAA4B;QACjD,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY,IAAI,GAAG,EAAE,CAAC;YAClD,OAAO,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;QACxC,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;gBACjB,OAAO,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACtC,CAAC;IAEO,eAAe,CAAC,GAA4B;QAClD,IAAI,CAAC,IAAI,CAAC,cAAc;YAAE,OAAO,SAAS,CAAC;QAE3C,MAAM,QAAQ,GAA4B,EAAE,CAAC;QAC7C,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxC,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;gBACjB,QAAQ,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF"}
+{"version":3,"file":"json-loader.js","sourceRoot":"","sources":["../../src/loaders/json-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,cAAc,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAOnD,MAAM,OAAO,UAAU;IACZ,cAAc,GAAG,CAAC,MAAM,CAAC,CAAC;IAClB,YAAY,CAAU;IACtB,cAAc,CAAY;IAE3C,YAAY,OAA2B;QACrC,IAAI,CAAC,YAAY,GAAG,OAAO,EAAE,YAAY,CAAC;QAC1C,IAAI,CAAC,cAAc,GAAG,OAAO,EAAE,cAAc,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAE9C,IAAI,IAAa,CAAC;QAClB,IAAI,CAAC;YACH,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACzB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,gCAAgC,QAAQ,MAAM,OAAO,EAAE,CAAC,CAAC;QAC3E,CAAC;QAED,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAClD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;IAClE,CAAC;IAEO,cAAc,CAAC,IAAa,EAAE,MAAc;QAClD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACrE,OAAO;gBACL,EAAE,EAAE,MAAM,EAAE;gBACZ,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC;gBACrB,MAAM;gBACN,UAAU,EAAE,MAAM;aACnB,CAAC;QACJ,CAAC;QACD,MAAM,GAAG,GAAG,IAA+B,CAAC;QAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QAE3C,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM;YACN,UAAU,EAAE,MAAM;YAClB,GAAG,CAAC,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC;SAClE,CAAC;IACJ,CAAC;IAEO,cAAc,CAAC,GAA4B;QACjD,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY,IAAI,GAAG,EAAE,CAAC;YAClD,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;QACpD,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;gBACjB,OAAO,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACtC,CAAC;IAEO,aAAa,CAAC,KAAc;QAClC,IAAI,OAAO,KAAK,KAAK,QAAQ;YAAE,OAAO,KAAK,CAAC;QAC5C,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACxC,CAAC;IAEO,eAAe,CAAC,GAA4B;QAClD,IAAI,CAAC,IAAI,CAAC,cAAc;YAAE,OAAO,SAAS,CAAC;QAE3C,MAAM,QAAQ,GAA4B,EAAE,CAAC;QAC7C,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxC,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;gBACjB,QAAQ,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF"}

package/dist/loaders/markdown-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"markdown-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/markdown-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAMvE,MAAM,WAAW,qBAAqB;IACpC,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,qBAAa,cAAe,YAAW,cAAc;IACnD,QAAQ,CAAC,cAAc,WAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAU;gBAE/B,OAAO,CAAC,EAAE,qBAAqB;IAIrC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YAWpC,QAAQ;IAsBtB,OAAO,CAAC,gBAAgB;YAWV,aAAa;CAiB5B"}
+{"version":3,"file":"markdown-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/markdown-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAWvE,MAAM,WAAW,qBAAqB;IACpC,gBAAgB,CAAC,EAAE,OAAO,CAAC;CAC5B;AAED,qBAAa,cAAe,YAAW,cAAc;IACnD,QAAQ,CAAC,cAAc,WAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAU;gBAE/B,OAAO,CAAC,EAAE,qBAAqB;IAIrC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YAWpC,QAAQ;IAsBtB,OAAO,CAAC,gBAAgB;YAWV,aAAa;CAiB5B"}

package/dist/loaders/markdown-loader.js

@@ -2,7 +2,11 @@ import { readFile, readdir, stat } from 'node:fs/promises';
 import { join, resolve } from 'node:path';
 import { nanoid } from 'nanoid';
 const MD_EXTENSIONS = new Set(['md', 'mdx']);
-const FRONTMATTER_RE = /^---\r?\n([\s\S]*?)\r?\n---\r?\n/;
+const FRONTMATTER_RE = /^---\r?\n([\s\S]*?)\r?\n---(?:\r?\n|$)/;
+function unquote(value) {
+    const quoted = /^(['"])(.*)\1$/.exec(value);
+    return quoted ? quoted[2] : value;
+}
 export class MarkdownLoader {
     supportedTypes = ['md', 'mdx'];
     stripFrontmatter;
@@ -38,16 +42,16 @@ export class MarkdownLoader {
     }
     parseFrontmatter(raw) {
         const result = {};
-        for (const line of raw.split('\n')) {
+        for (const line of raw.split(/\r?\n/)) {
             const match = /^(\w[\w\s-]*?):\s*(.+)$/.exec(line);
             if (match) {
-                result[match[1].trim()] = match[2].trim();
+                result[match[1].trim()] = unquote(match[2].trim());
             }
         }
         return result;
     }
     async loadDirectory(dirPath) {
-        const entries = await readdir(dirPath);
+        const entries = (await readdir(dirPath)).sort();
         const docs = [];
         for (const entry of entries) {
             const ext = entry.split('.').pop()?.toLowerCase() ?? '';

package/dist/loaders/markdown-loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"markdown-loader.js","sourceRoot":"","sources":["../../src/loaders/markdown-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;AAE7C,MAAM,cAAc,GAAG,kCAAkC,CAAC;AAM1D,MAAM,OAAO,cAAc;IAChB,cAAc,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACvB,gBAAgB,CAAU;IAE3C,YAAY,OAA+B;QACzC,IAAI,CAAC,gBAAgB,GAAG,OAAO,EAAE,gBAAgB,IAAI,KAAK,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;QAElC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,QAAgB;QACrC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC9C,IAAI,OAAO,GAAG,GAAG,CAAC;QAClB,IAAI,QAA6C,CAAC;QAElD,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACvC,IAAI,KAAK,EAAE,CAAC;gBACV,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC3C,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YACvC,CAAC;QACH,CAAC;QAED,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM,EAAE,QAAQ;YAChB,UAAU,EAAE,UAAU;YACtB,GAAG,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,CAAC;SAC9B,CAAC;IACJ,CAAC;IAEO,gBAAgB,CAAC,GAAW;QAClC,MAAM,MAAM,GAA4B,EAAE,CAAC;QAC3C,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnD,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC5C,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAe;QACzC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;QACvC,MAAM,IAAI,GAAkB,EAAE,CAAC;QAE/B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;YACxD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YAEtC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClC,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;gBAClB,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}
+{"version":3,"file":"markdown-loader.js","sourceRoot":"","sources":["../../src/loaders/markdown-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;AAE7C,MAAM,cAAc,GAAG,wCAAwC,CAAC;AAEhE,SAAS,OAAO,CAAC,KAAa;IAC5B,MAAM,MAAM,GAAG,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC5C,OAAO,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,KAAK,CAAC;AACrC,CAAC;AAMD,MAAM,OAAO,cAAc;IAChB,cAAc,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACvB,gBAAgB,CAAU;IAE3C,YAAY,OAA+B;QACzC,IAAI,CAAC,gBAAgB,GAAG,OAAO,EAAE,gBAAgB,IAAI,KAAK,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;QAElC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,QAAgB;QACrC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC9C,IAAI,OAAO,GAAG,GAAG,CAAC;QAClB,IAAI,QAA6C,CAAC;QAElD,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACvC,IAAI,KAAK,EAAE,CAAC;gBACV,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC3C,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YACvC,CAAC;QACH,CAAC;QAED,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM,EAAE,QAAQ;YAChB,UAAU,EAAE,UAAU;YACtB,GAAG,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,CAAC;SAC9B,CAAC;IACJ,CAAC;IAEO,gBAAgB,CAAC,GAAW;QAClC,MAAM,MAAM,GAA4B,EAAE,CAAC;QAC3C,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC;YACtC,MAAM,KAAK,GAAG,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnD,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,IAAI,EAAE,CAAC,CAAC;YACvD,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAe;QACzC,MAAM,OAAO,GAAG,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAChD,MAAM,IAAI,GAAkB,EAAE,CAAC;QAE/B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;YACxD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YAEtC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClC,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;gBAClB,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}

package/dist/loaders/pdf-loader.d.ts

@@ -9,6 +9,7 @@ export declare class PDFLoader implements DocumentLoader {
     load(source: string): Promise<RAGDocument[]>;
     private loadSingleDocument;
     private loadSplitPages;
+    private parseWithErrorContext;
     private buildPageDoc;
 }
 //# sourceMappingURL=pdf-loader.d.ts.map

package/dist/loaders/pdf-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pdf-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/pdf-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAuBvE,MAAM,WAAW,gBAAgB;IAC/B,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,qBAAa,SAAU,YAAW,cAAc;IAC9C,QAAQ,CAAC,cAAc,WAAW;IAClC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAU;gBAEzB,OAAO,CAAC,EAAE,gBAAgB;IAIhC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YAYpC,kBAAkB;YAsBlB,cAAc;IAmB5B,OAAO,CAAC,YAAY;CAkBrB"}
+{"version":3,"file":"pdf-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/pdf-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AA4BvE,MAAM,WAAW,gBAAgB;IAC/B,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,qBAAa,SAAU,YAAW,cAAc;IAC9C,QAAQ,CAAC,cAAc,WAAW;IAClC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAU;gBAEzB,OAAO,CAAC,EAAE,gBAAgB;IAIhC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YAYpC,kBAAkB;YAsBlB,cAAc;YA6Bd,qBAAqB;IAcnC,OAAO,CAAC,YAAY;CAkBrB"}

package/dist/loaders/pdf-loader.js

@@ -26,7 +26,7 @@ export class PDFLoader {
         return this.loadSingleDocument(pdfParse, buffer, filePath);
     }
     async loadSingleDocument(pdfParse, buffer, source) {
-        const result = await pdfParse(buffer);
+        const result = await this.parseWithErrorContext(pdfParse, buffer, source);
         const title = result.info?.Title;
         const metadata = { pages: result.numpages };
         if (title)
@@ -42,14 +42,33 @@ export class PDFLoader {
         ];
     }
     async loadSplitPages(pdfParse, buffer, source) {
-        const result = await pdfParse(buffer);
+        const pageTexts = [];
+        const result = await this.parseWithErrorContext(pdfParse, buffer, source, {
+            pagerender: async (pageData) => {
+                if (typeof pageData.getTextContent !== 'function')
+                    return '';
+                const textContent = await pageData.getTextContent();
+                const text = textContent.items.map((item) => item.str).join('');
+                pageTexts.push(text);
+                return text;
+            },
+        });
         const title = result.info?.Title;
-        const pages = result.text.split(/\f/).filter((p) => p.trim().length > 0);
+        const pages = pageTexts.filter((p) => p.trim().length > 0);
         if (pages.length === 0) {
             return [this.buildPageDoc(result.text, source, 1, result.numpages, title)];
         }
         return pages.map((text, i) => this.buildPageDoc(text.trim(), source, i + 1, result.numpages, title));
     }
+    async parseWithErrorContext(pdfParse, buffer, source, options) {
+        try {
+            return await pdfParse(buffer, options);
+        }
+        catch (err) {
+            const message = err instanceof Error ? err.message : String(err);
+            throw new Error(`PDFLoader: failed to parse "${source}": ${message}`);
+        }
+    }
     buildPageDoc(content, source, pageNumber, totalPages, title) {
         const metadata = { pageNumber, totalPages };
         if (title)

package/dist/loaders/pdf-loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"pdf-loader.js","sourceRoot":"","sources":["../../src/loaders/pdf-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAehC,KAAK,UAAU,YAAY;IACzB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;QACtC,OAAO,GAAG,CAAC,OAAgC,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;IACzF,CAAC;AACH,CAAC;AAMD,MAAM,OAAO,SAAS;IACX,cAAc,GAAG,CAAC,KAAK,CAAC,CAAC;IACjB,UAAU,CAAU;IAErC,YAAY,OAA0B;QACpC,IAAI,CAAC,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,KAAK,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,MAAM,YAAY,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAExC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;QACzD,CAAC;QAED,OAAO,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC7D,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,QAAoB,EACpB,MAAc,EACd,MAAc;QAEd,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC;QAEjC,MAAM,QAAQ,GAA4B,EAAE,KAAK,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;QACrE,IAAI,KAAK;YAAE,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;QAElC,OAAO;YACL;gBACE,EAAE,EAAE,MAAM,EAAE;gBACZ,OAAO,EAAE,MAAM,CAAC,IAAI;gBACpB,MAAM;gBACN,UAAU,EAAE,KAAK;gBACjB,QAAQ;aACT;SACF,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,QAAoB,EACpB,MAAc,EACd,MAAc;QAEd,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC;QAEjC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAEzE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;QAC7E,CAAC;QAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAC3B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,CACtE,CAAC;IACJ,CAAC;IAEO,YAAY,CAClB,OAAe,EACf,MAAc,EACd,UAAkB,EAClB,UAAkB,EAClB,KAAc;QAEd,MAAM,QAAQ,GAA4B,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;QACrE,IAAI,KAAK;YAAE,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;QAElC,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM;YACN,UAAU,EAAE,KAAK;YACjB,QAAQ;SACT,CAAC;IACJ,CAAC;CACF"}
+{"version":3,"file":"pdf-loader.js","sourceRoot":"","sources":["../../src/loaders/pdf-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAoBhC,KAAK,UAAU,YAAY;IACzB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;QACtC,OAAO,GAAG,CAAC,OAAgC,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;IACzF,CAAC;AACH,CAAC;AAMD,MAAM,OAAO,SAAS;IACX,cAAc,GAAG,CAAC,KAAK,CAAC,CAAC;IACjB,UAAU,CAAU;IAErC,YAAY,OAA0B;QACpC,IAAI,CAAC,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,KAAK,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,MAAM,YAAY,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAExC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;QACzD,CAAC;QAED,OAAO,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC7D,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,QAAoB,EACpB,MAAc,EACd,MAAc;QAEd,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QAC1E,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC;QAEjC,MAAM,QAAQ,GAA4B,EAAE,KAAK,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;QACrE,IAAI,KAAK;YAAE,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;QAElC,OAAO;YACL;gBACE,EAAE,EAAE,MAAM,EAAE;gBACZ,OAAO,EAAE,MAAM,CAAC,IAAI;gBACpB,MAAM;gBACN,UAAU,EAAE,KAAK;gBACjB,QAAQ;aACT;SACF,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,QAAoB,EACpB,MAAc,EACd,MAAc;QAEd,MAAM,SAAS,GAAa,EAAE,CAAC;QAE/B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,qBAAqB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE;YACxE,UAAU,EAAE,KAAK,EAAE,QAAqB,EAAE,EAAE;gBAC1C,IAAI,OAAO,QAAQ,CAAC,cAAc,KAAK,UAAU;oBAAE,OAAO,EAAE,CAAC;gBAC7D,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,cAAc,EAAE,CAAC;gBACpD,MAAM,IAAI,GAAG,WAAW,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAChE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACrB,OAAO,IAAI,CAAC;YACd,CAAC;SACF,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC;QACjC,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAE3D,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;QAC7E,CAAC;QAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAC3B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,CACtE,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,qBAAqB,CACjC,QAAoB,EACpB,MAAc,EACd,MAAc,EACd,OAAmC;QAEnC,IAAI,CAAC;YACH,OAAO,MAAM,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;QACzC,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YACjE,MAAM,IAAI,KAAK,CAAC,+BAA+B,MAAM,MAAM,OAAO,EAAE,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;IAEO,YAAY,CAClB,OAAe,EACf,MAAc,EACd,UAAkB,EAClB,UAAkB,EAClB,KAAc;QAEd,MAAM,QAAQ,GAA4B,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;QACrE,IAAI,KAAK;YAAE,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;QAElC,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM;YACN,UAAU,EAAE,KAAK;YACjB,QAAQ;SACT,CAAC;IACJ,CAAC;CACF"}