@cogitator-ai/rag 0.1.3 → 0.1.8

package/README.md

@@ -64,7 +64,7 @@ for (const r of results) {
 | ---------------- | --------------------- | ------------ | ----------------------------- |
 | `TextLoader`     | `.txt`                | —            | Files and directories         |
 | `MarkdownLoader` | `.md`                 | —            | Strips frontmatter by default |
-| `JSONLoader`     | `.json`               | —            | Configurable JSON path        |
+| `JSONLoader`     | `.json`               | —            | Configurable content field    |
 | `CSVLoader`      | `.csv`                | `papaparse`  | Column selection, row mapping |
 | `HTMLLoader`     | `.html`, `.htm`       | `cheerio`    | CSS selector support          |
 | `PDFLoader`      | `.pdf`                | `pdf-parse`  | Text extraction from PDFs     |
@@ -75,7 +75,7 @@ import { MarkdownLoader, WebLoader, CSVLoader } from '@cogitator-ai/rag';
 
 const md = new MarkdownLoader({ stripFrontmatter: true });
 const web = new WebLoader({ selector: 'article' });
-const csv = new CSVLoader({ columns: ['title', 'body'] });
+const csv = new CSVLoader({ contentColumn: 'body', metadataColumns: ['title'] });
 ```
 
 ---

package/dist/chunkers/create-chunker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"create-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/create-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAKnG,wBAAgB,aAAa,CAC3B,MAAM,EAAE,cAAc,EACtB,gBAAgB,CAAC,EAAE,gBAAgB,GAClC,OAAO,GAAG,YAAY,CAyBxB"}
+{"version":3,"file":"create-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/create-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAKnG,wBAAgB,aAAa,CAC3B,MAAM,EAAE,cAAc,EACtB,gBAAgB,CAAC,EAAE,gBAAgB,GAClC,OAAO,GAAG,YAAY,CA8BxB"}

package/dist/chunkers/create-chunker.js

@@ -1,6 +1,6 @@
-import { FixedSizeChunker } from './fixed-chunker';
-import { RecursiveChunker } from './recursive-chunker';
-import { SemanticChunker } from './semantic-chunker';
+import { FixedSizeChunker } from './fixed-chunker.js';
+import { RecursiveChunker } from './recursive-chunker.js';
+import { SemanticChunker } from './semantic-chunker.js';
 export function createChunker(config, embeddingService) {
     switch (config.strategy) {
         case 'fixed':
@@ -23,6 +23,10 @@ export function createChunker(config, embeddingService) {
                 maxChunkSize: config.chunkSize,
             });
         }
+        default: {
+            const _exhaustive = config.strategy;
+            throw new Error(`Unknown chunking strategy: ${_exhaustive}`);
+        }
     }
 }
 //# sourceMappingURL=create-chunker.js.map

package/dist/chunkers/create-chunker.js.map

@@ -1 +1 @@
-{"version":3,"file":"create-chunker.js","sourceRoot":"","sources":["../../src/chunkers/create-chunker.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAErD,MAAM,UAAU,aAAa,CAC3B,MAAsB,EACtB,gBAAmC;IAEnC,QAAQ,MAAM,CAAC,QAAQ,EAAE,CAAC;QACxB,KAAK,OAAO;YACV,OAAO,IAAI,gBAAgB,CAAC;gBAC1B,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,YAAY,EAAE,MAAM,CAAC,YAAY;aAClC,CAAC,CAAC;QAEL,KAAK,WAAW;YACd,OAAO,IAAI,gBAAgB,CAAC;gBAC1B,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,UAAU,EAAE,MAAM,CAAC,UAAU;aAC9B,CAAC,CAAC;QAEL,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;YACjF,CAAC;YACD,OAAO,IAAI,eAAe,CAAC;gBACzB,gBAAgB;gBAChB,YAAY,EAAE,MAAM,CAAC,SAAS;aAC/B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;AACH,CAAC"}
+{"version":3,"file":"create-chunker.js","sourceRoot":"","sources":["../../src/chunkers/create-chunker.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACvD,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AAErD,MAAM,UAAU,aAAa,CAC3B,MAAsB,EACtB,gBAAmC;IAEnC,QAAQ,MAAM,CAAC,QAAQ,EAAE,CAAC;QACxB,KAAK,OAAO;YACV,OAAO,IAAI,gBAAgB,CAAC;gBAC1B,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,YAAY,EAAE,MAAM,CAAC,YAAY;aAClC,CAAC,CAAC;QAEL,KAAK,WAAW;YACd,OAAO,IAAI,gBAAgB,CAAC;gBAC1B,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,UAAU,EAAE,MAAM,CAAC,UAAU;aAC9B,CAAC,CAAC;QAEL,KAAK,UAAU,CAAC,CAAC,CAAC;YAChB,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;YACjF,CAAC;YACD,OAAO,IAAI,eAAe,CAAC;gBACzB,gBAAgB;gBAChB,YAAY,EAAE,MAAM,CAAC,SAAS;aAC/B,CAAC,CAAC;QACL,CAAC;QAED,OAAO,CAAC,CAAC,CAAC;YACR,MAAM,WAAW,GAAU,MAAM,CAAC,QAAQ,CAAC;YAC3C,MAAM,IAAI,KAAK,CAAC,8BAA8B,WAAW,EAAE,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/chunkers/fixed-chunker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fixed-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/fixed-chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAElE,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,qBAAa,gBAAiB,YAAW,OAAO;IAC9C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;gBAE1B,OAAO,EAAE,uBAAuB;IAK5C,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,aAAa,EAAE;CAuBzD"}
+{"version":3,"file":"fixed-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/fixed-chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAElE,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,qBAAa,gBAAiB,YAAW,OAAO;IAC9C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;gBAE1B,OAAO,EAAE,uBAAuB;IAc5C,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,aAAa,EAAE;CAuBzD"}

package/dist/chunkers/fixed-chunker.js

@@ -3,6 +3,15 @@ export class FixedSizeChunker {
     chunkSize;
     chunkOverlap;
     constructor(options) {
+        if (options.chunkSize <= 0) {
+            throw new Error('chunkSize must be a positive number');
+        }
+        if (options.chunkOverlap < 0) {
+            throw new Error('chunkOverlap must be non-negative');
+        }
+        if (options.chunkOverlap >= options.chunkSize) {
+            throw new Error('chunkOverlap must be less than chunkSize');
+        }
         this.chunkSize = options.chunkSize;
         this.chunkOverlap = options.chunkOverlap;
     }

package/dist/chunkers/fixed-chunker.js.map

@@ -1 +1 @@
-{"version":3,"file":"fixed-chunker.js","sourceRoot":"","sources":["../../src/chunkers/fixed-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAQhC,MAAM,OAAO,gBAAgB;IACV,SAAS,CAAS;IAClB,YAAY,CAAS;IAEtC,YAAY,OAAgC;QAC1C,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,IAAY,EAAE,UAAkB;QACpC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC;QAChD,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,IAAI,EAAE,CAAC;YACvD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAC1D,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,MAAM,EAAE;gBACZ,UAAU;gBACV,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC;gBAC/B,WAAW,EAAE,KAAK;gBAClB,SAAS,EAAE,GAAG;gBACd,KAAK,EAAE,KAAK,EAAE;aACf,CAAC,CAAC;YAEH,IAAI,GAAG,KAAK,IAAI,CAAC,MAAM;gBAAE,MAAM;QACjC,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}
+{"version":3,"file":"fixed-chunker.js","sourceRoot":"","sources":["../../src/chunkers/fixed-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAQhC,MAAM,OAAO,gBAAgB;IACV,SAAS,CAAS;IAClB,YAAY,CAAS;IAEtC,YAAY,OAAgC;QAC1C,IAAI,OAAO,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,IAAY,EAAE,UAAkB;QACpC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC;QAChD,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,IAAI,EAAE,CAAC;YACvD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAC1D,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,MAAM,EAAE;gBACZ,UAAU;gBACV,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC;gBAC/B,WAAW,EAAE,KAAK;gBAClB,SAAS,EAAE,GAAG;gBACd,KAAK,EAAE,KAAK,EAAE;aACf,CAAC,CAAC;YAEH,IAAI,GAAG,KAAK,IAAI,CAAC,MAAM;gBAAE,MAAM;QACjC,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}

package/dist/chunkers/index.d.ts

@@ -1,5 +1,5 @@
-export { FixedSizeChunker, type FixedSizeChunkerOptions } from './fixed-chunker';
-export { RecursiveChunker, type RecursiveChunkerOptions } from './recursive-chunker';
-export { SemanticChunker, type SemanticChunkerOptions } from './semantic-chunker';
-export { createChunker } from './create-chunker';
+export { FixedSizeChunker, type FixedSizeChunkerOptions } from './fixed-chunker.js';
+export { RecursiveChunker, type RecursiveChunkerOptions } from './recursive-chunker.js';
+export { SemanticChunker, type SemanticChunkerOptions } from './semantic-chunker.js';
+export { createChunker } from './create-chunker.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/chunkers/index.js

@@ -1,5 +1,5 @@
-export { FixedSizeChunker } from './fixed-chunker';
-export { RecursiveChunker } from './recursive-chunker';
-export { SemanticChunker } from './semantic-chunker';
-export { createChunker } from './create-chunker';
+export { FixedSizeChunker } from './fixed-chunker.js';
+export { RecursiveChunker } from './recursive-chunker.js';
+export { SemanticChunker } from './semantic-chunker.js';
+export { createChunker } from './create-chunker.js';
 //# sourceMappingURL=index.js.map

package/dist/chunkers/recursive-chunker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"recursive-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/recursive-chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAIlE,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,qBAAa,gBAAiB,YAAW,OAAO;IAC9C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAW;gBAE1B,OAAO,EAAE,uBAAuB;IAM5C,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,aAAa,EAAE;IAOxD,OAAO,CAAC,SAAS;IA+CjB,OAAO,CAAC,SAAS;IAQjB,OAAO,CAAC,gBAAgB;IAwFxB,OAAO,CAAC,sBAAsB;CAwB/B"}
+{"version":3,"file":"recursive-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/recursive-chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAIlE,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,qBAAa,gBAAiB,YAAW,OAAO;IAC9C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAW;gBAE1B,OAAO,EAAE,uBAAuB;IAe5C,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,aAAa,EAAE;IAOxD,OAAO,CAAC,SAAS;IA+CjB,OAAO,CAAC,SAAS;IAQjB,OAAO,CAAC,gBAAgB;IAwFxB,OAAO,CAAC,sBAAsB;CAwB/B"}

package/dist/chunkers/recursive-chunker.js

@@ -5,6 +5,15 @@ export class RecursiveChunker {
     chunkOverlap;
     separators;
     constructor(options) {
+        if (options.chunkSize <= 0) {
+            throw new Error('chunkSize must be a positive number');
+        }
+        if (options.chunkOverlap < 0) {
+            throw new Error('chunkOverlap must be non-negative');
+        }
+        if (options.chunkOverlap >= options.chunkSize) {
+            throw new Error('chunkOverlap must be less than chunkSize');
+        }
         this.chunkSize = options.chunkSize;
         this.chunkOverlap = options.chunkOverlap;
         this.separators = options.separators ?? DEFAULT_SEPARATORS;

package/dist/chunkers/recursive-chunker.js.map

@@ -1 +1 @@
-{"version":3,"file":"recursive-chunker.js","sourceRoot":"","sources":["../../src/chunkers/recursive-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,kBAAkB,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;AAQzD,MAAM,OAAO,gBAAgB;IACV,SAAS,CAAS;IAClB,YAAY,CAAS;IACrB,UAAU,CAAW;IAEtC,YAAY,OAAgC;QAC1C,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,kBAAkB,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,IAAY,EAAE,UAAkB;QACpC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACtC,OAAO,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;IAClD,CAAC;IAEO,SAAS,CAAC,IAAY,EAAE,cAAsB;QACpD,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAEjD,IAAI,cAAc,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;YAC7C,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QAElD,IAAI,SAAS,KAAK,EAAE,EAAE,CAAC;YACrB,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAEpC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,cAAc,GAAG,CAAC,CAAC,CAAC;QAClD,CAAC;QAED,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,OAAO,GAAG,EAAE,CAAC;QAEjB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAEhC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,GAAG,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YAE9D,IAAI,SAAS,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACvC,OAAO,GAAG,SAAS,CAAC;YACtB,CAAC;iBAAM,CAAC;gBACN,IAAI,OAAO;oBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAElC,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;oBACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,cAAc,GAAG,CAAC,CAAC,CAAC;oBAC1D,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;oBACzB,OAAO,GAAG,EAAE,CAAC;gBACf,CAAC;qBAAM,CAAC;oBACN,OAAO,GAAG,IAAI,CAAC;gBACjB,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,OAAO;YAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAElC,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,SAAS,CAAC,IAAY;QAC5B,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACrD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;QACjD,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,gBAAgB,CAAC,KAAe,EAAE,UAAkB;QAC1D,IAAI,IAAI,CAAC,YAAY,KAAK,CAAC,EAAE,CAAC;YAC5B,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE;gBAC9B,MAAM,KAAK,GAAkB;oBAC3B,EAAE,EAAE,MAAM,EAAE;oBACZ,UAAU;oBACV,OAAO;oBACP,WAAW,EAAE,MAAM;oBACnB,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM;oBAClC,KAAK,EAAE,CAAC;iBACT,CAAC;gBACF,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC;gBACzB,OAAO,KAAK,CAAC;YACf,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,CAAC,GAAG,CAAC,CAAC;QAEV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YACxB,IAAI,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACvB,IAAI,QAAQ,GAAG,CAAC,CAAC;YAEjB,OAAO,CAAC,GAAG,QAAQ,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;gBACnC,MAAM,IAAI,GAAG,OAAO,GAAG,KAAK,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC;gBAC3C,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;oBAClC,OAAO,GAAG,IAAI,CAAC;oBACf,QAAQ,EAAE,CAAC;gBACb,CAAC;qBAAM,CAAC;oBACN,MAAM;gBACR,CAAC;YACH,CAAC;YAED,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAChC,OAAO,EACP,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAC/D,CAAC;YACF,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,MAAM,EAAE;gBACZ,UAAU;gBACV,OAAO;gBACP,WAAW,EAAE,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS;gBACvD,SAAS,EAAE,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,OAAO,CAAC,MAAM;gBACxE,KAAK,EAAE,KAAK,EAAE;aACf,CAAC,CAAC;YAEH,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC;YAE5B,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YACjE,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC,CAAC;YAEjE,CAAC,IAAI,QAAQ,CAAC;YAEd,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;gBACrB,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACjC,MAAM,WAAW,GAAG,WAAW,GAAG,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAErD,IAAI,WAAW,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;oBACzC,MAAM,CAAC,IAAI,CAAC;wBACV,EAAE,EAAE,MAAM,EAAE;wBACZ,UAAU;wBACV,OAAO,EAAE,WAAW;wBACpB,WAAW,EAAE,SAAS,GAAG,YAAY;wBACrC,SAAS,EAAE,SAAS,GAAG,YAAY,GAAG,WAAW,CAAC,MAAM;wBACxD,KAAK,EAAE,KAAK,EAAE;qBACf,CAAC,CAAC;oBACH,MAAM;gBACR,CAAC;gBAED,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;gBAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,sBAAsB,CAC3C,OAAO,EACP,UAAU,EACV,SAAS,GAAG,YAAY,EACxB,KAAK,CACN,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC1B,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,sBAAsB,CAC5B,KAAe,EACf,UAAkB,EAClB,UAAkB,EAClB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,MAAM,GAAG,UAAU,CAAC;QACxB,IAAI,KAAK,GAAG,UAAU,CAAC;QAEvB,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,MAAM,EAAE;gBACZ,UAAU;gBACV,OAAO;gBACP,WAAW,EAAE,MAAM;gBACnB,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM;gBAClC,KAAK,EAAE,KAAK,EAAE;aACf,CAAC,CAAC;YACH,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC;QAC3B,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}
+{"version":3,"file":"recursive-chunker.js","sourceRoot":"","sources":["../../src/chunkers/recursive-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,kBAAkB,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;AAQzD,MAAM,OAAO,gBAAgB;IACV,SAAS,CAAS;IAClB,YAAY,CAAS;IACrB,UAAU,CAAW;IAEtC,YAAY,OAAgC;QAC1C,IAAI,OAAO,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,kBAAkB,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,IAAY,EAAE,UAAkB;QACpC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjC,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACtC,OAAO,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;IAClD,CAAC;IAEO,SAAS,CAAC,IAAY,EAAE,cAAsB;QACpD,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,CAAC,IAAI,CAAC,CAAC;QAEjD,IAAI,cAAc,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;YAC7C,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;QAElD,IAAI,SAAS,KAAK,EAAE,EAAE,CAAC;YACrB,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC9B,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAEpC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,cAAc,GAAG,CAAC,CAAC,CAAC;QAClD,CAAC;QAED,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,IAAI,OAAO,GAAG,EAAE,CAAC;QAEjB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAEhC,MAAM,SAAS,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,GAAG,SAAS,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;YAE9D,IAAI,SAAS,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBACvC,OAAO,GAAG,SAAS,CAAC;YACtB,CAAC;iBAAM,CAAC;gBACN,IAAI,OAAO;oBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBAElC,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;oBACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,cAAc,GAAG,CAAC,CAAC,CAAC;oBAC1D,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;oBACzB,OAAO,GAAG,EAAE,CAAC;gBACf,CAAC;qBAAM,CAAC;oBACN,OAAO,GAAG,IAAI,CAAC;gBACjB,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,OAAO;YAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAElC,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,SAAS,CAAC,IAAY;QAC5B,MAAM,MAAM,GAAa,EAAE,CAAC;QAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACrD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;QACjD,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,gBAAgB,CAAC,KAAe,EAAE,UAAkB;QAC1D,IAAI,IAAI,CAAC,YAAY,KAAK,CAAC,EAAE,CAAC;YAC5B,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE;gBAC9B,MAAM,KAAK,GAAkB;oBAC3B,EAAE,EAAE,MAAM,EAAE;oBACZ,UAAU;oBACV,OAAO;oBACP,WAAW,EAAE,MAAM;oBACnB,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM;oBAClC,KAAK,EAAE,CAAC;iBACT,CAAC;gBACF,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC;gBACzB,OAAO,KAAK,CAAC;YACf,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,IAAI,CAAC,GAAG,CAAC,CAAC;QAEV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YACxB,IAAI,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACvB,IAAI,QAAQ,GAAG,CAAC,CAAC;YAEjB,OAAO,CAAC,GAAG,QAAQ,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;gBACnC,MAAM,IAAI,GAAG,OAAO,GAAG,KAAK,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC;gBAC3C,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;oBAClC,OAAO,GAAG,IAAI,CAAC;oBACf,QAAQ,EAAE,CAAC;gBACb,CAAC;qBAAM,CAAC;oBACN,MAAM;gBACR,CAAC;YACH,CAAC;YAED,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAChC,OAAO,EACP,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAC/D,CAAC;YACF,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,MAAM,EAAE;gBACZ,UAAU;gBACV,OAAO;gBACP,WAAW,EAAE,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS;gBACvD,SAAS,EAAE,CAAC,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC,GAAG,OAAO,CAAC,MAAM;gBACxE,KAAK,EAAE,KAAK,EAAE;aACf,CAAC,CAAC;YAEH,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC;YAE5B,MAAM,YAAY,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YACjE,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,YAAY,CAAC,CAAC;YAEjE,CAAC,IAAI,QAAQ,CAAC;YAEd,IAAI,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;gBACrB,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACjC,MAAM,WAAW,GAAG,WAAW,GAAG,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAErD,IAAI,WAAW,CAAC,MAAM,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;oBACzC,MAAM,CAAC,IAAI,CAAC;wBACV,EAAE,EAAE,MAAM,EAAE;wBACZ,UAAU;wBACV,OAAO,EAAE,WAAW;wBACpB,WAAW,EAAE,SAAS,GAAG,YAAY;wBACrC,SAAS,EAAE,SAAS,GAAG,YAAY,GAAG,WAAW,CAAC,MAAM;wBACxD,KAAK,EAAE,KAAK,EAAE;qBACf,CAAC,CAAC;oBACH,MAAM;gBACR,CAAC;gBAED,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;gBAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,sBAAsB,CAC3C,OAAO,EACP,UAAU,EACV,SAAS,GAAG,YAAY,EACxB,KAAK,CACN,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC1B,MAAM;YACR,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,sBAAsB,CAC5B,KAAe,EACf,UAAkB,EAClB,UAAkB,EAClB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,MAAM,GAAG,UAAU,CAAC;QACxB,IAAI,KAAK,GAAG,UAAU,CAAC;QAEvB,KAAK,MAAM,OAAO,IAAI,KAAK,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,MAAM,EAAE;gBACZ,UAAU;gBACV,OAAO;gBACP,WAAW,EAAE,MAAM;gBACnB,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM;gBAClC,KAAK,EAAE,KAAK,EAAE;aACf,CAAC,CAAC;YACH,MAAM,IAAI,OAAO,CAAC,MAAM,CAAC;QAC3B,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}

package/dist/chunkers/semantic-chunker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"semantic-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/semantic-chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAEzF,MAAM,WAAW,sBAAsB;IACrC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAwBD,qBAAa,eAAgB,YAAW,YAAY;IAClD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAmB;IACpD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAC7C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;gBAE1B,OAAO,EAAE,sBAAsB;IAOrC,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAkCvE,OAAO,CAAC,gBAAgB;IAqCxB,OAAO,CAAC,WAAW;IA8CnB,OAAO,CAAC,eAAe;CA8DxB"}
+{"version":3,"file":"semantic-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/semantic-chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAEzF,MAAM,WAAW,sBAAsB;IACrC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAyBD,qBAAa,eAAgB,YAAW,YAAY;IAClD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAmB;IACpD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAC7C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;gBAE1B,OAAO,EAAE,sBAAsB;IAOrC,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAkCvE,OAAO,CAAC,gBAAgB;IAqCxB,OAAO,CAAC,WAAW;IA8CnB,OAAO,CAAC,eAAe;CA8DxB"}

package/dist/chunkers/semantic-chunker.js

@@ -1,9 +1,10 @@
 import { nanoid } from 'nanoid';
 function cosineSimilarity(a, b) {
+    const len = Math.min(a.length, b.length);
     let dot = 0;
     let normA = 0;
     let normB = 0;
-    for (let i = 0; i < a.length; i++) {
+    for (let i = 0; i < len; i++) {
         dot += a[i] * b[i];
         normA += a[i] * a[i];
         normB += b[i] * b[i];
@@ -96,11 +97,11 @@ export class SemanticChunker {
     buildChunks(groups, originalText, documentId) {
         const chunks = [];
         let searchFrom = 0;
-        for (let i = 0; i < groups.length; i++) {
-            const content = groups[i].join(' ');
+        for (const group of groups) {
+            const content = group.join(' ');
             if (content.length <= this.maxChunkSize) {
-                const startOffset = originalText.indexOf(groups[i][0], searchFrom);
-                const lastSentence = groups[i][groups[i].length - 1];
+                const startOffset = originalText.indexOf(group[0], searchFrom);
+                const lastSentence = group[group.length - 1];
                 const lastStart = originalText.indexOf(lastSentence, startOffset >= 0 ? startOffset : 0);
                 const endOffset = lastStart >= 0 ? lastStart + lastSentence.length : startOffset + content.length;
                 chunks.push({
@@ -114,7 +115,7 @@ export class SemanticChunker {
                 searchFrom = endOffset;
             }
             else {
-                const subChunks = this.splitLargeGroup(groups[i], originalText, documentId, searchFrom, chunks.length);
+                const subChunks = this.splitLargeGroup(group, originalText, documentId, searchFrom, chunks.length);
                 chunks.push(...subChunks);
                 if (subChunks.length > 0) {
                     searchFrom = subChunks[subChunks.length - 1].endOffset;

package/dist/chunkers/semantic-chunker.js.map

@@ -1 +1 @@
-{"version":3,"file":"semantic-chunker.js","sourceRoot":"","sources":["../../src/chunkers/semantic-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAUhC,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IAChD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC;AACvC,CAAC;AAED,MAAM,iBAAiB,GAAG,eAAe,CAAC;AAE1C,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,IAAI;SACR,KAAK,CAAC,iBAAiB,CAAC;SACxB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,OAAO,eAAe;IACT,gBAAgB,CAAmB;IACnC,mBAAmB,CAAS;IAC5B,YAAY,CAAS;IACrB,YAAY,CAAS;IAEtC,YAAY,OAA+B;QACzC,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;QACjD,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,IAAI,GAAG,CAAC;QAC9D,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,GAAG,CAAC;QAChD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,UAAkB;QAC1C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjD,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEtC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAErE,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,YAAY,CAAC,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,MAAM,GAAe,EAAE,CAAC;QAC9B,IAAI,YAAY,GAAa,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QAE5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,IAAI,YAAY,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAC/C,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAC1B,YAAY,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAE1B,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAC7C,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IACpD,CAAC;IAEO,gBAAgB,CAAC,MAAkB;QACzC,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC;YAAE,OAAO,MAAM,CAAC;QAEtC,MAAM,MAAM,GAAe,EAAE,CAAC;QAC9B,IAAI,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrC,MAAM,cAAc,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;YAEhE,IAAI,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,cAAc,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClF,OAAO,GAAG,CAAC,GAAG,OAAO,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YACvC,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACrB,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,IAAI,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAChC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClE,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,CAAC;gBACpD,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvB,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,WAAW,CACjB,MAAkB,EAClB,YAAoB,EACpB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAEpC,IAAI,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACxC,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;gBACnE,MAAM,YAAY,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACrD,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,YAAY,EAAE,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACzF,MAAM,SAAS,GACb,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;gBAElF,MAAM,CAAC,IAAI,CAAC;oBACV,EAAE,EAAE,MAAM,EAAE;oBACZ,UAAU;oBACV,OAAO;oBACP,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC;oBACrC,SAAS;oBACT,KAAK,EAAE,MAAM,CAAC,MAAM;iBACrB,CAAC,CAAC;gBAEH,UAAU,GAAG,SAAS,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CACpC,MAAM,CAAC,CAAC,CAAC,EACT,YAAY,EACZ,UAAU,EACV,UAAU,EACV,MAAM,CAAC,MAAM,CACd,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC1B,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzB,UAAU,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;gBACzD,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,eAAe,CACrB,SAAmB,EACnB,YAAoB,EACpB,UAAkB,EAClB,UAAkB,EAClB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,gBAAgB,GAAa,EAAE,CAAC;QACpC,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,KAAK,GAAG,UAAU,CAAC;QACvB,IAAI,MAAM,GAAG,UAAU,CAAC;QAExB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,SAAS,GAAG,aAAa,GAAG,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;YAE1F,IAAI,SAAS,GAAG,IAAI,CAAC,YAAY,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACjE,MAAM,OAAO,GAAG,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC3C,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;gBACtE,MAAM,YAAY,GAAG,gBAAgB,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACnE,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,YAAY,EAAE,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACzF,MAAM,SAAS,GACb,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;gBAElF,MAAM,CAAC,IAAI,CAAC;oBACV,EAAE,EAAE,MAAM,EAAE;oBACZ,UAAU;oBACV,OAAO;oBACP,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC;oBACrC,SAAS;oBACT,KAAK,EAAE,KAAK,EAAE;iBACf,CAAC,CAAC;gBAEH,MAAM,GAAG,SAAS,CAAC;gBACnB,gBAAgB,GAAG,CAAC,QAAQ,CAAC,CAAC;gBAC9B,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACN,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAChC,aAAa,GAAG,SAAS,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,OAAO,GAAG,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC3C,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;YACtE,MAAM,YAAY,GAAG,gBAAgB,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACnE,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,YAAY,EAAE,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACzF,MAAM,SAAS,GACb,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;YAElF,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,MAAM,EAAE;gBACZ,UAAU;gBACV,OAAO;gBACP,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC;gBACrC,SAAS;gBACT,KAAK,EAAE,KAAK,EAAE;aACf,CAAC,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}
+{"version":3,"file":"semantic-chunker.js","sourceRoot":"","sources":["../../src/chunkers/semantic-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAUhC,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IAChD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;IACzC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC;AACvC,CAAC;AAED,MAAM,iBAAiB,GAAG,eAAe,CAAC;AAE1C,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,IAAI;SACR,KAAK,CAAC,iBAAiB,CAAC;SACxB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,OAAO,eAAe;IACT,gBAAgB,CAAmB;IACnC,mBAAmB,CAAS;IAC5B,YAAY,CAAS;IACrB,YAAY,CAAS;IAEtC,YAAY,OAA+B;QACzC,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;QACjD,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,IAAI,GAAG,CAAC;QAC9D,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,GAAG,CAAC;QAChD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,UAAkB;QAC1C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjD,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEtC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QAErE,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,YAAY,CAAC,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,MAAM,GAAe,EAAE,CAAC;QAC9B,IAAI,YAAY,GAAa,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QAE5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,IAAI,YAAY,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAC/C,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAC1B,YAAY,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAE1B,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAC7C,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IACpD,CAAC;IAEO,gBAAgB,CAAC,MAAkB;QACzC,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC;YAAE,OAAO,MAAM,CAAC;QAEtC,MAAM,MAAM,GAAe,EAAE,CAAC;QAC9B,IAAI,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrC,MAAM,cAAc,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;YAEhE,IAAI,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,cAAc,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClF,OAAO,GAAG,CAAC,GAAG,OAAO,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YACvC,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACrB,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,IAAI,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAChC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClE,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,CAAC;gBACpD,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvB,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,WAAW,CACjB,MAAkB,EAClB,YAAoB,EACpB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAEhC,IAAI,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACxC,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;gBAC/D,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBAC7C,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,YAAY,EAAE,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACzF,MAAM,SAAS,GACb,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;gBAElF,MAAM,CAAC,IAAI,CAAC;oBACV,EAAE,EAAE,MAAM,EAAE;oBACZ,UAAU;oBACV,OAAO;oBACP,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC;oBACrC,SAAS;oBACT,KAAK,EAAE,MAAM,CAAC,MAAM;iBACrB,CAAC,CAAC;gBAEH,UAAU,GAAG,SAAS,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CACpC,KAAK,EACL,YAAY,EACZ,UAAU,EACV,UAAU,EACV,MAAM,CAAC,MAAM,CACd,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC1B,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzB,UAAU,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;gBACzD,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,eAAe,CACrB,SAAmB,EACnB,YAAoB,EACpB,UAAkB,EAClB,UAAkB,EAClB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,gBAAgB,GAAa,EAAE,CAAC;QACpC,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,KAAK,GAAG,UAAU,CAAC;QACvB,IAAI,MAAM,GAAG,UAAU,CAAC;QAExB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,MAAM,SAAS,GAAG,aAAa,GAAG,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;YAE1F,IAAI,SAAS,GAAG,IAAI,CAAC,YAAY,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACjE,MAAM,OAAO,GAAG,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC3C,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;gBACtE,MAAM,YAAY,GAAG,gBAAgB,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACnE,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,YAAY,EAAE,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACzF,MAAM,SAAS,GACb,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;gBAElF,MAAM,CAAC,IAAI,CAAC;oBACV,EAAE,EAAE,MAAM,EAAE;oBACZ,UAAU;oBACV,OAAO;oBACP,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC;oBACrC,SAAS;oBACT,KAAK,EAAE,KAAK,EAAE;iBACf,CAAC,CAAC;gBAEH,MAAM,GAAG,SAAS,CAAC;gBACnB,gBAAgB,GAAG,CAAC,QAAQ,CAAC,CAAC;gBAC9B,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACN,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAChC,aAAa,GAAG,SAAS,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,OAAO,GAAG,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YAC3C,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;YACtE,MAAM,YAAY,GAAG,gBAAgB,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACnE,MAAM,SAAS,GAAG,YAAY,CAAC,OAAO,CAAC,YAAY,EAAE,WAAW,IAAI,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YACzF,MAAM,SAAS,GACb,SAAS,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC;YAElF,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,MAAM,EAAE;gBACZ,UAAU;gBACV,OAAO;gBACP,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,CAAC;gBACrC,SAAS;gBACT,KAAK,EAAE,KAAK,EAAE;aACf,CAAC,CAAC;QACL,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}

package/dist/index.d.ts

@@ -1,11 +1,11 @@
 export declare const VERSION = "0.1.0";
-export { RAGPipeline, type RAGPipelineDeps } from './rag-pipeline';
-export { RAGPipelineBuilder } from './rag-builder';
-export * from './loaders';
-export * from './chunkers';
-export * from './retrievers';
-export * from './rerankers';
-export * from './schema';
-export { createSearchTool, createIngestTool, ragTools, type RAGTool } from './tools';
+export { RAGPipeline, type RAGPipelineDeps } from './rag-pipeline.js';
+export { RAGPipelineBuilder } from './rag-builder.js';
+export * from './loaders/index.js';
+export * from './chunkers/index.js';
+export * from './retrievers/index.js';
+export * from './rerankers/index.js';
+export * from './schema.js';
+export { createSearchTool, createIngestTool, ragTools, type RAGTool } from './tools.js';
 export type { RAGDocument, DocumentChunk, DocumentLoader, Chunker, AsyncChunker, ChunkingConfig, ChunkingStrategy, Retriever, RetrievalConfig, RetrievalStrategy, RetrievalResult, Reranker, RAGPipelineConfig, EmbeddingService, EmbeddingAdapter, } from '@cogitator-ai/types';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.js

@@ -1,10 +1,10 @@
 export const VERSION = '0.1.0';
-export { RAGPipeline } from './rag-pipeline';
-export { RAGPipelineBuilder } from './rag-builder';
-export * from './loaders';
-export * from './chunkers';
-export * from './retrievers';
-export * from './rerankers';
-export * from './schema';
-export { createSearchTool, createIngestTool, ragTools } from './tools';
+export { RAGPipeline } from './rag-pipeline.js';
+export { RAGPipelineBuilder } from './rag-builder.js';
+export * from './loaders/index.js';
+export * from './chunkers/index.js';
+export * from './retrievers/index.js';
+export * from './rerankers/index.js';
+export * from './schema.js';
+export { createSearchTool, createIngestTool, ragTools } from './tools.js';
 //# sourceMappingURL=index.js.map

package/dist/loaders/index.d.ts

@@ -1,14 +1,14 @@
-export { TextLoader } from './text-loader';
-export { MarkdownLoader } from './markdown-loader';
-export type { MarkdownLoaderOptions } from './markdown-loader';
-export { JSONLoader } from './json-loader';
-export type { JSONLoaderOptions } from './json-loader';
-export { CSVLoader } from './csv-loader';
-export type { CSVLoaderOptions } from './csv-loader';
-export { HTMLLoader } from './html-loader';
-export type { HTMLLoaderOptions } from './html-loader';
-export { PDFLoader } from './pdf-loader';
-export type { PDFLoaderOptions } from './pdf-loader';
-export { WebLoader } from './web-loader';
-export type { WebLoaderOptions } from './web-loader';
+export { TextLoader } from './text-loader.js';
+export { MarkdownLoader } from './markdown-loader.js';
+export type { MarkdownLoaderOptions } from './markdown-loader.js';
+export { JSONLoader } from './json-loader.js';
+export type { JSONLoaderOptions } from './json-loader.js';
+export { CSVLoader } from './csv-loader.js';
+export type { CSVLoaderOptions } from './csv-loader.js';
+export { HTMLLoader } from './html-loader.js';
+export type { HTMLLoaderOptions } from './html-loader.js';
+export { PDFLoader } from './pdf-loader.js';
+export type { PDFLoaderOptions } from './pdf-loader.js';
+export { WebLoader } from './web-loader.js';
+export type { WebLoaderOptions } from './web-loader.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/loaders/index.js

@@ -1,8 +1,8 @@
-export { TextLoader } from './text-loader';
-export { MarkdownLoader } from './markdown-loader';
-export { JSONLoader } from './json-loader';
-export { CSVLoader } from './csv-loader';
-export { HTMLLoader } from './html-loader';
-export { PDFLoader } from './pdf-loader';
-export { WebLoader } from './web-loader';
+export { TextLoader } from './text-loader.js';
+export { MarkdownLoader } from './markdown-loader.js';
+export { JSONLoader } from './json-loader.js';
+export { CSVLoader } from './csv-loader.js';
+export { HTMLLoader } from './html-loader.js';
+export { PDFLoader } from './pdf-loader.js';
+export { WebLoader } from './web-loader.js';
 //# sourceMappingURL=index.js.map

package/dist/loaders/json-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"json-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/json-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAIvE,MAAM,WAAW,iBAAiB;IAChC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;CAC3B;AAED,qBAAa,UAAW,YAAW,cAAc;IAC/C,QAAQ,CAAC,cAAc,WAAY;IACnC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAW;gBAE/B,OAAO,CAAC,EAAE,iBAAiB;IAKjC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IASlD,OAAO,CAAC,cAAc;IActB,OAAO,CAAC,cAAc;IActB,OAAO,CAAC,eAAe;CAWxB"}
+{"version":3,"file":"json-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/json-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAIvE,MAAM,WAAW,iBAAiB;IAChC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,cAAc,CAAC,EAAE,MAAM,EAAE,CAAC;CAC3B;AAED,qBAAa,UAAW,YAAW,cAAc;IAC/C,QAAQ,CAAC,cAAc,WAAY;IACnC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAW;gBAE/B,OAAO,CAAC,EAAE,iBAAiB;IAKjC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IASlD,OAAO,CAAC,cAAc;IAsBtB,OAAO,CAAC,cAAc;IActB,OAAO,CAAC,eAAe;CAWxB"}

package/dist/loaders/json-loader.js

@@ -18,6 +18,14 @@ export class JSONLoader {
         return items.map((item) => this.itemToDocument(item, filePath));
     }
     itemToDocument(item, source) {
+        if (typeof item !== 'object' || item === null || Array.isArray(item)) {
+            return {
+                id: nanoid(),
+                content: String(item),
+                source,
+                sourceType: 'json',
+            };
+        }
         const obj = item;
         const content = this.extractContent(obj);
         const metadata = this.extractMetadata(obj);

package/dist/loaders/json-loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"json-loader.js","sourceRoot":"","sources":["../../src/loaders/json-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,cAAc,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAOnD,MAAM,OAAO,UAAU;IACZ,cAAc,GAAG,CAAC,MAAM,CAAC,CAAC;IAClB,YAAY,CAAU;IACtB,cAAc,CAAY;IAE3C,YAAY,OAA2B;QACrC,IAAI,CAAC,YAAY,GAAG,OAAO,EAAE,YAAY,CAAC;QAC1C,IAAI,CAAC,cAAc,GAAG,OAAO,EAAE,cAAc,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC9C,MAAM,IAAI,GAAY,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEtC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAClD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;IAClE,CAAC;IAEO,cAAc,CAAC,IAAa,EAAE,MAAc;QAClD,MAAM,GAAG,GAAG,IAA+B,CAAC;QAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QAE3C,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM;YACN,UAAU,EAAE,MAAM;YAClB,GAAG,CAAC,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC;SAClE,CAAC;IACJ,CAAC;IAEO,cAAc,CAAC,GAA4B;QACjD,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY,IAAI,GAAG,EAAE,CAAC;YAClD,OAAO,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;QACxC,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;gBACjB,OAAO,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACtC,CAAC;IAEO,eAAe,CAAC,GAA4B;QAClD,IAAI,CAAC,IAAI,CAAC,cAAc;YAAE,OAAO,SAAS,CAAC;QAE3C,MAAM,QAAQ,GAA4B,EAAE,CAAC;QAC7C,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxC,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;gBACjB,QAAQ,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF"}
+{"version":3,"file":"json-loader.js","sourceRoot":"","sources":["../../src/loaders/json-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,cAAc,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAOnD,MAAM,OAAO,UAAU;IACZ,cAAc,GAAG,CAAC,MAAM,CAAC,CAAC;IAClB,YAAY,CAAU;IACtB,cAAc,CAAY;IAE3C,YAAY,OAA2B;QACrC,IAAI,CAAC,YAAY,GAAG,OAAO,EAAE,YAAY,CAAC;QAC1C,IAAI,CAAC,cAAc,GAAG,OAAO,EAAE,cAAc,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC9C,MAAM,IAAI,GAAY,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAEtC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAClD,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAC;IAClE,CAAC;IAEO,cAAc,CAAC,IAAa,EAAE,MAAc;QAClD,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;YACrE,OAAO;gBACL,EAAE,EAAE,MAAM,EAAE;gBACZ,OAAO,EAAE,MAAM,CAAC,IAAI,CAAC;gBACrB,MAAM;gBACN,UAAU,EAAE,MAAM;aACnB,CAAC;QACJ,CAAC;QACD,MAAM,GAAG,GAAG,IAA+B,CAAC;QAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;QACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;QAE3C,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM;YACN,UAAU,EAAE,MAAM;YAClB,GAAG,CAAC,QAAQ,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC;SAClE,CAAC;IACJ,CAAC;IAEO,cAAc,CAAC,GAA4B;QACjD,IAAI,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC,YAAY,IAAI,GAAG,EAAE,CAAC;YAClD,OAAO,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;QACxC,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;gBACjB,OAAO,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACtC,CAAC;IAEO,eAAe,CAAC,GAA4B;QAClD,IAAI,CAAC,IAAI,CAAC,cAAc;YAAE,OAAO,SAAS,CAAC;QAE3C,MAAM,QAAQ,GAA4B,EAAE,CAAC;QAC7C,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxC,IAAI,KAAK,IAAI,GAAG,EAAE,CAAC;gBACjB,QAAQ,CAAC,KAAK,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC;YAC/B,CAAC;QACH,CAAC;QACD,OAAO,QAAQ,CAAC;IAClB,CAAC;CACF"}

package/dist/loaders/markdown-loader.js

@@ -22,7 +22,7 @@ export class MarkdownLoader {
         let content = raw;
         let metadata;
         if (this.stripFrontmatter) {
-            const match = raw.match(FRONTMATTER_RE);
+            const match = FRONTMATTER_RE.exec(raw);
             if (match) {
                 metadata = this.parseFrontmatter(match[1]);
                 content = raw.slice(match[0].length);
@@ -39,7 +39,7 @@ export class MarkdownLoader {
     parseFrontmatter(raw) {
         const result = {};
         for (const line of raw.split('\n')) {
-            const match = line.match(/^(\w[\w\s-]*?):\s*(.+)$/);
+            const match = /^(\w[\w\s-]*?):\s*(.+)$/.exec(line);
             if (match) {
                 result[match[1].trim()] = match[2].trim();
             }

package/dist/loaders/markdown-loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"markdown-loader.js","sourceRoot":"","sources":["../../src/loaders/markdown-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;AAE7C,MAAM,cAAc,GAAG,kCAAkC,CAAC;AAM1D,MAAM,OAAO,cAAc;IAChB,cAAc,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACvB,gBAAgB,CAAU;IAE3C,YAAY,OAA+B;QACzC,IAAI,CAAC,gBAAgB,GAAG,OAAO,EAAE,gBAAgB,IAAI,KAAK,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;QAElC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,QAAgB;QACrC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC9C,IAAI,OAAO,GAAG,GAAG,CAAC;QAClB,IAAI,QAA6C,CAAC;QAElD,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;YACxC,IAAI,KAAK,EAAE,CAAC;gBACV,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC3C,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YACvC,CAAC;QACH,CAAC;QAED,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM,EAAE,QAAQ;YAChB,UAAU,EAAE,UAAU;YACtB,GAAG,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,CAAC;SAC9B,CAAC;IACJ,CAAC;IAEO,gBAAgB,CAAC,GAAW;QAClC,MAAM,MAAM,GAA4B,EAAE,CAAC;QAC3C,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;YACpD,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC5C,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAe;QACzC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;QACvC,MAAM,IAAI,GAAkB,EAAE,CAAC;QAE/B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;YACxD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YAEtC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClC,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;gBAClB,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}
+{"version":3,"file":"markdown-loader.js","sourceRoot":"","sources":["../../src/loaders/markdown-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAC3D,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC;AAE7C,MAAM,cAAc,GAAG,kCAAkC,CAAC;AAM1D,MAAM,OAAO,cAAc;IAChB,cAAc,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IACvB,gBAAgB,CAAU;IAE3C,YAAY,OAA+B;QACzC,IAAI,CAAC,gBAAgB,GAAG,OAAO,EAAE,gBAAgB,IAAI,KAAK,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;QAElC,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;YACvB,OAAO,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;QACtC,CAAC;QAED,OAAO,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;IACzC,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,QAAgB;QACrC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC9C,IAAI,OAAO,GAAG,GAAG,CAAC;QAClB,IAAI,QAA6C,CAAC;QAElD,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC1B,MAAM,KAAK,GAAG,cAAc,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACvC,IAAI,KAAK,EAAE,CAAC;gBACV,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC3C,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;YACvC,CAAC;QACH,CAAC;QAED,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM,EAAE,QAAQ;YAChB,UAAU,EAAE,UAAU;YACtB,GAAG,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,CAAC;SAC9B,CAAC;IACJ,CAAC;IAEO,gBAAgB,CAAC,GAAW;QAClC,MAAM,MAAM,GAA4B,EAAE,CAAC;QAC3C,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,yBAAyB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACnD,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC5C,CAAC;QACH,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,OAAe;QACzC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;QACvC,MAAM,IAAI,GAAkB,EAAE,CAAC;QAE/B,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;YACxD,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,SAAS;YAEtC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;YACtC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC;YAClC,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;gBAClB,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;CACF"}

package/dist/loaders/pdf-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pdf-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/pdf-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAuBvE,MAAM,WAAW,gBAAgB;IAC/B,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,qBAAa,SAAU,YAAW,cAAc;IAC9C,QAAQ,CAAC,cAAc,WAAW;IAClC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAU;gBAEzB,OAAO,CAAC,EAAE,gBAAgB;IAIhC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YAYpC,kBAAkB;YAsBlB,cAAc;IA4B5B,OAAO,CAAC,YAAY;CAkBrB"}
+{"version":3,"file":"pdf-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/pdf-loader.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAuBvE,MAAM,WAAW,gBAAgB;IAC/B,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,qBAAa,SAAU,YAAW,cAAc;IAC9C,QAAQ,CAAC,cAAc,WAAW;IAClC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAU;gBAEzB,OAAO,CAAC,EAAE,gBAAgB;IAIhC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;YAYpC,kBAAkB;YAsBlB,cAAc;IAmB5B,OAAO,CAAC,YAAY;CAkBrB"}

package/dist/loaders/pdf-loader.js

@@ -42,19 +42,13 @@ export class PDFLoader {
         ];
     }
     async loadSplitPages(pdfParse, buffer, source) {
-        const pageContents = [];
-        const result = await pdfParse(buffer, {
-            pagerender(pageData) {
-                pageContents[pageData.pageIndex] = '';
-                return '';
-            },
-        });
+        const result = await pdfParse(buffer);
         const title = result.info?.Title;
-        if (pageContents.length === 0) {
-            const pages = result.text.split(/\f/);
-            return pages.map((text, i) => this.buildPageDoc(text.trim(), source, i + 1, result.numpages, title));
+        const pages = result.text.split(/\f/).filter((p) => p.trim().length > 0);
+        if (pages.length === 0) {
+            return [this.buildPageDoc(result.text, source, 1, result.numpages, title)];
         }
-        return pageContents.map((text, i) => this.buildPageDoc(text || '', source, i + 1, result.numpages, title));
+        return pages.map((text, i) => this.buildPageDoc(text.trim(), source, i + 1, result.numpages, title));
     }
     buildPageDoc(content, source, pageNumber, totalPages, title) {
         const metadata = { pageNumber, totalPages };

package/dist/loaders/pdf-loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"pdf-loader.js","sourceRoot":"","sources":["../../src/loaders/pdf-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAehC,KAAK,UAAU,YAAY;IACzB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;QACtC,OAAO,GAAG,CAAC,OAAgC,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;IACzF,CAAC;AACH,CAAC;AAMD,MAAM,OAAO,SAAS;IACX,cAAc,GAAG,CAAC,KAAK,CAAC,CAAC;IACjB,UAAU,CAAU;IAErC,YAAY,OAA0B;QACpC,IAAI,CAAC,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,KAAK,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,MAAM,YAAY,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAExC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;QACzD,CAAC;QAED,OAAO,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC7D,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,QAAoB,EACpB,MAAc,EACd,MAAc;QAEd,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC;QAEjC,MAAM,QAAQ,GAA4B,EAAE,KAAK,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;QACrE,IAAI,KAAK;YAAE,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;QAElC,OAAO;YACL;gBACE,EAAE,EAAE,MAAM,EAAE;gBACZ,OAAO,EAAE,MAAM,CAAC,IAAI;gBACpB,MAAM;gBACN,UAAU,EAAE,KAAK;gBACjB,QAAQ;aACT;SACF,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,QAAoB,EACpB,MAAc,EACd,MAAc;QAEd,MAAM,YAAY,GAAa,EAAE,CAAC;QAElC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,MAAM,EAAE;YACpC,UAAU,CAAC,QAA+B;gBACxC,YAAY,CAAC,QAAQ,CAAC,SAAS,CAAC,GAAG,EAAE,CAAC;gBACtC,OAAO,EAAE,CAAC;YACZ,CAAC;SACF,CAAC,CAAC;QAEH,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC;QAEjC,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACtC,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAC3B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,CACtE,CAAC;QACJ,CAAC;QAED,OAAO,YAAY,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAClC,IAAI,CAAC,YAAY,CAAC,IAAI,IAAI,EAAE,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,CACrE,CAAC;IACJ,CAAC;IAEO,YAAY,CAClB,OAAe,EACf,MAAc,EACd,UAAkB,EAClB,UAAkB,EAClB,KAAc;QAEd,MAAM,QAAQ,GAA4B,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;QACrE,IAAI,KAAK;YAAE,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;QAElC,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM;YACN,UAAU,EAAE,KAAK;YACjB,QAAQ;SACT,CAAC;IACJ,CAAC;CACF"}
+{"version":3,"file":"pdf-loader.js","sourceRoot":"","sources":["../../src/loaders/pdf-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAehC,KAAK,UAAU,YAAY;IACzB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAC;QACtC,OAAO,GAAG,CAAC,OAAgC,CAAC;IAC9C,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;IACzF,CAAC;AACH,CAAC;AAMD,MAAM,OAAO,SAAS;IACX,cAAc,GAAG,CAAC,KAAK,CAAC,CAAC;IACjB,UAAU,CAAU;IAErC,YAAY,OAA0B;QACpC,IAAI,CAAC,UAAU,GAAG,OAAO,EAAE,UAAU,IAAI,KAAK,CAAC;IACjD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,MAAM,YAAY,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;QACjC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAExC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;QACzD,CAAC;QAED,OAAO,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,CAAC;IAC7D,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,QAAoB,EACpB,MAAc,EACd,MAAc;QAEd,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC;QAEjC,MAAM,QAAQ,GAA4B,EAAE,KAAK,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;QACrE,IAAI,KAAK;YAAE,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;QAElC,OAAO;YACL;gBACE,EAAE,EAAE,MAAM,EAAE;gBACZ,OAAO,EAAE,MAAM,CAAC,IAAI;gBACpB,MAAM;gBACN,UAAU,EAAE,KAAK;gBACjB,QAAQ;aACT;SACF,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,QAAoB,EACpB,MAAc,EACd,MAAc;QAEd,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC;QAEjC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAEzE,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC,CAAC;QAC7E,CAAC;QAED,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,EAAE,CAC3B,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,MAAM,CAAC,QAAQ,EAAE,KAAK,CAAC,CACtE,CAAC;IACJ,CAAC;IAEO,YAAY,CAClB,OAAe,EACf,MAAc,EACd,UAAkB,EAClB,UAAkB,EAClB,KAAc;QAEd,MAAM,QAAQ,GAA4B,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC;QACrE,IAAI,KAAK;YAAE,QAAQ,CAAC,KAAK,GAAG,KAAK,CAAC;QAElC,OAAO;YACL,EAAE,EAAE,MAAM,EAAE;YACZ,OAAO;YACP,MAAM;YACN,UAAU,EAAE,KAAK;YACjB,QAAQ;SACT,CAAC;IACJ,CAAC;CACF"}

package/dist/loaders/web-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"web-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/web-loader.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAGvE,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAED,qBAAa,SAAU,YAAW,cAAc;IAC9C,QAAQ,CAAC,cAAc,WAAqB;IAC5C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAa;IACxC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAyB;gBAEtC,OAAO,CAAC,EAAE,gBAAgB;IAKhC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;CAcnD"}
+{"version":3,"file":"web-loader.d.ts","sourceRoot":"","sources":["../../src/loaders/web-loader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAGvE,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CAClC;AAED,qBAAa,SAAU,YAAW,cAAc;IAC9C,QAAQ,CAAC,cAAc,WAAqB;IAC5C,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAa;IACxC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAyB;gBAEtC,OAAO,CAAC,EAAE,gBAAgB;IAKhC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;CAcnD"}

package/dist/loaders/web-loader.js

@@ -1,5 +1,4 @@
-import { nanoid } from 'nanoid';
-import { HTMLLoader } from './html-loader';
+import { HTMLLoader } from './html-loader.js';
 export class WebLoader {
     supportedTypes = ['http', 'https'];
     htmlLoader;
@@ -17,7 +16,7 @@ export class WebLoader {
         }
         const html = await response.text();
         const doc = await this.htmlLoader.parseHTML(html, source, 'web');
-        return [{ ...doc, id: nanoid() }];
+        return [doc];
     }
 }
 //# sourceMappingURL=web-loader.js.map

package/dist/loaders/web-loader.js.map

@@ -1 +1 @@
-{"version":3,"file":"web-loader.js","sourceRoot":"","sources":["../../src/loaders/web-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAEhC,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAO3C,MAAM,OAAO,SAAS;IACX,cAAc,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC3B,UAAU,CAAa;IACvB,OAAO,CAA0B;IAElD,YAAY,OAA0B;QACpC,IAAI,CAAC,UAAU,GAAG,IAAI,UAAU,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;QAClE,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE;YACnC,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,EAAE;SAC5B,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,mBAAmB,MAAM,KAAK,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QAC1F,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;QAEjE,OAAO,CAAC,EAAE,GAAG,GAAG,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,CAAC,CAAC;IACpC,CAAC;CACF"}
+{"version":3,"file":"web-loader.js","sourceRoot":"","sources":["../../src/loaders/web-loader.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,eAAe,CAAC;AAO3C,MAAM,OAAO,SAAS;IACX,cAAc,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC3B,UAAU,CAAa;IACvB,OAAO,CAA0B;IAElD,YAAY,OAA0B;QACpC,IAAI,CAAC,UAAU,GAAG,IAAI,UAAU,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC;QAClE,IAAI,CAAC,OAAO,GAAG,OAAO,EAAE,OAAO,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,MAAc;QACvB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,MAAM,EAAE;YACnC,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,EAAE;SAC5B,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,mBAAmB,MAAM,KAAK,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;QAC1F,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;QAEjE,OAAO,CAAC,GAAG,CAAC,CAAC;IACf,CAAC;CACF"}

package/dist/rag-builder.d.ts

@@ -1,6 +1,6 @@
 import type { DocumentLoader, Chunker, AsyncChunker, EmbeddingService, EmbeddingAdapter, Retriever, Reranker } from '@cogitator-ai/types';
-import { type RAGPipelineConfigInput } from './schema';
-import { RAGPipeline } from './rag-pipeline';
+import { type RAGPipelineConfigInput } from './schema.js';
+import { RAGPipeline } from './rag-pipeline.js';
 export declare class RAGPipelineBuilder {
     private loader?;
     private chunker?;

package/dist/rag-builder.js

@@ -1,7 +1,7 @@
-import { RAGPipelineConfigSchema } from './schema';
-import { RAGPipeline } from './rag-pipeline';
-import { createChunker } from './chunkers/create-chunker';
-import { SimilarityRetriever } from './retrievers/similarity-retriever';
+import { RAGPipelineConfigSchema } from './schema.js';
+import { RAGPipeline } from './rag-pipeline.js';
+import { createChunker } from './chunkers/create-chunker.js';
+import { SimilarityRetriever } from './retrievers/similarity-retriever.js';
 export class RAGPipelineBuilder {
     loader;
     chunker;
@@ -66,7 +66,7 @@ export class RAGPipelineBuilder {
     }
     resolveConfig() {
         if (!this.configInput) {
-            throw new Error('config is required — call withConfig() or provide a chunker');
+            throw new Error('config is required — call withConfig() before build()');
         }
         return RAGPipelineConfigSchema.parse(this.configInput);
     }

package/dist/rag-builder.js.map

@@ -1 +1 @@
-{"version":3,"file":"rag-builder.js","sourceRoot":"","sources":["../src/rag-builder.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,uBAAuB,EAA+B,MAAM,UAAU,CAAC;AAChF,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AAExE,MAAM,OAAO,kBAAkB;IACrB,MAAM,CAAkB;IACxB,OAAO,CAA0B;IACjC,gBAAgB,CAAoB;IACpC,gBAAgB,CAAoB;IACpC,SAAS,CAAa;IACtB,QAAQ,CAAY;IACpB,WAAW,CAA0B;IAE7C,UAAU,CAAC,MAAsB;QAC/B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,WAAW,CAAC,OAA+B;QACzC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,oBAAoB,CAAC,OAAyB;QAC5C,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC;QAChC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,oBAAoB,CAAC,OAAyB;QAC5C,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC;QAChC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,aAAa,CAAC,SAAoB;QAChC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,YAAY,CAAC,QAAkB;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,UAAU,CAAC,MAA8B;QACvC,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC;QAC1B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;QAC3E,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;QAC/F,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;QAC/F,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACtF,MAAM,SAAS,GACb,IAAI,CAAC,SAAS;YACd,IAAI,mBAAmB,CAAC;gBACtB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;gBACvC,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;aACxC,CAAC,CAAC;QAEL,OAAO,IAAI,WAAW,CAAC,MAAM,EAAE;YAC7B,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO;YACP,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,SAAS;YACT,QAAQ,EAAE,IAAI,CAAC,QAAQ;SACxB,CAAC,CAAC;IACL,CAAC;IAEO,aAAa;QACnB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,6DAA6D,CAAC,CAAC;QACjF,CAAC;QACD,OAAO,uBAAuB,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACzD,CAAC;CACF"}
+{"version":3,"file":"rag-builder.js","sourceRoot":"","sources":["../src/rag-builder.ts"],"names":[],"mappings":"AAUA,OAAO,EAAE,uBAAuB,EAA+B,MAAM,UAAU,CAAC;AAChF,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAC7C,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAC1D,OAAO,EAAE,mBAAmB,EAAE,MAAM,mCAAmC,CAAC;AAExE,MAAM,OAAO,kBAAkB;IACrB,MAAM,CAAkB;IACxB,OAAO,CAA0B;IACjC,gBAAgB,CAAoB;IACpC,gBAAgB,CAAoB;IACpC,SAAS,CAAa;IACtB,QAAQ,CAAY;IACpB,WAAW,CAA0B;IAE7C,UAAU,CAAC,MAAsB;QAC/B,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,WAAW,CAAC,OAA+B;QACzC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,oBAAoB,CAAC,OAAyB;QAC5C,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC;QAChC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,oBAAoB,CAAC,OAAyB;QAC5C,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC;QAChC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,aAAa,CAAC,SAAoB;QAChC,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,YAAY,CAAC,QAAkB;QAC7B,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,UAAU,CAAC,MAA8B;QACvC,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC;QAC1B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK;QACH,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;QAC3E,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;QAC/F,CAAC;QACD,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,2EAA2E,CAAC,CAAC;QAC/F,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;QACpC,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,aAAa,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACtF,MAAM,SAAS,GACb,IAAI,CAAC,SAAS;YACd,IAAI,mBAAmB,CAAC;gBACtB,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;gBACvC,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;aACxC,CAAC,CAAC;QAEL,OAAO,IAAI,WAAW,CAAC,MAAM,EAAE;YAC7B,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO;YACP,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;YACvC,SAAS;YACT,QAAQ,EAAE,IAAI,CAAC,QAAQ;SACxB,CAAC,CAAC;IACL,CAAC;IAEO,aAAa;QACnB,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;QAC3E,CAAC;QACD,OAAO,uBAAuB,CAAC,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACzD,CAAC;CACF"}

package/dist/rag-pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"rag-pipeline.d.ts","sourceRoot":"","sources":["../src/rag-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,cAAc,EACd,OAAO,EACP,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,EAChB,SAAS,EACT,QAAQ,EACR,iBAAiB,EACjB,eAAe,EACf,eAAe,EAGhB,MAAM,qBAAqB,CAAC;AAG7B,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,cAAc,CAAC;IACvB,OAAO,EAAE,OAAO,GAAG,YAAY,CAAC;IAChC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,SAAS,EAAE,SAAS,CAAC;IACrB,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACrB;AAED,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoB;IAC3C,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAkB;IACvC,OAAO,CAAC,KAAK,CAAkE;gBAEnE,MAAM,EAAE,iBAAiB,EAAE,IAAI,EAAE,eAAe;IAKtD,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAsBtE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAWzF,QAAQ;;;;;YAIM,aAAa;YAKb,UAAU;CAkBzB"}
+{"version":3,"file":"rag-pipeline.d.ts","sourceRoot":"","sources":["../src/rag-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,cAAc,EACd,OAAO,EACP,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,EAChB,SAAS,EACT,QAAQ,EACR,iBAAiB,EACjB,eAAe,EACf,eAAe,EAGhB,MAAM,qBAAqB,CAAC;AAG7B,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,cAAc,CAAC;IACvB,OAAO,EAAE,OAAO,GAAG,YAAY,CAAC;IAChC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,SAAS,EAAE,SAAS,CAAC;IACrB,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACrB;AAED,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoB;IAC3C,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAkB;IACvC,OAAO,CAAC,KAAK,CAAkE;gBAEnE,MAAM,EAAE,iBAAiB,EAAE,IAAI,EAAE,eAAe;IAKtD,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IA8BtE,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAWzF,QAAQ;;;;;YAIM,aAAa;YAIb,UAAU;CAkBzB"}

package/dist/rag-pipeline.js

@@ -1,4 +1,4 @@
-import { RAGPipelineConfigSchema } from './schema';
+import { RAGPipelineConfigSchema } from './schema.js';
 export class RAGPipeline {
     config;
     deps;
@@ -12,8 +12,13 @@ export class RAGPipeline {
         let totalChunks = 0;
         for (const doc of documents) {
             const chunks = await this.chunkDocument(doc);
+            if (chunks.length === 0)
+                continue;
             const texts = chunks.map((c) => c.content);
             const vectors = await this.deps.embeddingService.embedBatch(texts);
+            if (vectors.length !== chunks.length) {
+                throw new Error(`Embedding count mismatch: got ${vectors.length} vectors for ${chunks.length} chunks`);
+            }
             for (let i = 0; i < chunks.length; i++) {
                 await this.storeChunk(chunks[i], vectors[i], doc);
             }
@@ -35,8 +40,7 @@ export class RAGPipeline {
         return { ...this.stats };
     }
     async chunkDocument(doc) {
-        const result = this.deps.chunker.chunk(doc.content, doc.id);
-        return result instanceof Promise ? result : result;
+        return this.deps.chunker.chunk(doc.content, doc.id);
     }
     async storeChunk(chunk, vector, doc) {
         const result = await this.deps.embeddingAdapter.addEmbedding({

package/dist/rag-pipeline.js.map

@@ -1 +1 @@
-{"version":3,"file":"rag-pipeline.js","sourceRoot":"","sources":["../src/rag-pipeline.ts"],"names":[],"mappings":"AAcA,OAAO,EAAE,uBAAuB,EAAE,MAAM,UAAU,CAAC;AAWnD,MAAM,OAAO,WAAW;IACL,MAAM,CAAoB;IAC1B,IAAI,CAAkB;IAC/B,KAAK,GAAG,EAAE,iBAAiB,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAC;IAE/E,YAAY,MAAyB,EAAE,IAAqB;QAC1D,IAAI,CAAC,MAAM,GAAG,uBAAuB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACpD,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,MAAc;QACzB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACtD,IAAI,WAAW,GAAG,CAAC,CAAC;QAEpB,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;YAC5B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YAC7C,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAC3C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAEnE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YACpD,CAAC;YAED,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC;QAC/B,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,iBAAiB,IAAI,SAAS,CAAC,MAAM,CAAC;QACjD,IAAI,CAAC,KAAK,CAAC,YAAY,IAAI,WAAW,CAAC;QAEvC,OAAO,EAAE,SAAS,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,OAAkC;QAC1D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAClE,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAE9B,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC;YACzD,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC9E,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,QAAQ;QACN,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,GAAgB;QAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAC5D,OAAO,MAAM,YAAY,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IACrD,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,KAAoB,EAAE,MAAgB,EAAE,GAAgB;QAC/E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC;YAC3D,QAAQ,EAAE,KAAK,CAAC,EAAE;YAClB,UAAU,EAAE,UAAU;YACtB,MAAM;YACN,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,QAAQ,EAAE;gBACR,GAAG,KAAK,CAAC,QAAQ;gBACjB,UAAU,EAAE,GAAG,CAAC,EAAE;gBAClB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,KAAK,EAAE,KAAK,CAAC,KAAK;aACnB;SACF,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;CACF"}
+{"version":3,"file":"rag-pipeline.js","sourceRoot":"","sources":["../src/rag-pipeline.ts"],"names":[],"mappings":"AAcA,OAAO,EAAE,uBAAuB,EAAE,MAAM,UAAU,CAAC;AAWnD,MAAM,OAAO,WAAW;IACL,MAAM,CAAoB;IAC1B,IAAI,CAAkB;IAC/B,KAAK,GAAG,EAAE,iBAAiB,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAC;IAE/E,YAAY,MAAyB,EAAE,IAAqB;QAC1D,IAAI,CAAC,MAAM,GAAG,uBAAuB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACpD,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,MAAc;QACzB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACtD,IAAI,WAAW,GAAG,CAAC,CAAC;QAEpB,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;YAC5B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;YAC7C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;gBAAE,SAAS;YAElC,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAC3C,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAEnE,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;gBACrC,MAAM,IAAI,KAAK,CACb,iCAAiC,OAAO,CAAC,MAAM,gBAAgB,MAAM,CAAC,MAAM,SAAS,CACtF,CAAC;YACJ,CAAC;YAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YACpD,CAAC;YAED,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC;QAC/B,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,iBAAiB,IAAI,SAAS,CAAC,MAAM,CAAC;QACjD,IAAI,CAAC,KAAK,CAAC,YAAY,IAAI,WAAW,CAAC;QAEvC,OAAO,EAAE,SAAS,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,OAAkC;QAC1D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAClE,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAE9B,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC;YACzD,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC9E,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,QAAQ;QACN,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,GAAgB;QAC1C,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;IACtD,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,KAAoB,EAAE,MAAgB,EAAE,GAAgB;QAC/E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC;YAC3D,QAAQ,EAAE,KAAK,CAAC,EAAE;YAClB,UAAU,EAAE,UAAU;YACtB,MAAM;YACN,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,QAAQ,EAAE;gBACR,GAAG,KAAK,CAAC,QAAQ;gBACjB,UAAU,EAAE,GAAG,CAAC,EAAE;gBAClB,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,KAAK,EAAE,KAAK,CAAC,KAAK;aACnB;SACF,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;CACF"}

package/dist/rerankers/cohere-reranker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cohere-reranker.d.ts","sourceRoot":"","sources":["../../src/rerankers/cohere-reranker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAErE,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAYD,qBAAa,cAAe,YAAW,QAAQ;IAC7C,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAAS;gBAEV,MAAM,EAAE,oBAAoB;IAKlC,MAAM,CACV,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,eAAe,EAAE,EAC1B,IAAI,CAAC,EAAE,MAAM,GACZ,OAAO,CAAC,eAAe,EAAE,CAAC;CA+B9B"}
+{"version":3,"file":"cohere-reranker.d.ts","sourceRoot":"","sources":["../../src/rerankers/cohere-reranker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAErE,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAYD,qBAAa,cAAe,YAAW,QAAQ;IAC7C,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAAS;gBAEV,MAAM,EAAE,oBAAoB;IAKlC,MAAM,CACV,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,eAAe,EAAE,EAC1B,IAAI,CAAC,EAAE,MAAM,GACZ,OAAO,CAAC,eAAe,EAAE,CAAC;CAmC9B"}

package/dist/rerankers/cohere-reranker.js

@@ -8,6 +8,8 @@ export class CohereReranker {
         this.model = config.model ?? DEFAULT_MODEL;
     }
     async rerank(query, results, topN) {
+        if (results.length === 0)
+            return [];
         const n = topN ?? results.length;
         const response = await fetch(COHERE_RERANK_URL, {
             method: 'POST',
@@ -27,7 +29,9 @@ export class CohereReranker {
             throw new Error(`Cohere rerank failed (${response.status}): ${text}`);
         }
         const data = (await response.json());
-        const mapped = data.results.map(({ index, relevance_score }) => ({
+        const mapped = data.results
+            .filter(({ index }) => index >= 0 && index < results.length)
+            .map(({ index, relevance_score }) => ({
             ...results[index],
             score: relevance_score,
         }));

package/dist/rerankers/cohere-reranker.js.map

@@ -1 +1 @@
-{"version":3,"file":"cohere-reranker.js","sourceRoot":"","sources":["../../src/rerankers/cohere-reranker.ts"],"names":[],"mappings":"AAcA,MAAM,aAAa,GAAG,aAAa,CAAC;AACpC,MAAM,iBAAiB,GAAG,kCAAkC,CAAC;AAE7D,MAAM,OAAO,cAAc;IACjB,MAAM,CAAS;IACf,KAAK,CAAS;IAEtB,YAAY,MAA4B;QACtC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,aAAa,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,MAAM,CACV,KAAa,EACb,OAA0B,EAC1B,IAAa;QAEb,MAAM,CAAC,GAAG,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC;QAEjC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,iBAAiB,EAAE;YAC9C,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;aACvC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,KAAK;gBACL,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;gBACxC,KAAK,EAAE,CAAC;aACT,CAAC;SACH,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,yBAAyB,QAAQ,CAAC,MAAM,MAAM,IAAI,EAAE,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAyB,CAAC;QAE7D,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC,CAAC;YAC/D,GAAG,OAAO,CAAC,KAAK,CAAC;YACjB,KAAK,EAAE,eAAe;SACvB,CAAC,CAAC,CAAC;QAEJ,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,CAAC;CACF"}
+{"version":3,"file":"cohere-reranker.js","sourceRoot":"","sources":["../../src/rerankers/cohere-reranker.ts"],"names":[],"mappings":"AAcA,MAAM,aAAa,GAAG,aAAa,CAAC;AACpC,MAAM,iBAAiB,GAAG,kCAAkC,CAAC;AAE7D,MAAM,OAAO,cAAc;IACjB,MAAM,CAAS;IACf,KAAK,CAAS;IAEtB,YAAY,MAA4B;QACtC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;QAC5B,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,aAAa,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,MAAM,CACV,KAAa,EACb,OAA0B,EAC1B,IAAa;QAEb,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEpC,MAAM,CAAC,GAAG,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC;QAEjC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,iBAAiB,EAAE;YAC9C,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;aACvC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;gBACnB,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,KAAK;gBACL,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC;gBACxC,KAAK,EAAE,CAAC;aACT,CAAC;SACH,CAAC,CAAC;QAEH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CAAC,yBAAyB,QAAQ,CAAC,MAAM,MAAM,IAAI,EAAE,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,IAAI,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAyB,CAAC;QAE7D,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO;aACxB,MAAM,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,KAAK,IAAI,CAAC,IAAI,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;aAC3D,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,EAAE,EAAE,CAAC,CAAC;YACpC,GAAG,OAAO,CAAC,KAAK,CAAC;YACjB,KAAK,EAAE,eAAe;SACvB,CAAC,CAAC,CAAC;QAEN,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,CAAC;CACF"}

package/dist/rerankers/index.d.ts

@@ -1,3 +1,3 @@
-export { LLMReranker, type LLMRerankerConfig } from './llm-reranker';
-export { CohereReranker, type CohereRerankerConfig } from './cohere-reranker';
+export { LLMReranker, type LLMRerankerConfig } from './llm-reranker.js';
+export { CohereReranker, type CohereRerankerConfig } from './cohere-reranker.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/rerankers/index.js

@@ -1,3 +1,3 @@
-export { LLMReranker } from './llm-reranker';
-export { CohereReranker } from './cohere-reranker';
+export { LLMReranker } from './llm-reranker.js';
+export { CohereReranker } from './cohere-reranker.js';
 //# sourceMappingURL=index.js.map

package/dist/rerankers/llm-reranker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"llm-reranker.d.ts","sourceRoot":"","sources":["../../src/rerankers/llm-reranker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAErE,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CACjD;AAED,qBAAa,WAAY,YAAW,QAAQ;IAC1C,OAAO,CAAC,UAAU,CAAsC;gBAE5C,MAAM,EAAE,iBAAiB;IAI/B,MAAM,CACV,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,eAAe,EAAE,EAC1B,IAAI,CAAC,EAAE,MAAM,GACZ,OAAO,CAAC,eAAe,EAAE,CAAC;IAwB7B,OAAO,CAAC,WAAW;IAcnB,OAAO,CAAC,WAAW;CAiBpB"}
+{"version":3,"file":"llm-reranker.d.ts","sourceRoot":"","sources":["../../src/rerankers/llm-reranker.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,QAAQ,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAErE,MAAM,WAAW,iBAAiB;IAChC,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,CAAC,CAAC;CACjD;AAED,qBAAa,WAAY,YAAW,QAAQ;IAC1C,OAAO,CAAC,UAAU,CAAsC;gBAE5C,MAAM,EAAE,iBAAiB;IAI/B,MAAM,CACV,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,eAAe,EAAE,EAC1B,IAAI,CAAC,EAAE,MAAM,GACZ,OAAO,CAAC,eAAe,EAAE,CAAC;IA0B7B,OAAO,CAAC,WAAW;IAcnB,OAAO,CAAC,WAAW;CAiBpB"}

package/dist/rerankers/llm-reranker.js

@@ -4,6 +4,8 @@ export class LLMReranker {
         this.generateFn = config.generateFn;
     }
     async rerank(query, results, topN) {
+        if (results.length === 0)
+            return [];
         try {
             const prompt = this.buildPrompt(query, results);
             const response = await this.generateFn(prompt);
@@ -15,7 +17,7 @@ export class LLMReranker {
             scored.sort((a, b) => b.llmScore - a.llmScore);
             const reranked = scored.map(({ result, llmScore }) => ({
                 ...result,
-                score: llmScore / 10,
+                score: Math.max(0, Math.min(1, llmScore / 10)),
             }));
             return topN ? reranked.slice(0, topN) : reranked;
         }
@@ -37,7 +39,7 @@ export class LLMReranker {
         ].join('\n');
     }
     parseScores(response, count) {
-        const jsonMatch = response.match(/\[[\s\S]*\]/);
+        const jsonMatch = /\[[\s\S]*?\]/.exec(response);
         if (!jsonMatch)
             throw new Error('No JSON array found in response');
         const parsed = JSON.parse(jsonMatch[0]);

package/dist/rerankers/llm-reranker.js.map

@@ -1 +1 @@
-{"version":3,"file":"llm-reranker.js","sourceRoot":"","sources":["../../src/rerankers/llm-reranker.ts"],"names":[],"mappings":"AAMA,MAAM,OAAO,WAAW;IACd,UAAU,CAAsC;IAExD,YAAY,MAAyB;QACnC,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,MAAM,CACV,KAAa,EACb,OAA0B,EAC1B,IAAa;QAEb,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAChD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YAE1D,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACvC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC;gBAChD,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YACjD,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;YAC/C,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;gBACrD,GAAG,MAAM;gBACT,KAAK,EAAE,QAAQ,GAAG,EAAE;aACrB,CAAC,CAAC,CAAC;YAEJ,OAAO,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;QACnD,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,QAAQ,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC;YAC9B,OAAO,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;QACnD,CAAC;IACH,CAAC;IAEO,WAAW,CAAC,KAAa,EAAE,OAA0B;QAC3D,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEvE,OAAO;YACL,kEAAkE;YAClE,kEAAkE;YAClE,EAAE;YACF,UAAU,KAAK,EAAE;YACjB,EAAE;YACF,YAAY;YACZ,IAAI;SACL,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC;IAEO,WAAW,CAAC,QAAgB,EAAE,KAAa;QACjD,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAChD,IAAI,CAAC,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;QAEnE,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QACjD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAExE,OAAO,MAAM,CAAC,MAAM,CAClB,CAAC,IAAI,EAA4C,EAAE,CACjD,OAAO,IAAI,KAAK,QAAQ;YACxB,IAAI,KAAK,IAAI;YACb,OAAQ,IAAgC,CAAC,KAAK,KAAK,QAAQ;YAC3D,OAAQ,IAAgC,CAAC,KAAK,KAAK,QAAQ;YAC1D,IAA0B,CAAC,KAAK,IAAI,CAAC;YACrC,IAA0B,CAAC,KAAK,GAAG,KAAK,CAC5C,CAAC;IACJ,CAAC;CACF"}
+{"version":3,"file":"llm-reranker.js","sourceRoot":"","sources":["../../src/rerankers/llm-reranker.ts"],"names":[],"mappings":"AAMA,MAAM,OAAO,WAAW;IACd,UAAU,CAAsC;IAExD,YAAY,MAAyB;QACnC,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,UAAU,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,MAAM,CACV,KAAa,EACb,OAA0B,EAC1B,IAAa;QAEb,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEpC,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;YAChD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YAE1D,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACvC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,CAAC;gBAChD,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,KAAK,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YACjD,CAAC,CAAC,CAAC;YAEH,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;YAC/C,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;gBACrD,GAAG,MAAM;gBACT,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,QAAQ,GAAG,EAAE,CAAC,CAAC;aAC/C,CAAC,CAAC,CAAC;YAEJ,OAAO,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;QACnD,CAAC;QAAC,MAAM,CAAC;YACP,MAAM,QAAQ,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC;YAC9B,OAAO,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;QACnD,CAAC;IACH,CAAC;IAEO,WAAW,CAAC,KAAa,EAAE,OAA0B;QAC3D,MAAM,IAAI,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEvE,OAAO;YACL,kEAAkE;YAClE,kEAAkE;YAClE,EAAE;YACF,UAAU,KAAK,EAAE;YACjB,EAAE;YACF,YAAY;YACZ,IAAI;SACL,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC;IAEO,WAAW,CAAC,QAAgB,EAAE,KAAa;QACjD,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAChD,IAAI,CAAC,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;QAEnE,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QACjD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAExE,OAAO,MAAM,CAAC,MAAM,CAClB,CAAC,IAAI,EAA4C,EAAE,CACjD,OAAO,IAAI,KAAK,QAAQ;YACxB,IAAI,KAAK,IAAI;YACb,OAAQ,IAAgC,CAAC,KAAK,KAAK,QAAQ;YAC3D,OAAQ,IAAgC,CAAC,KAAK,KAAK,QAAQ;YAC1D,IAA0B,CAAC,KAAK,IAAI,CAAC;YACrC,IAA0B,CAAC,KAAK,GAAG,KAAK,CAC5C,CAAC;IACJ,CAAC;CACF"}

package/dist/retrievers/create-retriever.d.ts

@@ -1,4 +1,4 @@
-import type { Retriever, EmbeddingAdapter, EmbeddingService } from '@cogitator-ai/types';
+import type { Retriever, EmbeddingAdapter, EmbeddingService, HybridSearchWeights } from '@cogitator-ai/types';
 import type { HybridSearch } from '@cogitator-ai/memory';
 interface BaseRetrieverDeps {
     embeddingAdapter: EmbeddingAdapter;
@@ -18,10 +18,7 @@ interface MMRDeps extends BaseRetrieverDeps {
 interface HybridDeps {
     strategy: 'hybrid';
     hybridSearch: HybridSearch;
-    weights?: {
-        bm25: number;
-        vector: number;
-    };
+    weights?: HybridSearchWeights;
     topK?: number;
     threshold?: number;
 }
@@ -29,7 +26,6 @@ interface MultiQueryDeps {
     strategy: 'multi-query';
     baseRetriever: Retriever;
     expandQuery: (query: string) => Promise<string[]>;
-    queryCount?: number;
 }
 export type CreateRetrieverConfig = SimilarityDeps | MMRDeps | HybridDeps | MultiQueryDeps;
 export declare function createRetriever(config: CreateRetrieverConfig): Retriever;

package/dist/retrievers/create-retriever.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"create-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/create-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,gBAAgB,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACzF,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAMzD,UAAU,iBAAiB;IACzB,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,gBAAgB,EAAE,gBAAgB,CAAC;CACpC;AAED,UAAU,cAAe,SAAQ,iBAAiB;IAChD,QAAQ,EAAE,YAAY,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,OAAQ,SAAQ,iBAAiB;IACzC,QAAQ,EAAE,KAAK,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,UAAU;IAClB,QAAQ,EAAE,QAAQ,CAAC;IACnB,YAAY,EAAE,YAAY,CAAC;IAC3B,OAAO,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC3C,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,cAAc;IACtB,QAAQ,EAAE,aAAa,CAAC;IACxB,aAAa,EAAE,SAAS,CAAC;IACzB,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,MAAM,qBAAqB,GAAG,cAAc,GAAG,OAAO,GAAG,UAAU,GAAG,cAAc,CAAC;AAE3F,wBAAgB,eAAe,CAAC,MAAM,EAAE,qBAAqB,GAAG,SAAS,CAkCxE"}
+{"version":3,"file":"create-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/create-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,SAAS,EACT,gBAAgB,EAChB,gBAAgB,EAChB,mBAAmB,EACpB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAMzD,UAAU,iBAAiB;IACzB,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,gBAAgB,EAAE,gBAAgB,CAAC;CACpC;AAED,UAAU,cAAe,SAAQ,iBAAiB;IAChD,QAAQ,EAAE,YAAY,CAAC;IACvB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,OAAQ,SAAQ,iBAAiB;IACzC,QAAQ,EAAE,KAAK,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,UAAU;IAClB,QAAQ,EAAE,QAAQ,CAAC;IACnB,YAAY,EAAE,YAAY,CAAC;IAC3B,OAAO,CAAC,EAAE,mBAAmB,CAAC;IAC9B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,UAAU,cAAc;IACtB,QAAQ,EAAE,aAAa,CAAC;IACxB,aAAa,EAAE,SAAS,CAAC;IACzB,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;CACnD;AAED,MAAM,MAAM,qBAAqB,GAAG,cAAc,GAAG,OAAO,GAAG,UAAU,GAAG,cAAc,CAAC;AAE3F,wBAAgB,eAAe,CAAC,MAAM,EAAE,qBAAqB,GAAG,SAAS,CAiCxE"}

package/dist/retrievers/create-retriever.js

@@ -1,7 +1,7 @@
-import { SimilarityRetriever } from './similarity-retriever';
-import { MMRRetriever } from './mmr-retriever';
-import { HybridRetriever } from './hybrid-retriever';
-import { MultiQueryRetriever } from './multi-query-retriever';
+import { SimilarityRetriever } from './similarity-retriever.js';
+import { MMRRetriever } from './mmr-retriever.js';
+import { HybridRetriever } from './hybrid-retriever.js';
+import { MultiQueryRetriever } from './multi-query-retriever.js';
 export function createRetriever(config) {
     switch (config.strategy) {
         case 'similarity':
@@ -30,7 +30,6 @@ export function createRetriever(config) {
             return new MultiQueryRetriever({
                 baseRetriever: config.baseRetriever,
                 expandQuery: config.expandQuery,
-                defaultQueryCount: config.queryCount,
             });
     }
 }

package/dist/retrievers/create-retriever.js.map

@@ -1 +1 @@
-{"version":3,"file":"create-retriever.js","sourceRoot":"","sources":["../../src/retrievers/create-retriever.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAqC9D,MAAM,UAAU,eAAe,CAAC,MAA6B;IAC3D,QAAQ,MAAM,CAAC,QAAQ,EAAE,CAAC;QACxB,KAAK,YAAY;YACf,OAAO,IAAI,mBAAmB,CAAC;gBAC7B,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,WAAW,EAAE,MAAM,CAAC,IAAI;gBACxB,gBAAgB,EAAE,MAAM,CAAC,SAAS;aACnC,CAAC,CAAC;QAEL,KAAK,KAAK;YACR,OAAO,IAAI,YAAY,CAAC;gBACtB,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,aAAa,EAAE,MAAM,CAAC,MAAM;gBAC5B,WAAW,EAAE,MAAM,CAAC,IAAI;gBACxB,gBAAgB,EAAE,MAAM,CAAC,SAAS;aACnC,CAAC,CAAC;QAEL,KAAK,QAAQ;YACX,OAAO,IAAI,eAAe,CAAC;gBACzB,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,cAAc,EAAE,MAAM,CAAC,OAAO;gBAC9B,WAAW,EAAE,MAAM,CAAC,IAAI;gBACxB,gBAAgB,EAAE,MAAM,CAAC,SAAS;aACnC,CAAC,CAAC;QAEL,KAAK,aAAa;YAChB,OAAO,IAAI,mBAAmB,CAAC;gBAC7B,aAAa,EAAE,MAAM,CAAC,aAAa;gBACnC,WAAW,EAAE,MAAM,CAAC,WAAW;gBAC/B,iBAAiB,EAAE,MAAM,CAAC,UAAU;aACrC,CAAC,CAAC;IACP,CAAC;AACH,CAAC"}
+{"version":3,"file":"create-retriever.js","sourceRoot":"","sources":["../../src/retrievers/create-retriever.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,mBAAmB,EAAE,MAAM,wBAAwB,CAAC;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,MAAM,oBAAoB,CAAC;AACrD,OAAO,EAAE,mBAAmB,EAAE,MAAM,yBAAyB,CAAC;AAoC9D,MAAM,UAAU,eAAe,CAAC,MAA6B;IAC3D,QAAQ,MAAM,CAAC,QAAQ,EAAE,CAAC;QACxB,KAAK,YAAY;YACf,OAAO,IAAI,mBAAmB,CAAC;gBAC7B,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,WAAW,EAAE,MAAM,CAAC,IAAI;gBACxB,gBAAgB,EAAE,MAAM,CAAC,SAAS;aACnC,CAAC,CAAC;QAEL,KAAK,KAAK;YACR,OAAO,IAAI,YAAY,CAAC;gBACtB,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,gBAAgB,EAAE,MAAM,CAAC,gBAAgB;gBACzC,aAAa,EAAE,MAAM,CAAC,MAAM;gBAC5B,WAAW,EAAE,MAAM,CAAC,IAAI;gBACxB,gBAAgB,EAAE,MAAM,CAAC,SAAS;aACnC,CAAC,CAAC;QAEL,KAAK,QAAQ;YACX,OAAO,IAAI,eAAe,CAAC;gBACzB,YAAY,EAAE,MAAM,CAAC,YAAY;gBACjC,cAAc,EAAE,MAAM,CAAC,OAAO;gBAC9B,WAAW,EAAE,MAAM,CAAC,IAAI;gBACxB,gBAAgB,EAAE,MAAM,CAAC,SAAS;aACnC,CAAC,CAAC;QAEL,KAAK,aAAa;YAChB,OAAO,IAAI,mBAAmB,CAAC;gBAC7B,aAAa,EAAE,MAAM,CAAC,aAAa;gBACnC,WAAW,EAAE,MAAM,CAAC,WAAW;aAChC,CAAC,CAAC;IACP,CAAC;AACH,CAAC"}

package/dist/retrievers/hybrid-retriever.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"hybrid-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/hybrid-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,SAAS,EACT,eAAe,EACf,eAAe,EACf,mBAAmB,EAEpB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEzD,MAAM,WAAW,qBAAqB;IACpC,YAAY,EAAE,YAAY,CAAC;IAC3B,cAAc,CAAC,EAAE,mBAAmB,CAAC;IACrC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAKD,qBAAa,eAAgB,YAAW,SAAS;IAC/C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAe;IAC5C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAsB;IACtD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;gBAE9B,MAAM,EAAE,qBAAqB;IAOnC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAgB7F,OAAO,CAAC,iBAAiB;CAgB1B"}
+{"version":3,"file":"hybrid-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/hybrid-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,SAAS,EACT,eAAe,EACf,eAAe,EACf,mBAAmB,EAEpB,MAAM,qBAAqB,CAAC;AAC7B,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEzD,MAAM,WAAW,qBAAqB;IACpC,YAAY,EAAE,YAAY,CAAC;IAC3B,cAAc,CAAC,EAAE,mBAAmB,CAAC;IACrC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAKD,qBAAa,eAAgB,YAAW,SAAS;IAC/C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAe;IAC5C,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAsB;IACtD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;gBAE9B,MAAM,EAAE,qBAAqB;IAOnC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAgB7F,OAAO,CAAC,iBAAiB;CAiB1B"}

package/dist/retrievers/hybrid-retriever.js

@@ -25,7 +25,8 @@ export class HybridRetriever {
         return result.data.map((entry) => this.toRetrievalResult(entry));
     }
     toRetrievalResult(entry) {
-        const documentId = entry.metadata?.documentId ?? entry.sourceId;
+        const rawDocId = entry.metadata?.documentId;
+        const documentId = typeof rawDocId === 'string' ? rawDocId : entry.sourceId;
         return {
             chunkId: entry.sourceId,
             documentId,

package/dist/retrievers/hybrid-retriever.js.map

@@ -1 +1 @@
-{"version":3,"file":"hybrid-retriever.js","sourceRoot":"","sources":["../../src/retrievers/hybrid-retriever.ts"],"names":[],"mappings":"AAgBA,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B,MAAM,OAAO,eAAe;IACT,YAAY,CAAe;IAC3B,cAAc,CAAuB;IACrC,WAAW,CAAS;IACpB,gBAAgB,CAAS;IAE1C,YAAY,MAA6B;QACvC,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC;QACxC,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,CAAC;QAC5C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,aAAa,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,iBAAiB,CAAC;IACvE,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;YAC5C,KAAK;YACL,QAAQ,EAAE,QAAQ;YAClB,OAAO,EAAE,IAAI,CAAC,cAAc;YAC5B,KAAK,EAAE,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,WAAW;YACxC,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,gBAAgB;SACvD,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;IACnE,CAAC;IAEO,iBAAiB,CAAC,KAAmB;QAC3C,MAAM,UAAU,GAAI,KAAK,CAAC,QAAQ,EAAE,UAAiC,IAAI,KAAK,CAAC,QAAQ,CAAC;QAExF,OAAO;YACL,OAAO,EAAE,KAAK,CAAC,QAAQ;YACvB,UAAU;YACV,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM,EAAE,KAAK,CAAC,UAAU;YACxB,QAAQ,EAAE;gBACR,GAAG,KAAK,CAAC,QAAQ;gBACjB,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,YAAY,EAAE,KAAK,CAAC,YAAY;aACjC;SACF,CAAC;IACJ,CAAC;CACF"}
+{"version":3,"file":"hybrid-retriever.js","sourceRoot":"","sources":["../../src/retrievers/hybrid-retriever.ts"],"names":[],"mappings":"AAgBA,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B,MAAM,OAAO,eAAe;IACT,YAAY,CAAe;IAC3B,cAAc,CAAuB;IACrC,WAAW,CAAS;IACpB,gBAAgB,CAAS;IAE1C,YAAY,MAA6B;QACvC,IAAI,CAAC,YAAY,GAAG,MAAM,CAAC,YAAY,CAAC;QACxC,IAAI,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,CAAC;QAC5C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,aAAa,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,iBAAiB,CAAC;IACvE,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;YAC5C,KAAK;YACL,QAAQ,EAAE,QAAQ;YAClB,OAAO,EAAE,IAAI,CAAC,cAAc;YAC5B,KAAK,EAAE,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,WAAW;YACxC,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,gBAAgB;SACvD,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;IACnE,CAAC;IAEO,iBAAiB,CAAC,KAAmB;QAC3C,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC;QAC5C,MAAM,UAAU,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC;QAE5E,OAAO;YACL,OAAO,EAAE,KAAK,CAAC,QAAQ;YACvB,UAAU;YACV,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM,EAAE,KAAK,CAAC,UAAU;YACxB,QAAQ,EAAE;gBACR,GAAG,KAAK,CAAC,QAAQ;gBACjB,WAAW,EAAE,KAAK,CAAC,WAAW;gBAC9B,YAAY,EAAE,KAAK,CAAC,YAAY;aACjC;SACF,CAAC;IACJ,CAAC;CACF"}

package/dist/retrievers/index.d.ts

@@ -1,11 +1,11 @@
-export { SimilarityRetriever } from './similarity-retriever';
-export type { SimilarityRetrieverConfig } from './similarity-retriever';
-export { MMRRetriever } from './mmr-retriever';
-export type { MMRRetrieverConfig } from './mmr-retriever';
-export { HybridRetriever } from './hybrid-retriever';
-export type { HybridRetrieverConfig } from './hybrid-retriever';
-export { MultiQueryRetriever } from './multi-query-retriever';
-export type { MultiQueryRetrieverConfig } from './multi-query-retriever';
-export { createRetriever } from './create-retriever';
-export type { CreateRetrieverConfig } from './create-retriever';
+export { SimilarityRetriever } from './similarity-retriever.js';
+export type { SimilarityRetrieverConfig } from './similarity-retriever.js';
+export { MMRRetriever } from './mmr-retriever.js';
+export type { MMRRetrieverConfig } from './mmr-retriever.js';
+export { HybridRetriever } from './hybrid-retriever.js';
+export type { HybridRetrieverConfig } from './hybrid-retriever.js';
+export { MultiQueryRetriever } from './multi-query-retriever.js';
+export type { MultiQueryRetrieverConfig } from './multi-query-retriever.js';
+export { createRetriever } from './create-retriever.js';
+export type { CreateRetrieverConfig } from './create-retriever.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/retrievers/index.js

@@ -1,6 +1,6 @@
-export { SimilarityRetriever } from './similarity-retriever';
-export { MMRRetriever } from './mmr-retriever';
-export { HybridRetriever } from './hybrid-retriever';
-export { MultiQueryRetriever } from './multi-query-retriever';
-export { createRetriever } from './create-retriever';
+export { SimilarityRetriever } from './similarity-retriever.js';
+export { MMRRetriever } from './mmr-retriever.js';
+export { HybridRetriever } from './hybrid-retriever.js';
+export { MultiQueryRetriever } from './multi-query-retriever.js';
+export { createRetriever } from './create-retriever.js';
 //# sourceMappingURL=index.js.map

package/dist/retrievers/mmr-retriever.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"mmr-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/mmr-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,gBAAgB,EAEhB,SAAS,EACT,eAAe,EACf,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAE7B,MAAM,WAAW,kBAAkB;IACjC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAsBD,qBAAa,YAAa,YAAW,SAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmB;IAC3C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;gBAE9B,MAAM,EAAE,kBAAkB;IAQhC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAuB7F,OAAO,CAAC,SAAS;IAsCjB,OAAO,CAAC,iBAAiB;CAY1B"}
+{"version":3,"file":"mmr-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/mmr-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,gBAAgB,EAEhB,SAAS,EACT,eAAe,EACf,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAE7B,MAAM,WAAW,kBAAkB;IACjC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAuBD,qBAAa,YAAa,YAAW,SAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmB;IAC3C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;gBAE9B,MAAM,EAAE,kBAAkB;IAQhC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAuB7F,OAAO,CAAC,SAAS;IAsCjB,OAAO,CAAC,iBAAiB;CAa1B"}

package/dist/retrievers/mmr-retriever.js

@@ -3,10 +3,11 @@ const DEFAULT_THRESHOLD = 0.0;
 const DEFAULT_LAMBDA = 0.7;
 const CANDIDATE_MULTIPLIER = 3;
 function cosineSimilarity(a, b) {
+    const len = Math.min(a.length, b.length);
     let dot = 0;
     let normA = 0;
     let normB = 0;
-    for (let i = 0; i < a.length; i++) {
+    for (let i = 0; i < len; i++) {
         dot += a[i] * b[i];
         normA += a[i] * a[i];
         normB += b[i] * b[i];
@@ -75,7 +76,8 @@ export class MMRRetriever {
         return selected;
     }
     toRetrievalResult(entry) {
-        const documentId = entry.metadata?.documentId ?? entry.sourceId;
+        const rawDocId = entry.metadata?.documentId;
+        const documentId = typeof rawDocId === 'string' ? rawDocId : entry.sourceId;
         return {
             chunkId: entry.sourceId,
             documentId,

package/dist/retrievers/mmr-retriever.js.map

@@ -1 +1 @@
-{"version":3,"file":"mmr-retriever.js","sourceRoot":"","sources":["../../src/retrievers/mmr-retriever.ts"],"names":[],"mappings":"AAiBA,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAC9B,MAAM,cAAc,GAAG,GAAG,CAAC;AAC3B,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAI/B,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IAChD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC;AACvC,CAAC;AAED,MAAM,OAAO,YAAY;IACN,OAAO,CAAmB;IAC1B,SAAS,CAAmB;IAC5B,aAAa,CAAS;IACtB,WAAW,CAAS;IACpB,gBAAgB,CAAS;IAE1C,YAAY,MAA0B;QACpC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACvC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACzC,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,cAAc,CAAC;QAC5D,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,aAAa,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,iBAAiB,CAAC;IACvE,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC;QAC/C,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,gBAAgB,CAAC;QAC9D,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEtD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YACvC,MAAM,EAAE,WAAW;YACnB,KAAK;YACL,KAAK,EAAE,IAAI,GAAG,oBAAoB;YAClC,SAAS;SACV,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC;QAC/B,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEvC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QAClD,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;IAChE,CAAC;IAEO,SAAS,CACf,UAA6B,EAC7B,IAAY;QAEZ,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC;QAClC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,MAAM,QAAQ,GAA+C,EAAE,CAAC;QAEhE,OAAO,QAAQ,CAAC,MAAM,GAAG,IAAI,IAAI,SAAS,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACpD,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC;YACjB,IAAI,SAAS,GAAG,CAAC,QAAQ,CAAC;YAE1B,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC5B,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC;gBAExC,IAAI,cAAc,GAAG,CAAC,CAAC;gBACvB,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;oBAC3B,MAAM,GAAG,GAAG,gBAAgB,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;oBACjE,IAAI,GAAG,GAAG,cAAc;wBAAE,cAAc,GAAG,GAAG,CAAC;gBACjD,CAAC;gBAED,MAAM,QAAQ,GAAG,MAAM,GAAG,SAAS,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,cAAc,CAAC;gBAEpE,IAAI,QAAQ,GAAG,SAAS,EAAE,CAAC;oBACzB,SAAS,GAAG,QAAQ,CAAC;oBACrB,OAAO,GAAG,GAAG,CAAC;gBAChB,CAAC;YACH,CAAC;YAED,IAAI,OAAO,KAAK,CAAC,CAAC;gBAAE,MAAM;YAE1B,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC1B,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;QACjE,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,iBAAiB,CAAC,KAA6C;QACrE,MAAM,UAAU,GAAI,KAAK,CAAC,QAAQ,EAAE,UAAiC,IAAI,KAAK,CAAC,QAAQ,CAAC;QAExF,OAAO;YACL,OAAO,EAAE,KAAK,CAAC,QAAQ;YACvB,UAAU;YACV,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,QAAQ;YACrB,MAAM,EAAE,KAAK,CAAC,UAAU;YACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;SACzB,CAAC;IACJ,CAAC;CACF"}
+{"version":3,"file":"mmr-retriever.js","sourceRoot":"","sources":["../../src/retrievers/mmr-retriever.ts"],"names":[],"mappings":"AAiBA,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAC9B,MAAM,cAAc,GAAG,GAAG,CAAC;AAC3B,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAI/B,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IAChD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;IACzC,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC;AACvC,CAAC;AAED,MAAM,OAAO,YAAY;IACN,OAAO,CAAmB;IAC1B,SAAS,CAAmB;IAC5B,aAAa,CAAS;IACtB,WAAW,CAAS;IACpB,gBAAgB,CAAS;IAE1C,YAAY,MAA0B;QACpC,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACvC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACzC,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,IAAI,cAAc,CAAC;QAC5D,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,aAAa,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,iBAAiB,CAAC;IACvE,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC;QAC/C,MAAM,SAAS,GAAG,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,gBAAgB,CAAC;QAC9D,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEtD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YACvC,MAAM,EAAE,WAAW;YACnB,KAAK;YACL,KAAK,EAAE,IAAI,GAAG,oBAAoB;YAClC,SAAS;SACV,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC;QAC/B,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEvC,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC,CAAC;QAClD,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;IAChE,CAAC;IAEO,SAAS,CACf,UAA6B,EAC7B,IAAY;QAEZ,MAAM,MAAM,GAAG,IAAI,CAAC,aAAa,CAAC;QAClC,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,MAAM,QAAQ,GAA+C,EAAE,CAAC;QAEhE,OAAO,QAAQ,CAAC,MAAM,GAAG,IAAI,IAAI,SAAS,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACpD,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC;YACjB,IAAI,SAAS,GAAG,CAAC,QAAQ,CAAC;YAE1B,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;gBAC5B,MAAM,SAAS,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC;gBAExC,IAAI,cAAc,GAAG,CAAC,CAAC;gBACvB,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;oBAC3B,MAAM,GAAG,GAAG,gBAAgB,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC;oBACjE,IAAI,GAAG,GAAG,cAAc;wBAAE,cAAc,GAAG,GAAG,CAAC;gBACjD,CAAC;gBAED,MAAM,QAAQ,GAAG,MAAM,GAAG,SAAS,GAAG,CAAC,CAAC,GAAG,MAAM,CAAC,GAAG,cAAc,CAAC;gBAEpE,IAAI,QAAQ,GAAG,SAAS,EAAE,CAAC;oBACzB,SAAS,GAAG,QAAQ,CAAC;oBACrB,OAAO,GAAG,GAAG,CAAC;gBAChB,CAAC;YACH,CAAC;YAED,IAAI,OAAO,KAAK,CAAC,CAAC;gBAAE,MAAM;YAE1B,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC1B,QAAQ,CAAC,IAAI,CAAC,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,CAAC;QACjE,CAAC;QAED,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,iBAAiB,CAAC,KAA6C;QACrE,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC;QAC5C,MAAM,UAAU,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC;QAE5E,OAAO;YACL,OAAO,EAAE,KAAK,CAAC,QAAQ;YACvB,UAAU;YACV,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,QAAQ;YACrB,MAAM,EAAE,KAAK,CAAC,UAAU;YACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;SACzB,CAAC;IACJ,CAAC;CACF"}

package/dist/retrievers/multi-query-retriever.d.ts

@@ -2,7 +2,6 @@ import type { Retriever, RetrievalConfig, RetrievalResult } from '@cogitator-ai/
 export interface MultiQueryRetrieverConfig {
     baseRetriever: Retriever;
     expandQuery: (query: string) => Promise<string[]>;
-    defaultQueryCount?: number;
 }
 export declare class MultiQueryRetriever implements Retriever {
     private readonly baseRetriever;

package/dist/retrievers/multi-query-retriever.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"multi-query-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/multi-query-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAEvF,MAAM,WAAW,yBAAyB;IACxC,aAAa,EAAE,SAAS,CAAC;IACzB,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,iBAAiB,CAAC,EAAE,MAAM,CAAC;CAC5B;AAID,qBAAa,mBAAoB,YAAW,SAAS;IACnD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAY;IAC1C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAuC;gBAEvD,MAAM,EAAE,yBAAyB;IAKvC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAe7F,OAAO,CAAC,mBAAmB;CAY5B"}
+{"version":3,"file":"multi-query-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/multi-query-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAEvF,MAAM,WAAW,yBAAyB;IACxC,aAAa,EAAE,SAAS,CAAC;IACzB,WAAW,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;CACnD;AAID,qBAAa,mBAAoB,YAAW,SAAS;IACnD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAY;IAC1C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAuC;gBAEvD,MAAM,EAAE,yBAAyB;IAKvC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAe7F,OAAO,CAAC,mBAAmB;CAY5B"}

package/dist/retrievers/multi-query-retriever.js.map

@@ -1 +1 @@
-{"version":3,"file":"multi-query-retriever.js","sourceRoot":"","sources":["../../src/retrievers/multi-query-retriever.ts"],"names":[],"mappings":"AAQA,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB,MAAM,OAAO,mBAAmB;IACb,aAAa,CAAY;IACzB,WAAW,CAAuC;IAEnE,YAAY,MAAiC;QAC3C,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAE/C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAErC,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,GAAG,CAClC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CACzE,CAAC;QAEF,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3D,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,aAAa,CAAC;QAE5C,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAC/B,CAAC;IAEO,mBAAmB,CAAC,OAA0B;QACpD,MAAM,IAAI,GAAG,IAAI,GAAG,EAA2B,CAAC;QAEhD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC1C,IAAI,CAAC,QAAQ,IAAI,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;gBAC/C,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAC9D,CAAC;CACF"}
+{"version":3,"file":"multi-query-retriever.js","sourceRoot":"","sources":["../../src/retrievers/multi-query-retriever.ts"],"names":[],"mappings":"AAOA,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB,MAAM,OAAO,mBAAmB;IACb,aAAa,CAAY;IACzB,WAAW,CAAuC;IAEnE,YAAY,MAAiC;QAC3C,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,aAAa,CAAC;QAC1C,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QAE/C,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAErC,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,GAAG,CAClC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,IAAI,CAAC,aAAa,CAAC,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CACzE,CAAC;QAEF,MAAM,MAAM,GAAG,IAAI,CAAC,mBAAmB,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC;QAC3D,MAAM,IAAI,GAAG,OAAO,EAAE,IAAI,IAAI,aAAa,CAAC;QAE5C,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAC/B,CAAC;IAEO,mBAAmB,CAAC,OAA0B;QACpD,MAAM,IAAI,GAAG,IAAI,GAAG,EAA2B,CAAC;QAEhD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;YAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC1C,IAAI,CAAC,QAAQ,IAAI,MAAM,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,CAAC;gBAC/C,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;YACnC,CAAC;QACH,CAAC;QAED,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAC9D,CAAC;CACF"}

package/dist/retrievers/similarity-retriever.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"similarity-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/similarity-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,gBAAgB,EAEhB,SAAS,EACT,eAAe,EACf,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAE7B,MAAM,WAAW,yBAAyB;IACxC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAKD,qBAAa,mBAAoB,YAAW,SAAS;IACnD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmB;IAC3C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;gBAE9B,MAAM,EAAE,yBAAyB;IAOvC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAiB7F,OAAO,CAAC,iBAAiB;CAY1B"}
+{"version":3,"file":"similarity-retriever.d.ts","sourceRoot":"","sources":["../../src/retrievers/similarity-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,gBAAgB,EAChB,gBAAgB,EAEhB,SAAS,EACT,eAAe,EACf,eAAe,EAChB,MAAM,qBAAqB,CAAC;AAE7B,MAAM,WAAW,yBAAyB;IACxC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAKD,qBAAa,mBAAoB,YAAW,SAAS;IACnD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmB;IAC3C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAmB;IAC7C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAS;gBAE9B,MAAM,EAAE,yBAAyB;IAOvC,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAiB7F,OAAO,CAAC,iBAAiB;CAa1B"}

package/dist/retrievers/similarity-retriever.js

@@ -1,5 +1,5 @@
 const DEFAULT_TOP_K = 10;
-const DEFAULT_THRESHOLD = 0.5;
+const DEFAULT_THRESHOLD = 0.0;
 export class SimilarityRetriever {
     adapter;
     embedding;
@@ -25,7 +25,8 @@ export class SimilarityRetriever {
         return result.data.map((entry) => this.toRetrievalResult(entry));
     }
     toRetrievalResult(entry) {
-        const documentId = entry.metadata?.documentId ?? entry.sourceId;
+        const rawDocId = entry.metadata?.documentId;
+        const documentId = typeof rawDocId === 'string' ? rawDocId : entry.sourceId;
         return {
             chunkId: entry.sourceId,
             documentId,

package/dist/retrievers/similarity-retriever.js.map

@@ -1 +1 @@
-{"version":3,"file":"similarity-retriever.js","sourceRoot":"","sources":["../../src/retrievers/similarity-retriever.ts"],"names":[],"mappings":"AAgBA,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B,MAAM,OAAO,mBAAmB;IACb,OAAO,CAAmB;IAC1B,SAAS,CAAmB;IAC5B,WAAW,CAAS;IACpB,gBAAgB,CAAS;IAE1C,YAAY,MAAiC;QAC3C,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACvC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACzC,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,aAAa,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,iBAAiB,CAAC;IACvE,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEjD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YACvC,MAAM;YACN,KAAK;YACL,KAAK,EAAE,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,WAAW;YACxC,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,gBAAgB;SACvD,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;IACnE,CAAC;IAEO,iBAAiB,CAAC,KAAoC;QAC5D,MAAM,UAAU,GAAI,KAAK,CAAC,QAAQ,EAAE,UAAiC,IAAI,KAAK,CAAC,QAAQ,CAAC;QAExF,OAAO;YACL,OAAO,EAAE,KAAK,CAAC,QAAQ;YACvB,UAAU;YACV,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM,EAAE,KAAK,CAAC,UAAU;YACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;SACzB,CAAC;IACJ,CAAC;CACF"}
+{"version":3,"file":"similarity-retriever.js","sourceRoot":"","sources":["../../src/retrievers/similarity-retriever.ts"],"names":[],"mappings":"AAgBA,MAAM,aAAa,GAAG,EAAE,CAAC;AACzB,MAAM,iBAAiB,GAAG,GAAG,CAAC;AAE9B,MAAM,OAAO,mBAAmB;IACb,OAAO,CAAmB;IAC1B,SAAS,CAAmB;IAC5B,WAAW,CAAS;IACpB,gBAAgB,CAAS;IAE1C,YAAY,MAAiC;QAC3C,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACvC,IAAI,CAAC,SAAS,GAAG,MAAM,CAAC,gBAAgB,CAAC;QACzC,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC,WAAW,IAAI,aAAa,CAAC;QACvD,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,iBAAiB,CAAC;IACvE,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAkC;QAC9D,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAEjD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YACvC,MAAM;YACN,KAAK;YACL,KAAK,EAAE,OAAO,EAAE,IAAI,IAAI,IAAI,CAAC,WAAW;YACxC,SAAS,EAAE,OAAO,EAAE,SAAS,IAAI,IAAI,CAAC,gBAAgB;SACvD,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC;IACnE,CAAC;IAEO,iBAAiB,CAAC,KAAoC;QAC5D,MAAM,QAAQ,GAAG,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC;QAC5C,MAAM,UAAU,GAAG,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC;QAE5E,OAAO;YACL,OAAO,EAAE,KAAK,CAAC,QAAQ;YACvB,UAAU;YACV,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,MAAM,EAAE,KAAK,CAAC,UAAU;YACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;SACzB,CAAC;IACJ,CAAC;CACF"}

package/dist/tools.d.ts

@@ -1,5 +1,5 @@
 import { z } from 'zod';
-import type { RAGPipeline } from './rag-pipeline';
+import type { RAGPipeline } from './rag-pipeline.js';
 export interface RAGTool<TParams = unknown> {
     name: string;
     description: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cogitator-ai/rag",
-  "version": "0.1.3",
+  "version": "0.1.8",
   "description": "Retrieval Augmented Generation for Cogitator AI agents",
   "type": "module",
   "main": "./dist/index.js",
@@ -28,13 +28,9 @@
     "zod": "^4.3.6"
   },
   "peerDependencies": {
-    "@cogitator-ai/core": "workspace:*",
     "@cogitator-ai/memory": "workspace:*"
   },
   "peerDependenciesMeta": {
-    "@cogitator-ai/core": {
-      "optional": true
-    },
     "@cogitator-ai/memory": {
       "optional": true
     }
@@ -45,7 +41,6 @@
     "pdf-parse": "^1.1.1"
   },
   "devDependencies": {
-    "@cogitator-ai/core": "workspace:*",
     "@cogitator-ai/memory": "workspace:*",
     "@types/papaparse": "^5.3.15",
     "@types/pdf-parse": "^1.1.5",