@cogitator-ai/rag 0.7.9 → 0.8.1

package/README.md

@@ -48,7 +48,7 @@ const pipeline = new RAGPipelineBuilder()
   .build();
 
 // ingest documents from a file or directory
-const { documents, chunks } = await pipeline.ingest('./docs');
+const { documents, chunks, unchanged } = await pipeline.ingest('./docs');
 
 // query the knowledge base (options override the configured retrieval for this call)
 const results = await pipeline.query('How does authentication work?', { topK: 3 });
@@ -389,6 +389,8 @@ const agent = new Agent({
 
 Ingesting a source again replaces what the pipeline stored for it: once the new chunks are embedded, the old chunks of that `source` are deleted, so no duplicates or stale text are left. Document ids come from the source (plus the row, item or page within it) and chunk ids from the document id and position, so they stay the same across re-ingests. `removeSource(source)` deletes a source, using the `source` results carry (the resolved path for files).
 
+A source that did not change is skipped: each chunk keeps a hash of the document, the chunker's `fingerprint` and the embedding model, and `ingest()` embeds nothing when every hash matches (`unchanged` in its result counts those documents). The built-in chunkers have a fingerprint, a chunker of your own needs one (`readonly fingerprint = 'my-chunker:v1'`) for its sources to be skipped. A changed source is written as a new generation first and the old one deleted after, so queries never find it missing, and its unchanged documents keep their vectors.
+
 ```typescript
 await pipeline.ingest('./docs/handbook.md'); // again after an edit: replaced, not duplicated
 await pipeline.removeSource('/srv/app/docs/handbook.md');

package/dist/chunkers/fixed-chunker.d.ts

@@ -7,6 +7,7 @@ export declare class FixedSizeChunker implements Chunker {
     private readonly chunkSize;
     private readonly chunkOverlap;
     constructor(options: FixedSizeChunkerOptions);
+    get fingerprint(): string;
     chunk(text: string, documentId: string): DocumentChunk[];
 }
 //# sourceMappingURL=fixed-chunker.d.ts.map

package/dist/chunkers/fixed-chunker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fixed-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/fixed-chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAElE,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,qBAAa,gBAAiB,YAAW,OAAO;IAC9C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;gBAE1B,OAAO,EAAE,uBAAuB;IAc5C,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,aAAa,EAAE;CAuBzD"}
+{"version":3,"file":"fixed-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/fixed-chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAElE,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,qBAAa,gBAAiB,YAAW,OAAO;IAC9C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;gBAE1B,OAAO,EAAE,uBAAuB;IAc5C,IAAI,WAAW,IAAI,MAAM,CAExB;IAED,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,aAAa,EAAE;CAuBzD"}

package/dist/chunkers/fixed-chunker.js

@@ -15,6 +15,9 @@ export class FixedSizeChunker {
         this.chunkSize = options.chunkSize;
         this.chunkOverlap = options.chunkOverlap;
     }
+    get fingerprint() {
+        return JSON.stringify(['fixed', 1, this.chunkSize, this.chunkOverlap]);
+    }
     chunk(text, documentId) {
         if (text.length === 0)
             return [];

package/dist/chunkers/fixed-chunker.js.map

@@ -1 +1 @@
-{"version":3,"file":"fixed-chunker.js","sourceRoot":"","sources":["../../src/chunkers/fixed-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAQxC,MAAM,OAAO,gBAAgB;IACV,SAAS,CAAS;IAClB,YAAY,CAAS;IAEtC,YAAY,OAAgC;QAC1C,IAAI,OAAO,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,IAAY,EAAE,UAAkB;QACpC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC;QAChD,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,IAAI,EAAE,CAAC;YACvD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAC1D,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC;gBAC9B,UAAU;gBACV,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC;gBAC/B,WAAW,EAAE,KAAK;gBAClB,SAAS,EAAE,GAAG;gBACd,KAAK,EAAE,KAAK,EAAE;aACf,CAAC,CAAC;YAEH,IAAI,GAAG,KAAK,IAAI,CAAC,MAAM;gBAAE,MAAM;QACjC,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}
+{"version":3,"file":"fixed-chunker.js","sourceRoot":"","sources":["../../src/chunkers/fixed-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAQxC,MAAM,OAAO,gBAAgB;IACV,SAAS,CAAS;IAClB,YAAY,CAAS;IAEtC,YAAY,OAAgC;QAC1C,IAAI,OAAO,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;IAC3C,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC,EAAE,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,IAAY,EAAE,UAAkB;QACpC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjC,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,YAAY,CAAC;QAChD,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,IAAI,EAAE,CAAC;YACvD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAC1D,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC;gBAC9B,UAAU;gBACV,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC;gBAC/B,WAAW,EAAE,KAAK;gBAClB,SAAS,EAAE,GAAG;gBACd,KAAK,EAAE,KAAK,EAAE;aACf,CAAC,CAAC;YAEH,IAAI,GAAG,KAAK,IAAI,CAAC,MAAM;gBAAE,MAAM;QACjC,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}

package/dist/chunkers/recursive-chunker.d.ts

@@ -9,6 +9,7 @@ export declare class RecursiveChunker implements Chunker {
     private readonly chunkOverlap;
     private readonly separators;
     constructor(options: RecursiveChunkerOptions);
+    get fingerprint(): string;
     chunk(text: string, documentId: string): DocumentChunk[];
     private mergePieces;
     private splitSpan;

package/dist/chunkers/recursive-chunker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"recursive-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/recursive-chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAIlE,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,qBAAa,gBAAiB,YAAW,OAAO;IAC9C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAW;gBAE1B,OAAO,EAAE,uBAAuB;IAe5C,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,aAAa,EAAE;IAOxD,OAAO,CAAC,WAAW;IAwCnB,OAAO,CAAC,SAAS;IAyDjB,OAAO,CAAC,SAAS;CAOlB"}
+{"version":3,"file":"recursive-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/recursive-chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AAIlE,MAAM,WAAW,uBAAuB;IACtC,SAAS,EAAE,MAAM,CAAC;IAClB,YAAY,EAAE,MAAM,CAAC;IACrB,UAAU,CAAC,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,qBAAa,gBAAiB,YAAW,OAAO;IAC9C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAW;gBAE1B,OAAO,EAAE,uBAAuB;IAe5C,IAAI,WAAW,IAAI,MAAM,CAExB;IAED,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,aAAa,EAAE;IAOxD,OAAO,CAAC,WAAW;IAwCnB,OAAO,CAAC,SAAS;IAyDjB,OAAO,CAAC,SAAS;CAOlB"}

package/dist/chunkers/recursive-chunker.js

@@ -18,6 +18,9 @@ export class RecursiveChunker {
         this.chunkOverlap = options.chunkOverlap;
         this.separators = options.separators ?? DEFAULT_SEPARATORS;
     }
+    get fingerprint() {
+        return JSON.stringify(['recursive', 1, this.chunkSize, this.chunkOverlap, this.separators]);
+    }
     chunk(text, documentId) {
         if (text.length === 0)
             return [];

package/dist/chunkers/recursive-chunker.js.map

@@ -1 +1 @@
-{"version":3,"file":"recursive-chunker.js","sourceRoot":"","sources":["../../src/chunkers/recursive-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,MAAM,kBAAkB,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;AAQzD,MAAM,OAAO,gBAAgB;IACV,SAAS,CAAS;IAClB,YAAY,CAAS;IACrB,UAAU,CAAW;IAEtC,YAAY,OAAgC;QAC1C,IAAI,OAAO,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,kBAAkB,CAAC;IAC7D,CAAC;IAED,KAAK,CAAC,IAAY,EAAE,UAAkB;QACpC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC;QACvE,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IACpD,CAAC;IAEO,WAAW,CAAC,MAAc,EAAE,IAAY,EAAE,UAAkB;QAClE,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEV,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;YACzB,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC;YAEpC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC,CAAE,CAAC,GAAG,GAAG,UAAU,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC1E,CAAC,EAAE,CAAC;YACN,CAAC;YAED,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,GAAG,CAAC;YACpC,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC;gBACtC,UAAU;gBACV,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC;gBACzC,WAAW,EAAE,UAAU;gBACvB,SAAS,EAAE,QAAQ;gBACnB,KAAK,EAAE,MAAM,CAAC,MAAM;aACrB,CAAC,CAAC;YAEH,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM;gBAAE,MAAM;YAE9B,IAAI,IAAI,GAAG,CAAC,CAAC;YACb,IAAI,IAAI,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,aAAa,GAAG,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC;gBACnD,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC,CAAE,CAAC,KAAK,IAAI,aAAa,EAAE,CAAC;oBAChE,IAAI,EAAE,CAAC;gBACT,CAAC;gBACD,OAAO,IAAI,GAAG,CAAC,IAAI,MAAM,CAAC,CAAC,CAAE,CAAC,GAAG,GAAG,MAAM,CAAC,IAAI,CAAE,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;oBACzE,IAAI,EAAE,CAAC;gBACT,CAAC;YACH,CAAC;YACD,CAAC,GAAG,IAAI,CAAC;QACX,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,SAAS,CAAC,IAAY,EAAE,IAAU,EAAE,cAAsB;QAChE,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACrC,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,CAAC;QACxB,IAAI,OAAO,CAAC,GAAG,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,CAAC,OAAO,CAAC,CAAC;QAEpE,IAAI,KAAK,GAAG,cAAc,CAAC;QAC3B,OAAO,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;YACtC,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAE,CAAC;YAC1C,IAAI,SAAS,KAAK,EAAE;gBAAE,MAAM;YAC5B,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAAE,MAAM;YACtE,KAAK,EAAE,CAAC;QACV,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,KAAK,EAAE,EAAE,CAAC;YAChD,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACjC,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC/C,MAAM,QAAQ,GAAW,EAAE,CAAC;QAC5B,IAAI,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC;QAC3B,SAAS,CAAC;YACR,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YAC3C,IAAI,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;gBACnC,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;gBACnD,MAAM;YACR,CAAC;YACD,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACxE,MAAM,GAAG,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC;QACjC,CAAC;QAED,MAAM,MAAM,GAAW,EAAE,CAAC;QAC1B,IAAI,OAAO,GAAgB,IAAI,CAAC;QAEhC,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YACpC,IAAI,CAAC,OAAO;gBAAE,SAAS;YAEvB,IAAI,OAAO,IAAI,OAAO,CAAC,GAAG,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC7D,OAAO,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;gBACrD,SAAS;YACX,CAAC;YAED,IAAI,OAAO;gBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAElC,IAAI,OAAO,CAAC,GAAG,GAAG,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;gBACjD,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;gBACzD,OAAO,GAAG,IAAI,CAAC;YACjB,CAAC;iBAAM,CAAC;gBACN,OAAO,GAAG,OAAO,CAAC;YACpB,CAAC;QACH,CAAC;QAED,IAAI,OAAO;YAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAClC,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,SAAS,CAAC,IAAU;QAC1B,MAAM,MAAM,GAAW,EAAE,CAAC;QAC1B,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACvE,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAOD,SAAS,QAAQ,CAAC,IAAY,EAAE,IAAU;IACxC,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IAC1B,OAAO,KAAK,GAAG,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAE,CAAC;QAAE,KAAK,EAAE,CAAC;IACvD,OAAO,GAAG,GAAG,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAE,CAAC;QAAE,GAAG,EAAE,CAAC;IACvD,OAAO,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7C,CAAC"}
+{"version":3,"file":"recursive-chunker.js","sourceRoot":"","sources":["../../src/chunkers/recursive-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAGxC,MAAM,kBAAkB,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;AAQzD,MAAM,OAAO,gBAAgB;IACV,SAAS,CAAS;IAClB,YAAY,CAAS;IACrB,UAAU,CAAW;IAEtC,YAAY,OAAgC;QAC1C,IAAI,OAAO,CAAC,SAAS,IAAI,CAAC,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;QACD,IAAI,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAC;QAC9D,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,kBAAkB,CAAC;IAC7D,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE,CAAC,EAAE,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,YAAY,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;IAC9F,CAAC;IAED,KAAK,CAAC,IAAY,EAAE,UAAkB;QACpC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjC,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,CAAC;QACvE,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IACpD,CAAC;IAEO,WAAW,CAAC,MAAc,EAAE,IAAY,EAAE,UAAkB;QAClE,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEV,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;YACzB,MAAM,UAAU,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC,KAAK,CAAC;YAEpC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,IAAI,MAAM,CAAC,CAAC,CAAE,CAAC,GAAG,GAAG,UAAU,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC1E,CAAC,EAAE,CAAC;YACN,CAAC;YAED,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAE,CAAC,GAAG,CAAC;YACpC,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC;gBACtC,UAAU;gBACV,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC;gBACzC,WAAW,EAAE,UAAU;gBACvB,SAAS,EAAE,QAAQ;gBACnB,KAAK,EAAE,MAAM,CAAC,MAAM;aACrB,CAAC,CAAC;YAEH,IAAI,CAAC,IAAI,MAAM,CAAC,MAAM;gBAAE,MAAM;YAE9B,IAAI,IAAI,GAAG,CAAC,CAAC;YACb,IAAI,IAAI,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,aAAa,GAAG,QAAQ,GAAG,IAAI,CAAC,YAAY,CAAC;gBACnD,OAAO,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,IAAI,GAAG,CAAC,CAAE,CAAC,KAAK,IAAI,aAAa,EAAE,CAAC;oBAChE,IAAI,EAAE,CAAC;gBACT,CAAC;gBACD,OAAO,IAAI,GAAG,CAAC,IAAI,MAAM,CAAC,CAAC,CAAE,CAAC,GAAG,GAAG,MAAM,CAAC,IAAI,CAAE,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;oBACzE,IAAI,EAAE,CAAC;gBACT,CAAC;YACH,CAAC;YACD,CAAC,GAAG,IAAI,CAAC;QACX,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,SAAS,CAAC,IAAY,EAAE,IAAU,EAAE,cAAsB;QAChE,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QACrC,IAAI,CAAC,OAAO;YAAE,OAAO,EAAE,CAAC;QACxB,IAAI,OAAO,CAAC,GAAG,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,CAAC,OAAO,CAAC,CAAC;QAEpE,IAAI,KAAK,GAAG,cAAc,CAAC;QAC3B,OAAO,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,CAAC;YACtC,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAE,CAAC;YAC1C,IAAI,SAAS,KAAK,EAAE;gBAAE,MAAM;YAC5B,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,SAAS,CAAC;gBAAE,MAAM;YACtE,KAAK,EAAE,CAAC;QACV,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACzC,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,KAAK,EAAE,EAAE,CAAC;YAChD,OAAO,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QACjC,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC/C,MAAM,QAAQ,GAAW,EAAE,CAAC;QAC5B,IAAI,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC;QAC3B,SAAS,CAAC;YACR,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;YAC3C,IAAI,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE,IAAI,OAAO,CAAC,GAAG,EAAE,CAAC;gBACnC,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;gBACnD,MAAM;YACR,CAAC;YACD,QAAQ,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,GAAG,IAAI,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YACxE,MAAM,GAAG,EAAE,GAAG,SAAS,CAAC,MAAM,CAAC;QACjC,CAAC;QAED,MAAM,MAAM,GAAW,EAAE,CAAC;QAC1B,IAAI,OAAO,GAAgB,IAAI,CAAC;QAEhC,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;YAC3B,MAAM,OAAO,GAAG,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YACpC,IAAI,CAAC,OAAO;gBAAE,SAAS;YAEvB,IAAI,OAAO,IAAI,OAAO,CAAC,GAAG,GAAG,OAAO,CAAC,KAAK,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC7D,OAAO,GAAG,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,EAAE,CAAC;gBACrD,SAAS;YACX,CAAC;YAED,IAAI,OAAO;gBAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAElC,IAAI,OAAO,CAAC,GAAG,GAAG,OAAO,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,CAAC;gBACjD,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC;gBACzD,OAAO,GAAG,IAAI,CAAC;YACjB,CAAC;iBAAM,CAAC;gBACN,OAAO,GAAG,OAAO,CAAC;YACpB,CAAC;QACH,CAAC;QAED,IAAI,OAAO;YAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAClC,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,SAAS,CAAC,IAAU;QAC1B,MAAM,MAAM,GAAW,EAAE,CAAC;QAC1B,KAAK,IAAI,KAAK,GAAG,IAAI,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,KAAK,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACvE,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF;AAOD,SAAS,QAAQ,CAAC,IAAY,EAAE,IAAU;IACxC,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC;IAC1B,OAAO,KAAK,GAAG,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAE,CAAC;QAAE,KAAK,EAAE,CAAC;IACvD,OAAO,GAAG,GAAG,KAAK,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,CAAE,CAAC;QAAE,GAAG,EAAE,CAAC;IACvD,OAAO,KAAK,GAAG,GAAG,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7C,CAAC"}

package/dist/chunkers/semantic-chunker.d.ts

@@ -11,6 +11,7 @@ export declare class SemanticChunker implements AsyncChunker {
     private readonly minChunkSize;
     private readonly maxChunkSize;
     constructor(options: SemanticChunkerOptions);
+    get fingerprint(): string;
     chunk(text: string, documentId: string): Promise<DocumentChunk[]>;
     private mergeSmallGroups;
     private findSentenceOffset;

package/dist/chunkers/semantic-chunker.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"semantic-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/semantic-chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAEzF,MAAM,WAAW,sBAAsB;IACrC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AA8BD,qBAAa,eAAgB,YAAW,YAAY;IAClD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAmB;IACpD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAC7C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;gBAE1B,OAAO,EAAE,sBAAsB;IAqBrC,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAuCvE,OAAO,CAAC,gBAAgB;IAqCxB,OAAO,CAAC,kBAAkB;IAK1B,OAAO,CAAC,WAAW;IA4CnB,OAAO,CAAC,eAAe;IA2EvB,OAAO,CAAC,iBAAiB;CAqB1B"}
+{"version":3,"file":"semantic-chunker.d.ts","sourceRoot":"","sources":["../../src/chunkers/semantic-chunker.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAEzF,MAAM,WAAW,sBAAsB;IACrC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AA8BD,qBAAa,eAAgB,YAAW,YAAY;IAClD,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAmB;IACpD,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAS;IAC7C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;gBAE1B,OAAO,EAAE,sBAAsB;IAqB3C,IAAI,WAAW,IAAI,MAAM,CAUxB;IAEK,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAuCvE,OAAO,CAAC,gBAAgB;IAqCxB,OAAO,CAAC,kBAAkB;IAK1B,OAAO,CAAC,WAAW;IA4CnB,OAAO,CAAC,eAAe;IA2EvB,OAAO,CAAC,iBAAiB;CAqB1B"}

package/dist/chunkers/semantic-chunker.js

@@ -42,6 +42,17 @@ export class SemanticChunker {
         this.minChunkSize = minChunkSize;
         this.maxChunkSize = maxChunkSize;
     }
+    get fingerprint() {
+        return JSON.stringify([
+            'semantic',
+            1,
+            this.breakpointThreshold,
+            this.minChunkSize,
+            this.maxChunkSize,
+            this.embeddingService.model,
+            this.embeddingService.dimensions,
+        ]);
+    }
     async chunk(text, documentId) {
         if (!text || text.trim().length === 0)
             return [];

package/dist/chunkers/semantic-chunker.js.map

@@ -1 +1 @@
-{"version":3,"file":"semantic-chunker.js","sourceRoot":"","sources":["../../src/chunkers/semantic-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAUxC,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IAChD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,iCAAiC,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,MAAM,IAAI;YAC1D,iDAAiD,CACpD,CAAC;IACJ,CAAC;IACD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC;AACvC,CAAC;AAED,MAAM,iBAAiB,GAAG,uBAAuB,CAAC;AAElD,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,IAAI;SACR,KAAK,CAAC,iBAAiB,CAAC;SACxB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,OAAO,eAAe;IACT,gBAAgB,CAAmB;IACnC,mBAAmB,CAAS;IAC5B,YAAY,CAAS;IACrB,YAAY,CAAS;IAEtC,YAAY,OAA+B;QACzC,IACE,OAAO,CAAC,mBAAmB,KAAK,SAAS;YACzC,CAAC,OAAO,CAAC,mBAAmB,GAAG,CAAC,IAAI,OAAO,CAAC,mBAAmB,GAAG,CAAC,CAAC,EACpE,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;QACjE,CAAC;QAED,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC;QAClD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QAEzE,IAAI,YAAY,GAAG,YAAY,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;QACjD,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,IAAI,GAAG,CAAC;QAC9D,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,UAAkB;QAC1C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjD,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEtC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QACrE,IAAI,UAAU,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,EAAE,CAAC;YAC3C,MAAM,IAAI,KAAK,CACb,iCAAiC,UAAU,CAAC,MAAM,gBAAgB,SAAS,CAAC,MAAM,YAAY,CAC/F,CAAC;QACJ,CAAC;QAED,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,YAAY,CAAC,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,MAAM,GAAe,EAAE,CAAC;QAC9B,IAAI,YAAY,GAAa,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QAE5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,IAAI,YAAY,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAC/C,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAC1B,YAAY,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAE1B,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAC7C,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IACpD,CAAC;IAEO,gBAAgB,CAAC,MAAkB;QACzC,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC;YAAE,OAAO,MAAM,CAAC;QAEtC,MAAM,MAAM,GAAe,EAAE,CAAC;QAC9B,IAAI,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrC,MAAM,cAAc,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;YAEhE,IAAI,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,cAAc,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClF,OAAO,GAAG,CAAC,GAAG,OAAO,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YACvC,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACrB,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,IAAI,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAChC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClE,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,CAAC;gBACpD,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvB,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,kBAAkB,CAAC,QAAgB,EAAE,YAAoB,EAAE,UAAkB;QACnF,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QACvD,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC;IACrC,CAAC;IAEO,WAAW,CACjB,MAAkB,EAClB,YAAoB,EACpB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC;YAChF,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;YACnF,MAAM,SAAS,GAAG,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC;YAElD,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;YAE3D,IAAI,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACxC,MAAM,CAAC,IAAI,CAAC;oBACV,EAAE,EAAE,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC;oBACtC,UAAU;oBACV,OAAO;oBACP,WAAW;oBACX,SAAS;oBACT,KAAK,EAAE,MAAM,CAAC,MAAM;iBACrB,CAAC,CAAC;gBACH,UAAU,GAAG,SAAS,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CACpC,KAAK,EACL,YAAY,EACZ,UAAU,EACV,UAAU,EACV,MAAM,CAAC,MAAM,CACd,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC1B,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzB,UAAU,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;gBACzD,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,eAAe,CACrB,SAAmB,EACnB,YAAoB,EACpB,UAAkB,EAClB,UAAkB,EAClB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,gBAAgB,GAAa,EAAE,CAAC;QACpC,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,KAAK,GAAG,UAAU,CAAC;QACvB,IAAI,MAAM,GAAG,UAAU,CAAC;QAExB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,IAAI,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;gBACxC,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAChC,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAClC,gBAAgB,EAChB,YAAY,EACZ,UAAU,EACV,MAAM,EACN,KAAK,EAAE,CACR,CAAC;oBACF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACnB,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC;oBACzB,gBAAgB,GAAG,EAAE,CAAC;oBACtB,aAAa,GAAG,CAAC,CAAC;gBACpB,CAAC;gBAED,MAAM,aAAa,GAAG,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;gBAC9E,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,QAAQ,CAAC,MAAM,EAAE,GAAG,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;oBACpE,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;oBAC5C,MAAM,CAAC,IAAI,CAAC;wBACV,EAAE,EAAE,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC;wBAC9B,UAAU;wBACV,OAAO,EAAE,KAAK;wBACd,WAAW,EAAE,aAAa,GAAG,GAAG;wBAChC,SAAS,EAAE,aAAa,GAAG,QAAQ;wBACnC,KAAK,EAAE,KAAK,EAAE;qBACf,CAAC,CAAC;gBACL,CAAC;gBACD,MAAM,GAAG,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC;gBACzC,SAAS;YACX,CAAC;YAED,MAAM,SAAS,GAAG,aAAa,GAAG,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;YAE1F,IAAI,SAAS,GAAG,IAAI,CAAC,YAAY,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACjE,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAClC,gBAAgB,EAChB,YAAY,EACZ,UAAU,EACV,MAAM,EACN,KAAK,EAAE,CACR,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACnB,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC;gBACzB,gBAAgB,GAAG,CAAC,QAAQ,CAAC,CAAC;gBAC9B,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACN,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAChC,aAAa,GAAG,SAAS,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,CAAC,IAAI,CACT,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,CAClF,CAAC;QACJ,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,iBAAiB,CACvB,SAAmB,EACnB,YAAoB,EACpB,UAAkB,EAClB,UAAkB,EAClB,KAAa;QAEb,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC;QACpF,MAAM,YAAY,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACrD,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;QACnF,MAAM,SAAS,GAAG,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC;QAElD,OAAO;YACL,EAAE,EAAE,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC;YAC9B,UAAU;YACV,OAAO,EAAE,YAAY,CAAC,KAAK,CAAC,WAAW,EAAE,SAAS,CAAC;YACnD,WAAW;YACX,SAAS;YACT,KAAK;SACN,CAAC;IACJ,CAAC;CACF"}
+{"version":3,"file":"semantic-chunker.js","sourceRoot":"","sources":["../../src/chunkers/semantic-chunker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAC;AAUxC,SAAS,gBAAgB,CAAC,CAAW,EAAE,CAAW;IAChD,IAAI,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CACb,iCAAiC,CAAC,CAAC,MAAM,OAAO,CAAC,CAAC,MAAM,IAAI;YAC1D,iDAAiD,CACpD,CAAC;IACJ,CAAC;IACD,IAAI,GAAG,GAAG,CAAC,CAAC;IACZ,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClD,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC;AACvC,CAAC;AAED,MAAM,iBAAiB,GAAG,uBAAuB,CAAC;AAElD,SAAS,cAAc,CAAC,IAAY;IAClC,OAAO,IAAI;SACR,KAAK,CAAC,iBAAiB,CAAC;SACxB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,OAAO,eAAe;IACT,gBAAgB,CAAmB;IACnC,mBAAmB,CAAS;IAC5B,YAAY,CAAS;IACrB,YAAY,CAAS;IAEtC,YAAY,OAA+B;QACzC,IACE,OAAO,CAAC,mBAAmB,KAAK,SAAS;YACzC,CAAC,OAAO,CAAC,mBAAmB,GAAG,CAAC,IAAI,OAAO,CAAC,mBAAmB,GAAG,CAAC,CAAC,EACpE,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;QACjE,CAAC;QAED,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC;QAClD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QAEzE,IAAI,YAAY,GAAG,YAAY,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;QAC/D,CAAC;QAED,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,CAAC;QACjD,IAAI,CAAC,mBAAmB,GAAG,OAAO,CAAC,mBAAmB,IAAI,GAAG,CAAC;QAC9D,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;IACnC,CAAC;IAED,IAAI,WAAW;QACb,OAAO,IAAI,CAAC,SAAS,CAAC;YACpB,UAAU;YACV,CAAC;YACD,IAAI,CAAC,mBAAmB;YACxB,IAAI,CAAC,YAAY;YACjB,IAAI,CAAC,YAAY;YACjB,IAAI,CAAC,gBAAgB,CAAC,KAAK;YAC3B,IAAI,CAAC,gBAAgB,CAAC,UAAU;SACjC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,UAAkB;QAC1C,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEjD,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAEtC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3B,OAAO,IAAI,CAAC,WAAW,CAAC,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QACrE,IAAI,UAAU,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,EAAE,CAAC;YAC3C,MAAM,IAAI,KAAK,CACb,iCAAiC,UAAU,CAAC,MAAM,gBAAgB,SAAS,CAAC,MAAM,YAAY,CAC/F,CAAC;QACJ,CAAC;QAED,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC/C,YAAY,CAAC,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACxE,CAAC;QAED,MAAM,MAAM,GAAe,EAAE,CAAC;QAC9B,IAAI,YAAY,GAAa,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QAE5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7C,IAAI,YAAY,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,mBAAmB,EAAE,CAAC;gBAC/C,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;gBAC1B,YAAY,GAAG,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACpC,CAAC;iBAAM,CAAC;gBACN,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACtC,CAAC;QACH,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAE1B,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC;QAC7C,OAAO,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC,CAAC;IACpD,CAAC;IAEO,gBAAgB,CAAC,MAAkB;QACzC,IAAI,MAAM,CAAC,MAAM,IAAI,CAAC;YAAE,OAAO,MAAM,CAAC;QAEtC,MAAM,MAAM,GAAe,EAAE,CAAC;QAC9B,IAAI,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;QAExB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,MAAM,QAAQ,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACrC,MAAM,cAAc,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;YAEhE,IAAI,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,cAAc,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClF,OAAO,GAAG,CAAC,GAAG,OAAO,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YACvC,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACrB,OAAO,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACtB,CAAC;QACH,CAAC;QAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACvB,MAAM,WAAW,GAAG,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACtC,IAAI,WAAW,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAChE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;gBACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;gBAChC,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClE,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,IAAI,EAAE,GAAG,OAAO,CAAC,CAAC;gBACpD,CAAC;qBAAM,CAAC;oBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;gBACvB,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,kBAAkB,CAAC,QAAgB,EAAE,YAAoB,EAAE,UAAkB;QACnF,MAAM,GAAG,GAAG,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QACvD,OAAO,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC;IACrC,CAAC;IAEO,WAAW,CACjB,MAAkB,EAClB,YAAoB,EACpB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC;YAChF,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAC7C,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;YACnF,MAAM,SAAS,GAAG,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC;YAElD,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;YAE3D,IAAI,OAAO,CAAC,MAAM,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;gBACxC,MAAM,CAAC,IAAI,CAAC;oBACV,EAAE,EAAE,OAAO,CAAC,UAAU,EAAE,MAAM,CAAC,MAAM,CAAC;oBACtC,UAAU;oBACV,OAAO;oBACP,WAAW;oBACX,SAAS;oBACT,KAAK,EAAE,MAAM,CAAC,MAAM;iBACrB,CAAC,CAAC;gBACH,UAAU,GAAG,SAAS,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACN,MAAM,SAAS,GAAG,IAAI,CAAC,eAAe,CACpC,KAAK,EACL,YAAY,EACZ,UAAU,EACV,UAAU,EACV,MAAM,CAAC,MAAM,CACd,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC1B,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzB,UAAU,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;gBACzD,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,eAAe,CACrB,SAAmB,EACnB,YAAoB,EACpB,UAAkB,EAClB,UAAkB,EAClB,UAAkB;QAElB,MAAM,MAAM,GAAoB,EAAE,CAAC;QACnC,IAAI,gBAAgB,GAAa,EAAE,CAAC;QACpC,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,KAAK,GAAG,UAAU,CAAC;QACvB,IAAI,MAAM,GAAG,UAAU,CAAC;QAExB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YACjC,IAAI,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;gBACxC,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAChC,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAClC,gBAAgB,EAChB,YAAY,EACZ,UAAU,EACV,MAAM,EACN,KAAK,EAAE,CACR,CAAC;oBACF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACnB,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC;oBACzB,gBAAgB,GAAG,EAAE,CAAC;oBACtB,aAAa,GAAG,CAAC,CAAC;gBACpB,CAAC;gBAED,MAAM,aAAa,GAAG,IAAI,CAAC,kBAAkB,CAAC,QAAQ,EAAE,YAAY,EAAE,MAAM,CAAC,CAAC;gBAC9E,KAAK,IAAI,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,QAAQ,CAAC,MAAM,EAAE,GAAG,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;oBAClE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,YAAY,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;oBACpE,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;oBAC5C,MAAM,CAAC,IAAI,CAAC;wBACV,EAAE,EAAE,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC;wBAC9B,UAAU;wBACV,OAAO,EAAE,KAAK;wBACd,WAAW,EAAE,aAAa,GAAG,GAAG;wBAChC,SAAS,EAAE,aAAa,GAAG,QAAQ;wBACnC,KAAK,EAAE,KAAK,EAAE;qBACf,CAAC,CAAC;gBACL,CAAC;gBACD,MAAM,GAAG,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC;gBACzC,SAAS;YACX,CAAC;YAED,MAAM,SAAS,GAAG,aAAa,GAAG,CAAC,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,MAAM,CAAC;YAE1F,IAAI,SAAS,GAAG,IAAI,CAAC,YAAY,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACjE,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,CAClC,gBAAgB,EAChB,YAAY,EACZ,UAAU,EACV,MAAM,EACN,KAAK,EAAE,CACR,CAAC;gBACF,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACnB,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC;gBACzB,gBAAgB,GAAG,CAAC,QAAQ,CAAC,CAAC;gBAC9B,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACN,gBAAgB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAChC,aAAa,GAAG,SAAS,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,IAAI,gBAAgB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAChC,MAAM,CAAC,IAAI,CACT,IAAI,CAAC,iBAAiB,CAAC,gBAAgB,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,CAAC,CAClF,CAAC;QACJ,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,iBAAiB,CACvB,SAAmB,EACnB,YAAoB,EACpB,UAAkB,EAClB,UAAkB,EAClB,KAAa;QAEb,MAAM,WAAW,GAAG,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,UAAU,CAAC,CAAC;QACpF,MAAM,YAAY,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACrD,MAAM,SAAS,GAAG,IAAI,CAAC,kBAAkB,CAAC,YAAY,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC;QACnF,MAAM,SAAS,GAAG,SAAS,GAAG,YAAY,CAAC,MAAM,CAAC;QAElD,OAAO;YACL,EAAE,EAAE,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC;YAC9B,UAAU;YACV,OAAO,EAAE,YAAY,CAAC,KAAK,CAAC,WAAW,EAAE,SAAS,CAAC;YACnD,WAAW;YACX,SAAS;YACT,KAAK;SACN,CAAC;IACJ,CAAC;CACF"}

package/dist/rag-pipeline.d.ts

@@ -12,16 +12,39 @@ export declare class RAGPipeline {
     private readonly deps;
     private stats;
     private warnedAppendOnly;
+    /** Sources a local keyword index of the retriever holds the chunks of in this process */
+    private readonly locallyIndexed;
+    /** Ingests under way by source, so this process replaces a source one ingest at a time */
+    private readonly ingesting;
     constructor(config: RAGPipelineConfig, deps: RAGPipelineDeps, alreadyParsed?: boolean);
     /**
      * Loads `source` and stores its chunks. Each source the loader returns documents for replaces
-     * what this pipeline stored for it before, once the new chunks are embedded, so ingesting a
-     * file again (or an edited file) leaves no duplicates or stale text behind.
+     * what this pipeline stored for it before, so ingesting a file again (or an edited file) leaves
+     * no duplicates or stale text behind.
+     *
+     * A source whose documents, chunker and embedding model did not change since the last ingest
+     * is left as stored, without embedding anything (`unchanged` counts its documents). That needs
+     * a chunker with a `fingerprint`, which the built-in ones have. A changed source is written as
+     * a new generation before the previous one is deleted, so searches never find it empty, and
+     * the chunks of its unchanged documents keep their vectors. The first chunk of each document
+     * is written last, so a source an ingest stopped writing is never taken as unchanged. That
+     * needs a store with `getBySources` and `deleteByFilter`, which every built-in store has.
+     * Chunks stored before generations, or a store without them, are replaced as before: deleted,
+     * then written again.
      */
     ingest(source: string): Promise<{
         documents: number;
         chunks: number;
+        unchanged: number;
     }>;
+    /** Runs `work` once the ingests of `source` this process started before it are done */
+    private oneAtATime;
+    /**
+     * Ingests the documents of one source, leaving it as stored when unchanged. Generations older
+     * than the one written are deleted after it, newer ones belong to an ingest that started later
+     * and are left to it.
+     */
+    private ingestGroup;
     /**
      * Deletes every chunk this pipeline stored for `source`: the `source` its results carry (for
      * files, the resolved path the loader read). Needs an embedding store with `deleteByFilter`.
@@ -39,7 +62,56 @@ export declare class RAGPipeline {
      */
     private scope;
     private namespaceMetadata;
+    /**
+     * Hash of what a document's chunks are made from: its text, source and metadata, the chunker's
+     * fingerprint and the embedding model. None when the chunker has no fingerprint, as the same
+     * text may then give other chunks.
+     */
+    private contentHash;
+    /**
+     * Gives a local keyword index of the retriever the stored chunks of an unchanged source, once
+     * per process: a new pipeline has an empty one, and skipping the source would leave it so.
+     */
+    private indexLocally;
+    /**
+     * The stored chunks of this pipeline's namespace with these ids, by id: more than one for an id
+     * when an ingest stopped before deleting the generation it replaced. Empty when the store cannot
+     * read chunks by id.
+     */
+    private storedChunks;
+    /** The stored embeddings of this pipeline's documents with these chunk ids */
+    private storedEmbeddings;
+    private ownChunk;
+    /**
+     * What the last ingest left of a source: whether it is unchanged, and the generations to delete
+     * once a new one is written. No `generations` when the source has to be cleared first: nothing
+     * of it was found by id, or it was stored before generations.
+     */
+    private previousGeneration;
     private prepareDocument;
+    /**
+     * The stored vectors of an unchanged document's chunks, in their order, when the store keeps
+     * all of them from the same ingest
+     */
+    private keptVectors;
+    /**
+     * Generations other than `kept` found on the second chunks of the documents, which an ingest
+     * that stopped before the heads wrote first: older ones, and newer ones old enough that no
+     * ingest is still writing them
+     */
+    private unfinishedGenerations;
+    /**
+     * Generations other than `current` the store holds chunks of for these documents: ones an
+     * ingest stopped writing before their heads, which no head names
+     */
+    private leftGenerations;
+    /**
+     * Whether an ingest that started after the one that wrote `generation` already wrote heads of
+     * these documents, so the later one wins and `generation` goes
+     */
+    private supersededBy;
+    /** Deletes the chunks of `source` an earlier ingest wrote as `generation` */
+    private forgetGeneration;
     /** Deletes the stored chunks of `source` in this pipeline's namespace, and drops them from a local index. */
     private forgetSource;
     private chunkDocument;

package/dist/rag-pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"rag-pipeline.d.ts","sourceRoot":"","sources":["../src/rag-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,cAAc,EACd,OAAO,EACP,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,EAChB,SAAS,EACT,QAAQ,EACR,iBAAiB,EACjB,eAAe,EACf,eAAe,EAIhB,MAAM,qBAAqB,CAAC;AAI7B,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,cAAc,CAAC;IACvB,OAAO,EAAE,OAAO,GAAG,YAAY,CAAC;IAChC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,SAAS,EAAE,SAAS,CAAC;IACrB,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACrB;AA0BD,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoB;IAC3C,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAkB;IACvC,OAAO,CAAC,KAAK,CAAkE;IAC/E,OAAO,CAAC,gBAAgB,CAAS;gBAErB,MAAM,EAAE,iBAAiB,EAAE,IAAI,EAAE,eAAe,EAAE,aAAa,UAAQ;IAKnF;;;;OAIG;IACG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAsB5E;;;OAGG;IACG,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAS3C,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAkBzF,QAAQ;;;;;IAIR;;;OAGG;IACH,OAAO,CAAC,KAAK;IASb,OAAO,CAAC,iBAAiB;YAIX,eAAe;IAc7B,6GAA6G;YAC/F,YAAY;YA8BZ,aAAa;YAKb,WAAW;YAiBX,UAAU;CAmCzB"}
+{"version":3,"file":"rag-pipeline.d.ts","sourceRoot":"","sources":["../src/rag-pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACV,cAAc,EACd,OAAO,EACP,YAAY,EACZ,gBAAgB,EAChB,gBAAgB,EAChB,SAAS,EACT,QAAQ,EACR,iBAAiB,EACjB,eAAe,EACf,eAAe,EAKhB,MAAM,qBAAqB,CAAC;AAM7B,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,cAAc,CAAC;IACvB,OAAO,EAAE,OAAO,GAAG,YAAY,CAAC;IAChC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,SAAS,EAAE,SAAS,CAAC;IACrB,QAAQ,CAAC,EAAE,QAAQ,CAAC;CACrB;AA2FD,qBAAa,WAAW;IACtB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoB;IAC3C,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAkB;IACvC,OAAO,CAAC,KAAK,CAAkE;IAC/E,OAAO,CAAC,gBAAgB,CAAS;IACjC,yFAAyF;IACzF,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAqB;IACpD,0FAA0F;IAC1F,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAuC;gBAErD,MAAM,EAAE,iBAAiB,EAAE,IAAI,EAAE,eAAe,EAAE,aAAa,UAAQ;IAKnF;;;;;;;;;;;;;;OAcG;IACG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IAc/F,uFAAuF;YACzE,UAAU;IAWxB;;;;OAIG;YACW,WAAW;IAsEzB;;;OAGG;IACG,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAS3C,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,OAAO,CAAC,eAAe,CAAC,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAkBzF,QAAQ;;;;;IAIR;;;OAGG;IACH,OAAO,CAAC,KAAK;IASb,OAAO,CAAC,iBAAiB;IAIzB;;;;OAIG;IACH,OAAO,CAAC,WAAW;IAmBnB;;;OAGG;YACW,YAAY;IAqB1B;;;;OAIG;YACW,YAAY;IAS1B,8EAA8E;YAChE,gBAAgB;IAmB9B,OAAO,CAAC,QAAQ;IAgBhB;;;;OAIG;IACH,OAAO,CAAC,kBAAkB;YA4BZ,eAAe;IAmB7B;;;OAGG;YACW,WAAW;IAmBzB;;;;OAIG;IACH,OAAO,CAAC,qBAAqB;IAiB7B;;;OAGG;YACW,eAAe;IAsB7B;;;OAGG;YACW,YAAY;IAU1B,6EAA6E;YAC/D,gBAAgB;IAqB9B,6GAA6G;YAC/F,YAAY;YA8BZ,aAAa;YAKb,WAAW;YAwBX,UAAU;CAyCzB"}

package/dist/rag-pipeline.js

@@ -1,6 +1,36 @@
+import { createHash, randomUUID } from 'node:crypto';
 import { RAGPipelineConfigSchema } from './schema.js';
 import { isChunkIndexer } from './retrievers/chunk-indexer.js';
+import { chunkId } from './chunkers/chunk-id.js';
 const STORE_CONCURRENCY = 16;
+/** Raised when what the content hash covers changes, so every document is embedded again */
+const HASH_VERSION = 1;
+/** Source ids one `getBySources` call reads */
+const LOOKUP_BATCH = 500;
+/**
+ * Chunk positions of each document looked at for generations an earlier ingest left unfinished,
+ * at least: an ingest that stopped wrote some of the first chunks after the head
+ */
+const PROBED_CHUNKS = 16;
+/**
+ * How old a generation newer than the stored one must be to count as left by an ingest that
+ * stopped, rather than one another process is still writing
+ */
+const ABANDONED_AFTER_MS = 10 * 60_000;
+let lastGenerationTime = 0;
+/**
+ * A new generation id: its time first, so ids sort in the order ingests started, the time kept
+ * strictly growing in this process so two ingests started in one millisecond still sort
+ */
+function newGeneration() {
+    lastGenerationTime = Math.max(Date.now(), lastGenerationTime + 1);
+    return `${lastGenerationTime.toString(36).padStart(9, '0')}-${randomUUID()}`;
+}
+/** When a generation was started, 0 for ids without a time */
+function generationTime(generation) {
+    const match = /^([0-9a-z]{9})-/.exec(generation);
+    return match ? Number.parseInt(match[1], 36) : 0;
+}
 function definedEntries(value) {
     if (!value)
         return {};
@@ -23,32 +53,117 @@ export class RAGPipeline {
     deps;
     stats = { documentsIngested: 0, chunksStored: 0, queriesProcessed: 0 };
     warnedAppendOnly = false;
+    /** Sources a local keyword index of the retriever holds the chunks of in this process */
+    locallyIndexed = new Set();
+    /** Ingests under way by source, so this process replaces a source one ingest at a time */
+    ingesting = new Map();
     constructor(config, deps, alreadyParsed = false) {
         this.config = alreadyParsed ? config : RAGPipelineConfigSchema.parse(config);
         this.deps = deps;
     }
     /**
      * Loads `source` and stores its chunks. Each source the loader returns documents for replaces
-     * what this pipeline stored for it before, once the new chunks are embedded, so ingesting a
-     * file again (or an edited file) leaves no duplicates or stale text behind.
+     * what this pipeline stored for it before, so ingesting a file again (or an edited file) leaves
+     * no duplicates or stale text behind.
+     *
+     * A source whose documents, chunker and embedding model did not change since the last ingest
+     * is left as stored, without embedding anything (`unchanged` counts its documents). That needs
+     * a chunker with a `fingerprint`, which the built-in ones have. A changed source is written as
+     * a new generation before the previous one is deleted, so searches never find it empty, and
+     * the chunks of its unchanged documents keep their vectors. The first chunk of each document
+     * is written last, so a source an ingest stopped writing is never taken as unchanged. That
+     * needs a store with `getBySources` and `deleteByFilter`, which every built-in store has.
+     * Chunks stored before generations, or a store without them, are replaced as before: deleted,
+     * then written again.
      */
     async ingest(source) {
         const documents = await this.deps.loader.load(source);
         let totalChunks = 0;
+        let unchanged = 0;
         for (const group of groupBySource(documents)) {
-            const prepared = [];
-            for (const doc of group) {
-                prepared.push(await this.prepareDocument(doc));
+            const done = await this.oneAtATime(group[0].source, () => this.ingestGroup(group));
+            totalChunks += done.chunks;
+            unchanged += done.unchanged;
+        }
+        return { documents: documents.length, chunks: totalChunks, unchanged };
+    }
+    /** Runs `work` once the ingests of `source` this process started before it are done */
+    async oneAtATime(source, work) {
+        const before = this.ingesting.get(source) ?? Promise.resolve();
+        const run = before.catch(() => undefined).then(work);
+        this.ingesting.set(source, run);
+        try {
+            return await run;
+        }
+        finally {
+            if (this.ingesting.get(source) === run)
+                this.ingesting.delete(source);
+        }
+    }
+    /**
+     * Ingests the documents of one source, leaving it as stored when unchanged. Generations older
+     * than the one written are deleted after it, newer ones belong to an ingest that started later
+     * and are left to it.
+     */
+    async ingestGroup(group) {
+        const sourceName = group[0].source;
+        const hashes = new Map(group.map((doc) => [doc.id, this.contentHash(doc)]));
+        const stored = await this.storedChunks(group.flatMap((doc) => [chunkId(doc.id, 0), chunkId(doc.id, 1)]));
+        const heads = new Map(group.map((doc) => [chunkId(doc.id, 0), stored.get(chunkId(doc.id, 0)) ?? []]));
+        const previous = this.previousGeneration(group, hashes, heads);
+        if (previous.unchanged) {
+            const kept = previous.generations[0];
+            for (const old of this.unfinishedGenerations(group, stored, kept)) {
+                await this.forgetGeneration(sourceName, old);
             }
-            await this.forgetSource(group[0].source);
-            for (const { doc, chunks, vectors } of prepared) {
-                await this.storeChunks(chunks, vectors, doc);
-                totalChunks += chunks.length;
-                this.stats.chunksStored += chunks.length;
-                this.stats.documentsIngested++;
+            await this.indexLocally(sourceName, group);
+            return { chunks: 0, unchanged: group.length };
+        }
+        const prepared = [];
+        for (const doc of group) {
+            const hash = hashes.get(doc.id);
+            const reusable = hash === undefined
+                ? undefined
+                : heads
+                    .get(chunkId(doc.id, 0))
+                    ?.find((head) => head.generation !== undefined && head.hash === hash);
+            prepared.push(await this.prepareDocument(doc, reusable));
+        }
+        if (!previous.generations)
+            await this.forgetSource(sourceName);
+        const generation = newGeneration();
+        const writes = prepared.flatMap(({ doc, chunks, vectors }) => chunks.map((chunk, i) => ({ chunk, vector: vectors[i], doc })));
+        const isHead = ({ chunk, doc }) => chunk.id === chunkId(doc.id, 0);
+        const marksOf = (doc) => {
+            const hash = hashes.get(doc.id);
+            return {
+                ...(hash !== undefined && { ragContentHash: hash }),
+                ragGeneration: generation,
+                ragSourceDocuments: group.length,
+            };
+        };
+        await this.storeChunks(writes.filter((write) => !isHead(write)), marksOf);
+        await this.storeChunks(writes.filter(isHead), marksOf);
+        let chunks = 0;
+        for (const document of prepared) {
+            chunks += document.chunks.length;
+            this.stats.chunksStored += document.chunks.length;
+            this.stats.documentsIngested++;
+        }
+        this.locallyIndexed.add(sourceName);
+        if (previous.generations) {
+            const stale = new Set(previous.generations);
+            for (const old of await this.leftGenerations(group, prepared, generation))
+                stale.add(old);
+            for (const old of stale) {
+                if (old < generation)
+                    await this.forgetGeneration(sourceName, old);
+            }
+            if (await this.supersededBy(group, generation)) {
+                await this.forgetGeneration(sourceName, generation);
             }
         }
-        return { documents: documents.length, chunks: totalChunks };
+        return { chunks, unchanged: 0 };
     }
     /**
      * Deletes every chunk this pipeline stored for `source`: the `source` its results carry (for
@@ -93,16 +208,228 @@ export class RAGPipeline {
     namespaceMetadata() {
         return this.config.namespace === undefined ? {} : { namespace: this.config.namespace };
     }
-    async prepareDocument(doc) {
+    /**
+     * Hash of what a document's chunks are made from: its text, source and metadata, the chunker's
+     * fingerprint and the embedding model. None when the chunker has no fingerprint, as the same
+     * text may then give other chunks.
+     */
+    contentHash(doc) {
+        const { embeddingService, chunker } = this.deps;
+        if (chunker.fingerprint === undefined)
+            return undefined;
+        return createHash('sha256')
+            .update(JSON.stringify([
+            HASH_VERSION,
+            doc.content,
+            doc.source,
+            doc.sourceType,
+            doc.metadata ?? {},
+            chunker.fingerprint,
+            embeddingService.model,
+            embeddingService.dimensions,
+        ]))
+            .digest('hex');
+    }
+    /**
+     * Gives a local keyword index of the retriever the stored chunks of an unchanged source, once
+     * per process: a new pipeline has an empty one, and skipping the source would leave it so.
+     */
+    async indexLocally(source, group) {
+        const retriever = this.deps.retriever;
+        if (!isChunkIndexer(retriever) || this.locallyIndexed.has(source))
+            return;
+        const ids = [];
+        for (const doc of group) {
+            for (const chunk of await this.chunkDocument(doc))
+                ids.push(chunk.id);
+        }
+        const stored = await this.storedEmbeddings(ids);
+        for (const embedding of stored) {
+            const metadata = embedding.metadata ?? {};
+            retriever.indexChunk({
+                embeddingId: embedding.id,
+                chunkId: embedding.sourceId,
+                documentId: String(metadata.documentId),
+                content: embedding.content,
+                metadata,
+            });
+        }
+        this.locallyIndexed.add(source);
+    }
+    /**
+     * The stored chunks of this pipeline's namespace with these ids, by id: more than one for an id
+     * when an ingest stopped before deleting the generation it replaced. Empty when the store cannot
+     * read chunks by id.
+     */
+    async storedChunks(ids) {
+        const found = new Map();
+        for (const embedding of await this.storedEmbeddings(ids)) {
+            const chunk = this.ownChunk(embedding);
+            if (chunk)
+                found.set(embedding.sourceId, [...(found.get(embedding.sourceId) ?? []), chunk]);
+        }
+        return found;
+    }
+    /** The stored embeddings of this pipeline's documents with these chunk ids */
+    async storedEmbeddings(ids) {
+        const adapter = this.deps.embeddingAdapter;
+        if (!adapter.getBySources)
+            return [];
+        const found = [];
+        for (let start = 0; start < ids.length; start += LOOKUP_BATCH) {
+            const result = await adapter.getBySources(ids.slice(start, start + LOOKUP_BATCH));
+            if (!result.success) {
+                throw new Error(`Could not read the stored chunks: ${result.error}`);
+            }
+            for (const embedding of result.data) {
+                const metadata = embedding.metadata ?? {};
+                if (embedding.sourceType === 'document' && metadata.namespace === this.config.namespace) {
+                    found.push(embedding);
+                }
+            }
+        }
+        return found;
+    }
+    ownChunk(embedding) {
+        const metadata = embedding.metadata ?? {};
+        if (embedding.sourceType !== 'document' || metadata.namespace !== this.config.namespace) {
+            return undefined;
+        }
+        return {
+            documentId: String(metadata.documentId),
+            vector: embedding.vector,
+            ...(typeof metadata.ragContentHash === 'string' && { hash: metadata.ragContentHash }),
+            ...(typeof metadata.ragGeneration === 'string' && { generation: metadata.ragGeneration }),
+            ...(typeof metadata.ragSourceDocuments === 'number' && {
+                documents: metadata.ragSourceDocuments,
+            }),
+        };
+    }
+    /**
+     * What the last ingest left of a source: whether it is unchanged, and the generations to delete
+     * once a new one is written. No `generations` when the source has to be cleared first: nothing
+     * of it was found by id, or it was stored before generations.
+     */
+    previousGeneration(group, hashes, heads) {
+        const found = group.flatMap((doc) => (heads.get(chunkId(doc.id, 0)) ?? []).map((head) => ({ doc, head })));
+        if (!this.deps.embeddingAdapter.deleteByFilter ||
+            found.length === 0 ||
+            found.some(({ head }) => head.generation === undefined)) {
+            return { unchanged: false };
+        }
+        const generations = [...new Set(found.map(({ head }) => head.generation))];
+        const unchanged = found.length === group.length &&
+            generations.length === 1 &&
+            found.every(({ doc, head }) => head.hash !== undefined &&
+                head.hash === hashes.get(doc.id) &&
+                head.documents === group.length);
+        return { unchanged, generations };
+    }
+    async prepareDocument(doc, unchangedHead) {
         const chunks = await this.chunkDocument(doc);
-        const vectors = chunks.length > 0
-            ? await this.deps.embeddingService.embedBatch(chunks.map((c) => c.content))
-            : [];
+        const kept = unchangedHead ? await this.keptVectors(chunks, unchangedHead) : undefined;
+        const vectors = kept ??
+            (chunks.length > 0
+                ? await this.deps.embeddingService.embedBatch(chunks.map((c) => c.content))
+                : []);
         if (vectors.length !== chunks.length) {
             throw new Error(`Embedding count mismatch: got ${vectors.length} vectors for ${chunks.length} chunks`);
         }
         return { doc, chunks, vectors };
     }
+    /**
+     * The stored vectors of an unchanged document's chunks, in their order, when the store keeps
+     * all of them from the same ingest
+     */
+    async keptVectors(chunks, head) {
+        const stored = await this.storedChunks(chunks.map((chunk) => chunk.id));
+        const vectors = [];
+        for (const chunk of chunks) {
+            const kept = stored
+                .get(chunk.id)
+                ?.find((candidate) => candidate.generation === head.generation && candidate.hash === head.hash);
+            if (!kept)
+                return undefined;
+            if (kept.vector.length !== this.deps.embeddingService.dimensions)
+                return undefined;
+            vectors.push(kept.vector);
+        }
+        return vectors;
+    }
+    /**
+     * Generations other than `kept` found on the second chunks of the documents, which an ingest
+     * that stopped before the heads wrote first: older ones, and newer ones old enough that no
+     * ingest is still writing them
+     */
+    unfinishedGenerations(group, stored, kept) {
+        const found = new Set();
+        for (const doc of group) {
+            for (const chunk of stored.get(chunkId(doc.id, 1)) ?? []) {
+                const generation = chunk.generation;
+                if (!generation || generation === kept)
+                    continue;
+                const abandoned = generationTime(generation) < Date.now() - ABANDONED_AFTER_MS;
+                if (generation < kept || abandoned)
+                    found.add(generation);
+            }
+        }
+        return [...found];
+    }
+    /**
+     * Generations other than `current` the store holds chunks of for these documents: ones an
+     * ingest stopped writing before their heads, which no head names
+     */
+    async leftGenerations(group, prepared, current) {
+        const ids = new Set();
+        for (const { doc, chunks } of prepared) {
+            for (const chunk of chunks)
+                ids.add(chunk.id);
+            for (let order = 0; order < PROBED_CHUNKS; order++)
+                ids.add(chunkId(doc.id, order));
+        }
+        const documents = new Set(group.map((doc) => doc.id));
+        const found = new Set();
+        for (const chunks of (await this.storedChunks([...ids])).values()) {
+            for (const chunk of chunks) {
+                if (chunk.generation && chunk.generation !== current && documents.has(chunk.documentId)) {
+                    found.add(chunk.generation);
+                }
+            }
+        }
+        return [...found];
+    }
+    /**
+     * Whether an ingest that started after the one that wrote `generation` already wrote heads of
+     * these documents, so the later one wins and `generation` goes
+     */
+    async supersededBy(group, generation) {
+        const heads = await this.storedChunks(group.map((doc) => chunkId(doc.id, 0)));
+        for (const chunks of heads.values()) {
+            if (chunks.some((chunk) => chunk.generation !== undefined && chunk.generation > generation)) {
+                return true;
+            }
+        }
+        return false;
+    }
+    /** Deletes the chunks of `source` an earlier ingest wrote as `generation` */
+    async forgetGeneration(source, generation) {
+        const adapter = this.deps.embeddingAdapter;
+        if (!adapter.deleteByFilter)
+            return;
+        const result = await adapter.deleteByFilter({
+            sourceType: 'document',
+            metadata: { source, ragGeneration: generation, ...this.namespaceMetadata() },
+        });
+        if (!result.success) {
+            throw new Error(`Could not remove the previous chunks of ${source}: ${result.error}`);
+        }
+        const retriever = this.deps.retriever;
+        if (isChunkIndexer(retriever)) {
+            retriever.removeChunks?.((chunk) => chunk.metadata.source === source &&
+                chunk.metadata.ragGeneration === generation &&
+                chunk.metadata.namespace === this.config.namespace);
+        }
+    }
     /** Deletes the stored chunks of `source` in this pipeline's namespace, and drops them from a local index. */
     async forgetSource(source) {
         const adapter = this.deps.embeddingAdapter;
@@ -130,17 +457,27 @@ export class RAGPipeline {
         const chunks = await this.deps.chunker.chunk(doc.content, doc.id);
         return chunks.filter((chunk) => chunk.content.trim().length > 0);
     }
-    async storeChunks(chunks, vectors, doc) {
+    async storeChunks(writes, marksOf) {
         let next = 0;
+        let failed = false;
         const worker = async () => {
-            while (next < chunks.length) {
-                const index = next++;
-                await this.storeChunk(chunks[index], vectors[index], doc);
+            while (next < writes.length && !failed) {
+                const { chunk, vector, doc } = writes[next++];
+                try {
+                    await this.storeChunk(chunk, vector, doc, marksOf(doc));
+                }
+                catch (error) {
+                    failed = true;
+                    throw error;
+                }
             }
         };
-        await Promise.all(Array.from({ length: Math.min(STORE_CONCURRENCY, chunks.length) }, () => worker()));
+        const settled = await Promise.allSettled(Array.from({ length: Math.min(STORE_CONCURRENCY, writes.length) }, () => worker()));
+        const failure = settled.find((result) => result.status === 'rejected');
+        if (failure)
+            throw failure.reason;
     }
-    async storeChunk(chunk, vector, doc) {
+    async storeChunk(chunk, vector, doc, marks) {
         const metadata = {
             ...doc.metadata,
             ...chunk.metadata,
@@ -150,6 +487,7 @@ export class RAGPipeline {
             order: chunk.order,
             startOffset: chunk.startOffset,
             endOffset: chunk.endOffset,
+            ...marks,
             ...this.namespaceMetadata(),
         };
         const result = await this.deps.embeddingAdapter.addEmbedding({

package/dist/rag-pipeline.js.map

@@ -1 +1 @@
-{"version":3,"file":"rag-pipeline.js","sourceRoot":"","sources":["../src/rag-pipeline.ts"],"names":[],"mappings":"AAeA,OAAO,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAW/D,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAE7B,SAAS,cAAc,CAAmB,KAAoB;IAC5D,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,CAAC;IACtB,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAe,CAAC;AACpG,CAAC;AAQD,kFAAkF;AAClF,SAAS,aAAa,CAAC,SAAwB;IAC7C,MAAM,MAAM,GAAG,IAAI,GAAG,EAAyB,CAAC;IAChD,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACrC,IAAI,KAAK;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;;YACtB,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;AAC9B,CAAC;AAED,MAAM,OAAO,WAAW;IACL,MAAM,CAAoB;IAC1B,IAAI,CAAkB;IAC/B,KAAK,GAAG,EAAE,iBAAiB,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAC;IACvE,gBAAgB,GAAG,KAAK,CAAC;IAEjC,YAAY,MAAyB,EAAE,IAAqB,EAAE,aAAa,GAAG,KAAK;QACjF,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,uBAAuB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC7E,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED;;;;OAIG;IACH,KAAK,CAAC,MAAM,CAAC,MAAc;QACzB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACtD,IAAI,WAAW,GAAG,CAAC,CAAC;QAEpB,KAAK,MAAM,KAAK,IAAI,aAAa,CAAC,SAAS,CAAC,EAAE,CAAC;YAC7C,MAAM,QAAQ,GAAuB,EAAE,CAAC;YACxC,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACxB,QAAQ,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC;YACjD,CAAC;YAED,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC,CAAC;YAC1C,KAAK,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,IAAI,QAAQ,EAAE,CAAC;gBAChD,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,OAAO,EAAE,GAAG,CAAC,CAAC;gBAC7C,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC;gBAC7B,IAAI,CAAC,KAAK,CAAC,YAAY,IAAI,MAAM,CAAC,MAAM,CAAC;gBACzC,IAAI,CAAC,KAAK,CAAC,iBAAiB,EAAE,CAAC;YACjC,CAAC;QACH,CAAC;QAED,OAAO,EAAE,SAAS,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,CAAC;IAC9D,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAC,MAAc;QAC/B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CACb,yFAAyF,CAC1F,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,OAAkC;QAC1D,MAAM,MAAM,GAA6B;YACvC,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS;YACxB,GAAG,cAAc,CAAC,OAAO,CAAC;SAC3B,CAAC;QACF,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE;YACvD,GAAG,MAAM;YACT,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;SAClC,CAAC,CAAC;QACH,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAE9B,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC;YACzD,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC9E,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,QAAQ;QACN,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,MAAgC;QAC5C,MAAM,QAAQ,GAAG,EAAE,GAAG,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,iBAAiB,EAAE,EAAE,CAAC;QACtE,OAAO;YACL,GAAG,MAAM;YACT,UAAU,EAAE,UAAU;YACtB,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC;SACtD,CAAC;IACJ,CAAC;IAEO,iBAAiB;QACvB,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;IACzF,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,GAAgB;QAC5C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;QAC7C,MAAM,OAAO,GACX,MAAM,CAAC,MAAM,GAAG,CAAC;YACf,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAC3E,CAAC,CAAC,EAAE,CAAC;QACT,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CACb,iCAAiC,OAAO,CAAC,MAAM,gBAAgB,MAAM,CAAC,MAAM,SAAS,CACtF,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IAClC,CAAC;IAED,6GAA6G;IACrG,KAAK,CAAC,YAAY,CAAC,MAAc;QACvC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC;QAC3C,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YAC5B,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC3B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;gBAC7B,OAAO,CAAC,IAAI,CACV,sFAAsF;oBACpF,gEAAgE,CACnE,CAAC;YACJ,CAAC;YACD,OAAO;QACT,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,cAAc,CAAC;YAC1C,UAAU,EAAE,UAAU;YACtB,QAAQ,EAAE,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,iBAAiB,EAAE,EAAE;SAClD,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,2CAA2C,MAAM,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QACxF,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;QACtC,IAAI,cAAc,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9B,SAAS,CAAC,YAAY,EAAE,CACtB,CAAC,KAAK,EAAE,EAAE,CACR,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,KAAK,IAAI,CAAC,MAAM,CAAC,SAAS,CACzF,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,GAAgB;QAC1C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAClE,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,MAAuB,EACvB,OAAmB,EACnB,GAAgB;QAEhB,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;YACxB,OAAO,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC;gBAC5B,MAAM,KAAK,GAAG,IAAI,EAAE,CAAC;gBACrB,MAAM,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,KAAK,CAAE,EAAE,OAAO,CAAC,KAAK,CAAE,EAAE,GAAG,CAAC,CAAC;YAC9D,CAAC;QACH,CAAC,CAAC;QACF,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,iBAAiB,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CACnF,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,KAAoB,EAAE,MAAgB,EAAE,GAAgB;QAC/E,MAAM,QAAQ,GAA4B;YACxC,GAAG,GAAG,CAAC,QAAQ;YACf,GAAG,KAAK,CAAC,QAAQ;YACjB,UAAU,EAAE,GAAG,CAAC,EAAE;YAClB,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,GAAG,IAAI,CAAC,iBAAiB,EAAE;SAC5B,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC;YAC3D,QAAQ,EAAE,KAAK,CAAC,EAAE;YAClB,UAAU,EAAE,UAAU;YACtB,MAAM;YACN,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,QAAQ;SACT,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YACxC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC;gBAC7B,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE;gBAC3B,OAAO,EAAE,KAAK,CAAC,EAAE;gBACjB,UAAU,EAAE,GAAG,CAAC,EAAE;gBAClB,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,QAAQ;aACT,CAAC,CAAC;QACL,CAAC;IACH,CAAC;CACF"}
+{"version":3,"file":"rag-pipeline.js","sourceRoot":"","sources":["../src/rag-pipeline.ts"],"names":[],"mappings":"AAgBA,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,uBAAuB,EAAE,MAAM,aAAa,CAAC;AACtD,OAAO,EAAE,cAAc,EAAE,MAAM,+BAA+B,CAAC;AAC/D,OAAO,EAAE,OAAO,EAAE,MAAM,wBAAwB,CAAC;AAWjD,MAAM,iBAAiB,GAAG,EAAE,CAAC;AAE7B,4FAA4F;AAC5F,MAAM,YAAY,GAAG,CAAC,CAAC;AAEvB,+CAA+C;AAC/C,MAAM,YAAY,GAAG,GAAG,CAAC;AAEzB;;;GAGG;AACH,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB;;;GAGG;AACH,MAAM,kBAAkB,GAAG,EAAE,GAAG,MAAM,CAAC;AAEvC,IAAI,kBAAkB,GAAG,CAAC,CAAC;AAE3B;;;GAGG;AACH,SAAS,aAAa;IACpB,kBAAkB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,kBAAkB,GAAG,CAAC,CAAC,CAAC;IAClE,OAAO,GAAG,kBAAkB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,UAAU,EAAE,EAAE,CAAC;AAC/E,CAAC;AAED,8DAA8D;AAC9D,SAAS,cAAc,CAAC,UAAkB;IACxC,MAAM,KAAK,GAAG,iBAAiB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACjD,OAAO,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAE,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACpD,CAAC;AAED,SAAS,cAAc,CAAmB,KAAoB;IAC5D,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,CAAC;IACtB,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAe,CAAC;AACpG,CAAC;AAsCD,kFAAkF;AAClF,SAAS,aAAa,CAAC,SAAwB;IAC7C,MAAM,MAAM,GAAG,IAAI,GAAG,EAAyB,CAAC;IAChD,KAAK,MAAM,GAAG,IAAI,SAAS,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QACrC,IAAI,KAAK;YAAE,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;;YACtB,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;AAC9B,CAAC;AAED,MAAM,OAAO,WAAW;IACL,MAAM,CAAoB;IAC1B,IAAI,CAAkB;IAC/B,KAAK,GAAG,EAAE,iBAAiB,EAAE,CAAC,EAAE,YAAY,EAAE,CAAC,EAAE,gBAAgB,EAAE,CAAC,EAAE,CAAC;IACvE,gBAAgB,GAAG,KAAK,CAAC;IACjC,yFAAyF;IACxE,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;IACpD,0FAA0F;IACzE,SAAS,GAAG,IAAI,GAAG,EAA4B,CAAC;IAEjE,YAAY,MAAyB,EAAE,IAAqB,EAAE,aAAa,GAAG,KAAK;QACjF,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,uBAAuB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QAC7E,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC;IACnB,CAAC;IAED;;;;;;;;;;;;;;OAcG;IACH,KAAK,CAAC,MAAM,CAAC,MAAc;QACzB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QACtD,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAI,SAAS,GAAG,CAAC,CAAC;QAElB,KAAK,MAAM,KAAK,IAAI,aAAa,CAAC,SAAS,CAAC,EAAE,CAAC;YAC7C,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,MAAM,EAAE,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC;YACpF,WAAW,IAAI,IAAI,CAAC,MAAM,CAAC;YAC3B,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC;QAC9B,CAAC;QAED,OAAO,EAAE,SAAS,EAAE,SAAS,CAAC,MAAM,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,CAAC;IACzE,CAAC;IAED,uFAAuF;IAC/E,KAAK,CAAC,UAAU,CAAI,MAAc,EAAE,IAAsB;QAChE,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;QAC/D,MAAM,GAAG,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACrD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;QAChC,IAAI,CAAC;YACH,OAAO,MAAM,GAAG,CAAC;QACnB,CAAC;gBAAS,CAAC;YACT,IAAI,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,KAAK,GAAG;gBAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACxE,CAAC;IACH,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,WAAW,CAAC,KAAoB;QAC5C,MAAM,UAAU,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC,MAAM,CAAC;QACpC,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CACpC,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CACjE,CAAC;QACF,MAAM,KAAK,GAAG,IAAI,GAAG,CACnB,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAC/E,CAAC;QACF,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAAkB,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;QAC/D,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;YACvB,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAY,CAAC,CAAC,CAAE,CAAC;YACvC,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,qBAAqB,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC;gBAClE,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;YAC/C,CAAC;YACD,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;YAC3C,OAAO,EAAE,MAAM,EAAE,CAAC,EAAE,SAAS,EAAE,KAAK,CAAC,MAAM,EAAE,CAAC;QAChD,CAAC;QAED,MAAM,QAAQ,GAAuB,EAAE,CAAC;QACxC,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;YACxB,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAChC,MAAM,QAAQ,GACZ,IAAI,KAAK,SAAS;gBAChB,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,KAAK;qBACF,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;oBACxB,EAAE,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,KAAK,SAAS,IAAI,IAAI,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;YAC9E,QAAQ,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CAAC;QAC3D,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,WAAW;YAAE,MAAM,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;QAC/D,MAAM,UAAU,GAAG,aAAa,EAAE,CAAC;QACnC,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,EAAE,EAAE,CAC3D,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAE,EAAE,GAAG,EAAE,CAAC,CAAC,CAChE,CAAC;QACF,MAAM,MAAM,GAAG,CAAC,EAAE,KAAK,EAAE,GAAG,EAAc,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAC/E,MAAM,OAAO,GAAG,CAAC,GAAgB,EAAe,EAAE;YAChD,MAAM,IAAI,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;YAChC,OAAO;gBACL,GAAG,CAAC,IAAI,KAAK,SAAS,IAAI,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC;gBACnD,aAAa,EAAE,UAAU;gBACzB,kBAAkB,EAAE,KAAK,CAAC,MAAM;aACjC,CAAC;QACJ,CAAC,CAAC;QACF,MAAM,IAAI,CAAC,WAAW,CACpB,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EACxC,OAAO,CACR,CAAC;QACF,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;QACvD,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,KAAK,MAAM,QAAQ,IAAI,QAAQ,EAAE,CAAC;YAChC,MAAM,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC;YACjC,IAAI,CAAC,KAAK,CAAC,YAAY,IAAI,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC;YAClD,IAAI,CAAC,KAAK,CAAC,iBAAiB,EAAE,CAAC;QACjC,CAAC;QACD,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QACpC,IAAI,QAAQ,CAAC,WAAW,EAAE,CAAC;YACzB,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;YAC5C,KAAK,MAAM,GAAG,IAAI,MAAM,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,QAAQ,EAAE,UAAU,CAAC;gBAAE,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC1F,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;gBACxB,IAAI,GAAG,GAAG,UAAU;oBAAE,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;YACrE,CAAC;YACD,IAAI,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,EAAE,UAAU,CAAC,EAAE,CAAC;gBAC/C,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;YACtD,CAAC;QACH,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;IAClC,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,YAAY,CAAC,MAAc;QAC/B,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,CAAC;YAC/C,MAAM,IAAI,KAAK,CACb,yFAAyF,CAC1F,CAAC;QACJ,CAAC;QACD,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;IAClC,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,OAAkC;QAC1D,MAAM,MAAM,GAA6B;YACvC,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS;YACxB,GAAG,cAAc,CAAC,OAAO,CAAC;SAC3B,CAAC;QACF,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE;YACvD,GAAG,MAAM;YACT,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC;SAClC,CAAC,CAAC;QACH,IAAI,CAAC,KAAK,CAAC,gBAAgB,EAAE,CAAC;QAE9B,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,OAAO,EAAE,CAAC;YACzD,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QAC9E,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAED,QAAQ;QACN,OAAO,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;IAC3B,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,MAAgC;QAC5C,MAAM,QAAQ,GAAG,EAAE,GAAG,MAAM,EAAE,QAAQ,EAAE,GAAG,IAAI,CAAC,iBAAiB,EAAE,EAAE,CAAC;QACtE,OAAO;YACL,GAAG,MAAM;YACT,UAAU,EAAE,UAAU;YACtB,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC;SACtD,CAAC;IACJ,CAAC;IAEO,iBAAiB;QACvB,OAAO,IAAI,CAAC,MAAM,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;IACzF,CAAC;IAED;;;;OAIG;IACK,WAAW,CAAC,GAAgB;QAClC,MAAM,EAAE,gBAAgB,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC;QAChD,IAAI,OAAO,CAAC,WAAW,KAAK,SAAS;YAAE,OAAO,SAAS,CAAC;QACxD,OAAO,UAAU,CAAC,QAAQ,CAAC;aACxB,MAAM,CACL,IAAI,CAAC,SAAS,CAAC;YACb,YAAY;YACZ,GAAG,CAAC,OAAO;YACX,GAAG,CAAC,MAAM;YACV,GAAG,CAAC,UAAU;YACd,GAAG,CAAC,QAAQ,IAAI,EAAE;YAClB,OAAO,CAAC,WAAW;YACnB,gBAAgB,CAAC,KAAK;YACtB,gBAAgB,CAAC,UAAU;SAC5B,CAAC,CACH;aACA,MAAM,CAAC,KAAK,CAAC,CAAC;IACnB,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,YAAY,CAAC,MAAc,EAAE,KAAoB;QAC7D,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;QACtC,IAAI,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,OAAO;QAC1E,MAAM,GAAG,GAAa,EAAE,CAAC;QACzB,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;YACxB,KAAK,MAAM,KAAK,IAAI,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC;gBAAE,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QACxE,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAChD,KAAK,MAAM,SAAS,IAAI,MAAM,EAAE,CAAC;YAC/B,MAAM,QAAQ,GAAG,SAAS,CAAC,QAAQ,IAAI,EAAE,CAAC;YAC1C,SAAS,CAAC,UAAU,CAAC;gBACnB,WAAW,EAAE,SAAS,CAAC,EAAE;gBACzB,OAAO,EAAE,SAAS,CAAC,QAAQ;gBAC3B,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC;gBACvC,OAAO,EAAE,SAAS,CAAC,OAAO;gBAC1B,QAAQ;aACT,CAAC,CAAC;QACL,CAAC;QACD,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAClC,CAAC;IAED;;;;OAIG;IACK,KAAK,CAAC,YAAY,CAAC,GAAa;QACtC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAyB,CAAC;QAC/C,KAAK,MAAM,SAAS,IAAI,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC;YACzD,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;YACvC,IAAI,KAAK;gBAAE,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,QAAQ,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;QAC9F,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,8EAA8E;IACtE,KAAK,CAAC,gBAAgB,CAAC,GAAa;QAC1C,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC;QAC3C,IAAI,CAAC,OAAO,CAAC,YAAY;YAAE,OAAO,EAAE,CAAC;QACrC,MAAM,KAAK,GAAgB,EAAE,CAAC;QAC9B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,GAAG,CAAC,MAAM,EAAE,KAAK,IAAI,YAAY,EAAE,CAAC;YAC9D,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,YAAY,CAAC,CAAC,CAAC;YAClF,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBACpB,MAAM,IAAI,KAAK,CAAC,qCAAqC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;YACvE,CAAC;YACD,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBACpC,MAAM,QAAQ,GAAG,SAAS,CAAC,QAAQ,IAAI,EAAE,CAAC;gBAC1C,IAAI,SAAS,CAAC,UAAU,KAAK,UAAU,IAAI,QAAQ,CAAC,SAAS,KAAK,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;oBACxF,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACxB,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,QAAQ,CAAC,SAAoB;QACnC,MAAM,QAAQ,GAAG,SAAS,CAAC,QAAQ,IAAI,EAAE,CAAC;QAC1C,IAAI,SAAS,CAAC,UAAU,KAAK,UAAU,IAAI,QAAQ,CAAC,SAAS,KAAK,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;YACxF,OAAO,SAAS,CAAC;QACnB,CAAC;QACD,OAAO;YACL,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC;YACvC,MAAM,EAAE,SAAS,CAAC,MAAM;YACxB,GAAG,CAAC,OAAO,QAAQ,CAAC,cAAc,KAAK,QAAQ,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC,cAAc,EAAE,CAAC;YACrF,GAAG,CAAC,OAAO,QAAQ,CAAC,aAAa,KAAK,QAAQ,IAAI,EAAE,UAAU,EAAE,QAAQ,CAAC,aAAa,EAAE,CAAC;YACzF,GAAG,CAAC,OAAO,QAAQ,CAAC,kBAAkB,KAAK,QAAQ,IAAI;gBACrD,SAAS,EAAE,QAAQ,CAAC,kBAAkB;aACvC,CAAC;SACH,CAAC;IACJ,CAAC;IAED;;;;OAIG;IACK,kBAAkB,CACxB,KAAoB,EACpB,MAAuC,EACvC,KAAiC;QAEjC,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAClC,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC,CACrE,CAAC;QACF,IACE,CAAC,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,cAAc;YAC1C,KAAK,CAAC,MAAM,KAAK,CAAC;YAClB,KAAK,CAAC,IAAI,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,KAAK,SAAS,CAAC,EACvD,CAAC;YACD,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;QAC9B,CAAC;QACD,MAAM,WAAW,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,UAAW,CAAC,CAAC,CAAC,CAAC;QAC5E,MAAM,SAAS,GACb,KAAK,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM;YAC7B,WAAW,CAAC,MAAM,KAAK,CAAC;YACxB,KAAK,CAAC,KAAK,CACT,CAAC,EAAE,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,CAChB,IAAI,CAAC,IAAI,KAAK,SAAS;gBACvB,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;gBAChC,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,MAAM,CAClC,CAAC;QACJ,OAAO,EAAE,SAAS,EAAE,WAAW,EAAE,CAAC;IACpC,CAAC;IAEO,KAAK,CAAC,eAAe,CAC3B,GAAgB,EAChB,aAA2B;QAE3B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAC;QAC7C,MAAM,IAAI,GAAG,aAAa,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;QACvF,MAAM,OAAO,GACX,IAAI;YACJ,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC;gBAChB,CAAC,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;gBAC3E,CAAC,CAAC,EAAE,CAAC,CAAC;QACV,IAAI,OAAO,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;YACrC,MAAM,IAAI,KAAK,CACb,iCAAiC,OAAO,CAAC,MAAM,gBAAgB,MAAM,CAAC,MAAM,SAAS,CACtF,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IAClC,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,WAAW,CACvB,MAAuB,EACvB,IAAiB;QAEjB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC;QACxE,MAAM,OAAO,GAAe,EAAE,CAAC;QAC/B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,IAAI,GAAG,MAAM;iBAChB,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBACd,EAAE,IAAI,CACJ,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,UAAU,KAAK,IAAI,CAAC,UAAU,IAAI,SAAS,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,CACxF,CAAC;YACJ,IAAI,CAAC,IAAI;gBAAE,OAAO,SAAS,CAAC;YAC5B,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,UAAU;gBAAE,OAAO,SAAS,CAAC;YACnF,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC5B,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAED;;;;OAIG;IACK,qBAAqB,CAC3B,KAAoB,EACpB,MAAkC,EAClC,IAAY;QAEZ,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;QAChC,KAAK,MAAM,GAAG,IAAI,KAAK,EAAE,CAAC;YACxB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;gBACzD,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC;gBACpC,IAAI,CAAC,UAAU,IAAI,UAAU,KAAK,IAAI;oBAAE,SAAS;gBACjD,MAAM,SAAS,GAAG,cAAc,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,kBAAkB,CAAC;gBAC/E,IAAI,UAAU,GAAG,IAAI,IAAI,SAAS;oBAAE,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAC5D,CAAC;QACH,CAAC;QACD,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC;IACpB,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,eAAe,CAC3B,KAAoB,EACpB,QAA4B,EAC5B,OAAe;QAEf,MAAM,GAAG,GAAG,IAAI,GAAG,EAAU,CAAC;QAC9B,KAAK,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;YACvC,KAAK,MAAM,KAAK,IAAI,MAAM;gBAAE,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YAC9C,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,aAAa,EAAE,KAAK,EAAE;gBAAE,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC,CAAC;QACtF,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QACtD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;QAChC,KAAK,MAAM,MAAM,IAAI,CAAC,MAAM,IAAI,CAAC,YAAY,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC;YAClE,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,IAAI,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,KAAK,OAAO,IAAI,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC;oBACxF,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;gBAC9B,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC;IACpB,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,YAAY,CAAC,KAAoB,EAAE,UAAkB;QACjE,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9E,KAAK,MAAM,MAAM,IAAI,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC;YACpC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,KAAK,SAAS,IAAI,KAAK,CAAC,UAAU,GAAG,UAAU,CAAC,EAAE,CAAC;gBAC5F,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,6EAA6E;IACrE,KAAK,CAAC,gBAAgB,CAAC,MAAc,EAAE,UAAkB;QAC/D,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC;QAC3C,IAAI,CAAC,OAAO,CAAC,cAAc;YAAE,OAAO;QACpC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,cAAc,CAAC;YAC1C,UAAU,EAAE,UAAU;YACtB,QAAQ,EAAE,EAAE,MAAM,EAAE,aAAa,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC,iBAAiB,EAAE,EAAE;SAC7E,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,2CAA2C,MAAM,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QACxF,CAAC;QACD,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;QACtC,IAAI,cAAc,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9B,SAAS,CAAC,YAAY,EAAE,CACtB,CAAC,KAAK,EAAE,EAAE,CACR,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,MAAM;gBAChC,KAAK,CAAC,QAAQ,CAAC,aAAa,KAAK,UAAU;gBAC3C,KAAK,CAAC,QAAQ,CAAC,SAAS,KAAK,IAAI,CAAC,MAAM,CAAC,SAAS,CACrD,CAAC;QACJ,CAAC;IACH,CAAC;IAED,6GAA6G;IACrG,KAAK,CAAC,YAAY,CAAC,MAAc;QACvC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC;QAC3C,IAAI,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC;YAC5B,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC3B,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;gBAC7B,OAAO,CAAC,IAAI,CACV,sFAAsF;oBACpF,gEAAgE,CACnE,CAAC;YACJ,CAAC;YACD,OAAO;QACT,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,cAAc,CAAC;YAC1C,UAAU,EAAE,UAAU;YACtB,QAAQ,EAAE,EAAE,MAAM,EAAE,GAAG,IAAI,CAAC,iBAAiB,EAAE,EAAE;SAClD,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,2CAA2C,MAAM,KAAK,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC;QACxF,CAAC;QAED,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC;QACtC,IAAI,cAAc,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9B,SAAS,CAAC,YAAY,EAAE,CACtB,CAAC,KAAK,EAAE,EAAE,CACR,KAAK,CAAC,QAAQ,CAAC,MAAM,KAAK,MAAM,IAAI,KAAK,CAAC,QAAQ,CAAC,SAAS,KAAK,IAAI,CAAC,MAAM,CAAC,SAAS,CACzF,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CAAC,GAAgB;QAC1C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,OAAO,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAClE,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACnE,CAAC;IAEO,KAAK,CAAC,WAAW,CACvB,MAAoB,EACpB,OAA0C;QAE1C,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,IAAI,MAAM,GAAG,KAAK,CAAC;QACnB,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;YACxB,OAAO,IAAI,GAAG,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,EAAE,CAAC;gBACvC,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,MAAM,CAAC,IAAI,EAAE,CAAE,CAAC;gBAC/C,IAAI,CAAC;oBACH,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;gBAC1D,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,GAAG,IAAI,CAAC;oBACd,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC;QACH,CAAC,CAAC;QACF,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CACtC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,iBAAiB,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC,CACnF,CAAC;QACF,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC;QACvE,IAAI,OAAO;YAAE,MAAM,OAAO,CAAC,MAAM,CAAC;IACpC,CAAC;IAEO,KAAK,CAAC,UAAU,CACtB,KAAoB,EACpB,MAAgB,EAChB,GAAgB,EAChB,KAAkB;QAElB,MAAM,QAAQ,GAA4B;YACxC,GAAG,GAAG,CAAC,QAAQ;YACf,GAAG,KAAK,CAAC,QAAQ;YACjB,UAAU,EAAE,GAAG,CAAC,EAAE;YAClB,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,KAAK,EAAE,KAAK,CAAC,KAAK;YAClB,WAAW,EAAE,KAAK,CAAC,WAAW;YAC9B,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,GAAG,KAAK;YACR,GAAG,IAAI,CAAC,iBAAiB,EAAE;SAC5B,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,gBAAgB,CAAC,YAAY,CAAC;YAC3D,QAAQ,EAAE,KAAK,CAAC,EAAE;YAClB,UAAU,EAAE,UAAU;YACtB,MAAM;YACN,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,QAAQ;SACT,CAAC,CAAC;QAEH,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAChC,CAAC;QAED,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YACxC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC;gBAC7B,WAAW,EAAE,MAAM,CAAC,IAAI,CAAC,EAAE;gBAC3B,OAAO,EAAE,KAAK,CAAC,EAAE;gBACjB,UAAU,EAAE,GAAG,CAAC,EAAE;gBAClB,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,QAAQ;aACT,CAAC,CAAC;QACL,CAAC;IACH,CAAC;CACF"}

package/dist/tools.js

@@ -64,8 +64,8 @@ export function createIngestTool(pipeline, options = {}) {
             try {
                 const { source } = IngestParamsSchema.parse(params);
                 await assertAllowedSource(source, options);
-                const { documents, chunks } = await pipeline.ingest(source);
-                return { success: true, source, documents, chunks };
+                const { documents, chunks, unchanged } = await pipeline.ingest(source);
+                return { success: true, source, documents, chunks, unchanged };
             }
             catch (err) {
                 return { success: false, error: err instanceof Error ? err.message : String(err) };

package/dist/tools.js.map

@@ -1 +1 @@
-{"version":3,"file":"tools.js","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1D,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAUxB,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,qCAAqC,CAAC;IACxE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,qCAAqC,CAAC;IAC7F,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,+BAA+B,CAAC;CACzF,CAAC,CAAC;AAIH,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,uCAAuC,CAAC;CAC5E,CAAC,CAAC;AAIH,MAAM,UAAU,gBAAgB,CAAC,QAAqB;IACpD,OAAO;QACL,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,iDAAiD;QAC9D,UAAU,EAAE,kBAAkB;QAC9B,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YACxB,IAAI,CAAC;gBACH,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBACrE,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;gBACxE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YAClE,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACrF,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAYD,SAAS,KAAK,CAAC,MAAc;IAC3B,OAAO,0BAA0B,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACjD,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,IAAY;IACvC,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,MAAc,EAAE,OAA0B;IAC3E,IAAI,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;QAClB,IAAI,OAAO,CAAC,SAAS,KAAK,KAAK,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACjD,CAAC;QACD,OAAO;IACT,CAAC;IAED,IAAI,CAAC,OAAO,CAAC,YAAY;QAAE,OAAO;IAElC,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;IAC3C,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QACxC,MAAM,aAAa,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,GAAG,GAAG,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;QAC5C,IAAI,GAAG,KAAK,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YAAE,OAAO;IACxE,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,WAAW,MAAM,sCAAsC,CAAC,CAAC;AAC3E,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,QAAqB,EACrB,UAA6B,EAAE;IAE/B,OAAO;QACL,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,0CAA0C;QACvD,UAAU,EAAE,kBAAkB;QAC9B,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YACxB,IAAI,CAAC;gBACH,MAAM,EAAE,MAAM,EAAE,GAAG,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBACpD,MAAM,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;gBAC3C,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBAC5D,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;YACtD,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACrF,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,QAAQ,CACtB,QAAqB,EACrB,aAAiC;IAEjC,OAAO,CAAC,gBAAgB,CAAC,QAAQ,CAAC,EAAE,gBAAgB,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC;AACjF,CAAC"}
+{"version":3,"file":"tools.js","sourceRoot":"","sources":["../src/tools.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC1D,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAUxB,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,qCAAqC,CAAC;IACxE,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,qCAAqC,CAAC;IAC7F,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,QAAQ,CAAC,+BAA+B,CAAC;CACzF,CAAC,CAAC;AAIH,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,uCAAuC,CAAC;CAC5E,CAAC,CAAC;AAIH,MAAM,UAAU,gBAAgB,CAAC,QAAqB;IACpD,OAAO;QACL,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,iDAAiD;QAC9D,UAAU,EAAE,kBAAkB;QAC9B,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YACxB,IAAI,CAAC;gBACH,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBACrE,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;gBACxE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC;YAClE,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACrF,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAYD,SAAS,KAAK,CAAC,MAAc;IAC3B,OAAO,0BAA0B,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;AACjD,CAAC;AAED,KAAK,UAAU,aAAa,CAAC,IAAY;IACvC,IAAI,CAAC;QACH,OAAO,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC;IACvB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,MAAc,EAAE,OAA0B;IAC3E,IAAI,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC;QAClB,IAAI,OAAO,CAAC,SAAS,KAAK,KAAK,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;QACjD,CAAC;QACD,OAAO;IACT,CAAC;IAED,IAAI,CAAC,OAAO,CAAC,YAAY;QAAE,OAAO;IAElC,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,MAAM,CAAC,CAAC;IAC3C,KAAK,MAAM,IAAI,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;QACxC,MAAM,aAAa,GAAG,MAAM,aAAa,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,GAAG,GAAG,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC,CAAC;QAC5C,IAAI,GAAG,KAAK,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YAAE,OAAO;IACxE,CAAC;IACD,MAAM,IAAI,KAAK,CAAC,WAAW,MAAM,sCAAsC,CAAC,CAAC;AAC3E,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,QAAqB,EACrB,UAA6B,EAAE;IAE/B,OAAO;QACL,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,0CAA0C;QACvD,UAAU,EAAE,kBAAkB;QAC9B,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE;YACxB,IAAI,CAAC;gBACH,MAAM,EAAE,MAAM,EAAE,GAAG,kBAAkB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;gBACpD,MAAM,mBAAmB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;gBAC3C,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;gBACvE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,CAAC;YACjE,CAAC;YAAC,OAAO,GAAG,EAAE,CAAC;gBACb,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;YACrF,CAAC;QACH,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,QAAQ,CACtB,QAAqB,EACrB,aAAiC;IAEjC,OAAO,CAAC,gBAAgB,CAAC,QAAQ,CAAC,EAAE,gBAAgB,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC,CAAC;AACjF,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cogitator-ai/rag",
-  "version": "0.7.9",
+  "version": "0.8.1",
   "description": "Retrieval Augmented Generation for Cogitator AI agents",
   "keywords": [
     "cogitator",
@@ -31,11 +31,11 @@
     "node": ">=22.12.0"
   },
   "dependencies": {
-    "@cogitator-ai/types": "0.49.0",
+    "@cogitator-ai/types": "0.51.0",
     "zod": "^4.6.5"
   },
   "peerDependencies": {
-    "@cogitator-ai/memory": ">=0.14.0"
+    "@cogitator-ai/memory": ">=0.16.0"
   },
   "peerDependenciesMeta": {
     "@cogitator-ai/memory": {
@@ -48,7 +48,7 @@
     "pdf-parse": "^2.4.5"
   },
   "devDependencies": {
-    "@cogitator-ai/memory": "0.14.0",
+    "@cogitator-ai/memory": "0.16.0",
     "@types/papaparse": "^5.3.15",
     "typescript": "^6.0.3",
     "vitest": "^5.0.3"