@fllf/agent-sdk 0.1.1 → 0.1.3

package/dist/rag/generation/context-builder.js

@@ -6,15 +6,18 @@ class DefaultContextBuilder {
     maxContextTokens;
     maxChunkCharacters;
     includeScores;
+    minFusedScore;
     constructor(options = {}) {
         this.maxContextTokens = normalizePositiveInteger(options.maxContextTokens, 3000);
         this.maxChunkCharacters = normalizeOptionalPositiveInteger(options.maxChunkCharacters);
         this.includeScores = options.includeScores ?? false;
+        this.minFusedScore = options.minFusedScore;
     }
     async build(query, retrieved, options = {}) {
         const maxContextTokens = normalizePositiveInteger(options.maxContextTokens, this.maxContextTokens);
+        const minFusedScore = options.minFusedScore ?? this.minFusedScore;
         const candidates = dedupeRetrievedChunks(retrieved)
-            .filter((item) => options.minScore === undefined || item.score >= options.minScore);
+            .filter((item) => minFusedScore === undefined || item.score >= minFusedScore);
         const items = [];
         let tokenTotal = 0;
         let citationIndex = 1;

package/dist/rag/generation/context-builder.js.map

@@ -1 +1 @@
-{"version":3,"file":"context-builder.js","sourceRoot":"","sources":["../../../src/rag/generation/context-builder.ts"],"names":[],"mappings":";;;AAsNA,wCAKC;AArMD,MAAa,qBAAqB;IACb,gBAAgB,CAAS;IACzB,kBAAkB,CAAqB;IACvC,aAAa,CAAU;IAExC,YAAY,UAAwC,EAAE;QAClD,IAAI,CAAC,gBAAgB,GAAG,wBAAwB,CAAC,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAC;QACjF,IAAI,CAAC,kBAAkB,GAAG,gCAAgC,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;QACvF,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,KAAK,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,KAAK,CACP,KAAa,EACb,SAA2B,EAC3B,UAAyB,EAAE;QAE3B,MAAM,gBAAgB,GAAG,wBAAwB,CAC7C,OAAO,CAAC,gBAAgB,EACxB,IAAI,CAAC,gBAAgB,CACxB,CAAC;QACF,MAAM,UAAU,GAAG,qBAAqB,CAAC,SAAS,CAAC;aAC9C,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC,QAAQ,CAAC,CAAC;QACxF,MAAM,KAAK,GAAqB,EAAE,CAAC;QACnC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC5B,MAAM,OAAO,GAAG,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC;YAClF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvB,SAAS;YACb,CAAC;YAED,MAAM,QAAQ,GAAG,cAAc,CAAC,aAAa,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;YAC9D,MAAM,QAAQ,GAAG,kBAAkB,CAC/B,QAAQ,EACR,OAAO,EACP,mBAAmB,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,CACtD,CAAC;YACF,MAAM,aAAa,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;YAC/C,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,UAAU,GAAG,aAAa,GAAG,gBAAgB,EAAE,CAAC;gBACpE,MAAM;YACV,CAAC;YAED,KAAK,CAAC,IAAI,CAAC;gBACP,QAAQ;gBACR,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,OAAO;gBACP,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,aAAa;aAChB,CAAC,CAAC;YACH,UAAU,IAAI,aAAa,CAAC;YAC5B,aAAa,IAAI,CAAC,CAAC;QACvB,CAAC;QAED,MAAM,IAAI,GAAG,KAAK;aACb,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,kBAAkB,CAC7B,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,OAAO,EACZ,mBAAmB,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,CACtD,CAAC;aACD,IAAI,CAAC,MAAM,CAAC,CAAC;QAElB,OAAO;YACH,KAAK;YACL,IAAI;YACJ,KAAK;YACL,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC;YAC7C,eAAe,EAAE,UAAU;SAC9B,CAAC;IACN,CAAC;CACJ;AAvED,sDAuEC;AAED,SAAS,qBAAqB,CAAC,SAA2B;IACtD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IACvC,MAAM,eAAe,GAAG,IAAI,GAAG,EAAU,CAAC;IAC1C,MAAM,MAAM,GAAqB,EAAE,CAAC;IAEpC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,mBAAmB,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC3D,IAAI,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,eAAe,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;YACrE,SAAS;QACb,CAAC;QAED,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChC,eAAe,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAChC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtB,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,SAAS,cAAc,CACnB,KAAa,EACb,SAAyB,EACzB,OAAe;IAEf,MAAM,QAAQ,GAAa;QACvB,KAAK;QACL,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,EAAE;QAC3B,UAAU,EAAE,SAAS,CAAC,KAAK,CAAC,UAAU;QACtC,MAAM,EAAE,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM;KAC1C,CAAC;IAEF,eAAe,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACnE,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAEjE,IAAI,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;QACrD,QAAQ,CAAC,WAAW,GAAG,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IACrE,CAAC;IAED,QAAQ,CAAC,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IAEtC,MAAM,QAAQ,GAAG;QACb,eAAe,EAAE,SAAS,CAAC,MAAM;QACjC,KAAK,EAAE,SAAS,CAAC,KAAK;QACtB,GAAG,CAAC,SAAS,CAAC,QAAQ,IAAI,EAAE,CAAC;KAChC,CAAC;IACF,QAAQ,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAE7B,OAAO,QAAQ,CAAC;AACpB,CAAC;AAED,SAAS,kBAAkB,CACvB,QAAkB,EAClB,OAAe,EACf,UAA8B,EAAE;IAEhC,MAAM,UAAU,GAAG;QACf,UAAU,eAAe,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;KAC/C,CAAC;IAEF,IAAI,QAAQ,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC/B,UAAU,CAAC,IAAI,CAAC,SAAS,eAAe,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAChE,CAAC;IACD,IAAI,QAAQ,CAAC,WAAW,KAAK,SAAS,IAAI,QAAQ,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxE,UAAU,CAAC,IAAI,CAAC,WAAW,eAAe,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;IACpF,CAAC;IACD,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC9B,UAAU,CAAC,IAAI,CAAC,QAAQ,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAC7C,CAAC;IACD,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC9B,UAAU,CAAC,IAAI,CAAC,SAAS,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED,OAAO,IAAI,QAAQ,CAAC,KAAK,KAAK,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,OAAO,EAAE,CAAC;AACrE,CAAC;AAED,SAAS,mBAAmB,CAAC,aAAsB,EAAE,KAAa;IAC9D,MAAM,OAAO,GAAuB,EAAE,CAAC;IACvC,IAAI,aAAa,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC;IAC1B,CAAC;IAED,OAAO,OAAO,CAAC;AACnB,CAAC;AAED,SAAS,eAAe,CAAC,KAAa;IAClC,IAAI,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACpC,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;AACjC,CAAC;AAED,SAAS,WAAW,CAAC,KAAa;IAC9B,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACrE,CAAC;AAED,SAAS,oBAAoB,CAAC,OAAe,EAAE,kBAAsC;IACjF,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAClC,IAAI,kBAAkB,KAAK,SAAS,IAAI,UAAU,CAAC,MAAM,IAAI,kBAAkB,EAAE,CAAC;QAC9E,OAAO,UAAU,CAAC;IACtB,CAAC;IAED,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC,OAAO,EAAE,CAAC;AAC7D,CAAC;AAED,SAAS,WAAW,CAAC,OAAe;IAChC,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAEvD,OAAO,UAAU,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC;AACpF,CAAC;AAED,SAAS,mBAAmB,CAAC,OAAe;IACxC,OAAO,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AAC7D,CAAC;AAED;;;GAGG;AACH,SAAgB,cAAc,CAAC,IAAY;IACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC;IAC7D,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC;IAE3C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC;AAC9D,CAAC;AAED,SAAS,wBAAwB,CAAC,KAAyB,EAAE,QAAgB;IACzE,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QAC/D,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,gCAAgC,CAAC,KAAyB;IAC/D,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QAC/D,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,eAAe,CACpB,MAAS,EACT,GAAM,EACN,KAAuB;IAEvB,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;IACxB,CAAC;AACL,CAAC"}
+{"version":3,"file":"context-builder.js","sourceRoot":"","sources":["../../../src/rag/generation/context-builder.ts"],"names":[],"mappings":";;;AA+NA,wCAKC;AAxMD,MAAa,qBAAqB;IACb,gBAAgB,CAAS;IACzB,kBAAkB,CAAqB;IACvC,aAAa,CAAU;IACvB,aAAa,CAAqB;IAEnD,YAAY,UAAwC,EAAE;QAClD,IAAI,CAAC,gBAAgB,GAAG,wBAAwB,CAAC,OAAO,CAAC,gBAAgB,EAAE,IAAI,CAAC,CAAC;QACjF,IAAI,CAAC,kBAAkB,GAAG,gCAAgC,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC;QACvF,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,KAAK,CAAC;QACpD,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,KAAK,CACP,KAAa,EACb,SAA2B,EAC3B,UAAyB,EAAE;QAE3B,MAAM,gBAAgB,GAAG,wBAAwB,CAC7C,OAAO,CAAC,gBAAgB,EACxB,IAAI,CAAC,gBAAgB,CACxB,CAAC;QACF,MAAM,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,IAAI,CAAC,aAAa,CAAC;QAClE,MAAM,UAAU,GAAG,qBAAqB,CAAC,SAAS,CAAC;aAC9C,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,aAAa,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,IAAI,aAAa,CAAC,CAAC;QAClF,MAAM,KAAK,GAAqB,EAAE,CAAC;QACnC,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC5B,MAAM,OAAO,GAAG,oBAAoB,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,kBAAkB,CAAC,CAAC;YAClF,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvB,SAAS;YACb,CAAC;YAED,MAAM,QAAQ,GAAG,cAAc,CAAC,aAAa,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;YAC9D,MAAM,QAAQ,GAAG,kBAAkB,CAC/B,QAAQ,EACR,OAAO,EACP,mBAAmB,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,CACtD,CAAC;YACF,MAAM,aAAa,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;YAC/C,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,UAAU,GAAG,aAAa,GAAG,gBAAgB,EAAE,CAAC;gBACpE,MAAM;YACV,CAAC;YAED,KAAK,CAAC,IAAI,CAAC;gBACP,QAAQ;gBACR,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,OAAO;gBACP,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,aAAa;aAChB,CAAC,CAAC;YACH,UAAU,IAAI,aAAa,CAAC;YAC5B,aAAa,IAAI,CAAC,CAAC;QACvB,CAAC;QAED,MAAM,IAAI,GAAG,KAAK;aACb,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,kBAAkB,CAC7B,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,OAAO,EACZ,mBAAmB,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,KAAK,CAAC,CACtD,CAAC;aACD,IAAI,CAAC,MAAM,CAAC,CAAC;QAElB,OAAO;YACH,KAAK;YACL,IAAI;YACJ,KAAK;YACL,SAAS,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC;YAC7C,eAAe,EAAE,UAAU;SAC9B,CAAC;IACN,CAAC;CACJ;AA1ED,sDA0EC;AAED,SAAS,qBAAqB,CAAC,SAA2B;IACtD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IACvC,MAAM,eAAe,GAAG,IAAI,GAAG,EAAU,CAAC;IAC1C,MAAM,MAAM,GAAqB,EAAE,CAAC;IAEpC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC3B,MAAM,UAAU,GAAG,mBAAmB,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC3D,IAAI,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,IAAI,eAAe,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;YACrE,SAAS;QACb,CAAC;QAED,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChC,eAAe,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAChC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACtB,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,SAAS,cAAc,CACnB,KAAa,EACb,SAAyB,EACzB,OAAe;IAEf,MAAM,QAAQ,GAAa;QACvB,KAAK;QACL,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,EAAE;QAC3B,UAAU,EAAE,SAAS,CAAC,KAAK,CAAC,UAAU;QACtC,MAAM,EAAE,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM;KAC1C,CAAC;IAEF,eAAe,CAAC,QAAQ,EAAE,OAAO,EAAE,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACnE,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;IAEjE,IAAI,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;QACrD,QAAQ,CAAC,WAAW,GAAG,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IACrE,CAAC;IAED,QAAQ,CAAC,KAAK,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;IAEtC,MAAM,QAAQ,GAAG;QACb,eAAe,EAAE,SAAS,CAAC,MAAM;QACjC,KAAK,EAAE,SAAS,CAAC,KAAK;QACtB,GAAG,CAAC,SAAS,CAAC,QAAQ,IAAI,EAAE,CAAC;KAChC,CAAC;IACF,QAAQ,CAAC,QAAQ,GAAG,QAAQ,CAAC;IAE7B,OAAO,QAAQ,CAAC;AACpB,CAAC;AAED,SAAS,kBAAkB,CACvB,QAAkB,EAClB,OAAe,EACf,UAA8B,EAAE;IAEhC,MAAM,UAAU,GAAG;QACf,UAAU,eAAe,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE;KAC/C,CAAC;IAEF,IAAI,QAAQ,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC/B,UAAU,CAAC,IAAI,CAAC,SAAS,eAAe,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAChE,CAAC;IACD,IAAI,QAAQ,CAAC,WAAW,KAAK,SAAS,IAAI,QAAQ,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxE,UAAU,CAAC,IAAI,CAAC,WAAW,eAAe,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC;IACpF,CAAC;IACD,IAAI,QAAQ,CAAC,IAAI,KAAK,SAAS,EAAE,CAAC;QAC9B,UAAU,CAAC,IAAI,CAAC,QAAQ,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC;IAC7C,CAAC;IACD,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;QAC9B,UAAU,CAAC,IAAI,CAAC,SAAS,WAAW,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC3D,CAAC;IAED,OAAO,IAAI,QAAQ,CAAC,KAAK,KAAK,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,OAAO,EAAE,CAAC;AACrE,CAAC;AAED,SAAS,mBAAmB,CAAC,aAAsB,EAAE,KAAa;IAC9D,MAAM,OAAO,GAAuB,EAAE,CAAC;IACvC,IAAI,aAAa,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,GAAG,KAAK,CAAC;IAC1B,CAAC;IAED,OAAO,OAAO,CAAC;AACnB,CAAC;AAED,SAAS,eAAe,CAAC,KAAa;IAClC,IAAI,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACpC,OAAO,KAAK,CAAC;IACjB,CAAC;IAED,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;AACjC,CAAC;AAED,SAAS,WAAW,CAAC,KAAa;IAC9B,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AACrE,CAAC;AAED,SAAS,oBAAoB,CAAC,OAAe,EAAE,kBAAsC;IACjF,MAAM,UAAU,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;IAClC,IAAI,kBAAkB,KAAK,SAAS,IAAI,UAAU,CAAC,MAAM,IAAI,kBAAkB,EAAE,CAAC;QAC9E,OAAO,UAAU,CAAC;IACtB,CAAC;IAED,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAC,OAAO,EAAE,CAAC;AAC7D,CAAC;AAED,SAAS,WAAW,CAAC,OAAe;IAChC,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;IAEvD,OAAO,UAAU,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CAAC;AACpF,CAAC;AAED,SAAS,mBAAmB,CAAC,OAAe;IACxC,OAAO,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;AAC7D,CAAC;AAED;;;GAGG;AACH,SAAgB,cAAc,CAAC,IAAY;IACvC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,EAAE,MAAM,IAAI,CAAC,CAAC;IAC7D,MAAM,WAAW,GAAG,IAAI,CAAC,MAAM,GAAG,QAAQ,CAAC;IAE3C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,GAAG,WAAW,GAAG,CAAC,CAAC,CAAC,CAAC;AAC9D,CAAC;AAED,SAAS,wBAAwB,CAAC,KAAyB,EAAE,QAAgB;IACzE,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QAC/D,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,gCAAgC,CAAC,KAAyB;IAC/D,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QAC/D,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,eAAe,CACpB,MAAS,EACT,GAAM,EACN,KAAuB;IAEvB,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;IACxB,CAAC;AACL,CAAC"}

package/dist/rag/ingestion/metadata.d.ts

@@ -21,11 +21,16 @@ export declare class DefaultMetadataResolver implements MetadataResolver {
 }
 export interface DocumentIdInput {
     source: string;
-    checksum: string;
     tenantId?: string;
     knowledgeBaseId?: string;
     version?: string;
 }
 export declare function createContentChecksum(content: string): string;
+/**
+ * documentId 必须是"同一份文档"的稳定标识：只由 [tenantId, knowledgeBaseId, source, version] 派生。
+ * 故意不掺入 checksum——否则内容一变 ID 就变，replaceExisting 找不到旧文档，
+ * 旧 chunks/vectors/keyword entries 会变成永远检索得到的僵尸数据。
+ * 内容指纹保留在 document.metadata.checksum 里，供 pipeline 判断内容是否变化。
+ */
 export declare function createDocumentId(input: DocumentIdInput): string;
 //# sourceMappingURL=metadata.d.ts.map

package/dist/rag/ingestion/metadata.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"metadata.d.ts","sourceRoot":"","sources":["../../../src/rag/ingestion/metadata.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAEjD,MAAM,WAAW,uBAAuB;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAC;IACrC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,wBAAwB;IACrC,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,gBAAgB,CAAC;CAC9B;AAED,MAAM,WAAW,gBAAgB;IAC7B,OAAO,CAAC,KAAK,EAAE,uBAAuB,GAAG,wBAAwB,CAAC;CACrE;AAED,qBAAa,uBAAwB,YAAW,gBAAgB;IAC5D,OAAO,CAAC,KAAK,EAAE,uBAAuB,GAAG,wBAAwB;CAkDpE;AAED,MAAM,WAAW,eAAe;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAE7D;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,eAAe,GAAG,MAAM,CAU/D"}
+{"version":3,"file":"metadata.d.ts","sourceRoot":"","sources":["../../../src/rag/ingestion/metadata.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAEjD,MAAM,WAAW,uBAAuB;IACpC,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,QAAQ,CAAC,EAAE,OAAO,CAAC,gBAAgB,CAAC,CAAC;IACrC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,wBAAwB;IACrC,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,gBAAgB,CAAC;CAC9B;AAED,MAAM,WAAW,gBAAgB;IAC7B,OAAO,CAAC,KAAK,EAAE,uBAAuB,GAAG,wBAAwB,CAAC;CACrE;AAED,qBAAa,uBAAwB,YAAW,gBAAgB;IAC5D,OAAO,CAAC,KAAK,EAAE,uBAAuB,GAAG,wBAAwB;CAiDpE;AAED,MAAM,WAAW,eAAe;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,wBAAgB,qBAAqB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAE7D;AAED;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,eAAe,GAAG,MAAM,CAS/D"}

package/dist/rag/ingestion/metadata.js

@@ -32,7 +32,6 @@ class DefaultMetadataResolver {
         }
         const idInput = {
             source,
-            checksum,
         };
         assignIfDefined(idInput, 'tenantId', tenantId);
         assignIfDefined(idInput, 'knowledgeBaseId', knowledgeBaseId);
@@ -47,13 +46,18 @@ exports.DefaultMetadataResolver = DefaultMetadataResolver;
 function createContentChecksum(content) {
     return (0, node_crypto_1.createHash)('sha256').update(content).digest('hex');
 }
+/**
+ * documentId 必须是"同一份文档"的稳定标识：只由 [tenantId, knowledgeBaseId, source, version] 派生。
+ * 故意不掺入 checksum——否则内容一变 ID 就变，replaceExisting 找不到旧文档，
+ * 旧 chunks/vectors/keyword entries 会变成永远检索得到的僵尸数据。
+ * 内容指纹保留在 document.metadata.checksum 里，供 pipeline 判断内容是否变化。
+ */
 function createDocumentId(input) {
     const stableKey = [
         input.tenantId ?? '',
         input.knowledgeBaseId ?? '',
         input.source,
         input.version ?? '',
-        input.checksum,
     ].join('\0');
     return `doc_${createContentChecksum(stableKey).slice(0, 16)}`;
 }

package/dist/rag/ingestion/metadata.js.map

@@ -1 +1 @@
-{"version":3,"file":"metadata.js","sourceRoot":"","sources":["../../../src/rag/ingestion/metadata.ts"],"names":[],"mappings":";;;AAoFA,sDAEC;AAED,4CAUC;AAlGD,6CAAyC;AAuBzC,MAAa,uBAAuB;IAChC,OAAO,CAAC,KAA8B;QAClC,MAAM,MAAM,GAAG,kBAAkB,CAC7B,KAAK,CAAC,QAAQ,EAAE,MAAM,IAAI,KAAK,CAAC,MAAM,IAAI,QAAQ,CACrD,IAAI,QAAQ,CAAC;QACd,MAAM,QAAQ,GAAG,kBAAkB,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC;eACtD,qBAAqB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC5C,MAAM,QAAQ,GAAG,kBAAkB,CAAC,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAChF,MAAM,eAAe,GAAG,kBAAkB,CACtC,KAAK,CAAC,eAAe,IAAI,KAAK,CAAC,QAAQ,EAAE,eAAe,CAC3D,CAAC;QACF,MAAM,OAAO,GAAG,kBAAkB,CAAC,KAAK,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC5D,MAAM,QAAQ,GAAqB;YAC/B,MAAM;YACN,QAAQ;SACX,CAAC;QAEF,eAAe,CAAC,QAAQ,EAAE,OAAO,EAAE,kBAAkB,CACjD,KAAK,CAAC,QAAQ,EAAE,KAAK,IAAI,KAAK,CAAC,KAAK,IAAI,oBAAoB,CAAC,MAAM,CAAC,CACvE,CAAC,CAAC;QACH,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE,kBAAkB,CACpD,KAAK,CAAC,QAAQ,EAAE,QAAQ,IAAI,KAAK,CAAC,QAAQ,CAC7C,CAAC,CAAC;QACH,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;QAChD,eAAe,CAAC,QAAQ,EAAE,iBAAiB,EAAE,eAAe,CAAC,CAAC;QAC9D,eAAe,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAC9C,eAAe,CAAC,QAAQ,EAAE,WAAW,EAAE,kBAAkB,CAAC,KAAK,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;QACtF,eAAe,CAAC,QAAQ,EAAE,WAAW,EAAE,kBAAkB,CAAC,KAAK,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;QAEtF,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC;QAC7C,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACpB,QAAQ,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC;QAC5B,CAAC;QAED,IAAI,KAAK,CAAC,QAAQ,EAAE,KAAK,KAAK,SAAS,EAAE,CAAC;YACtC,QAAQ,CAAC,KAAK,GAAG,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACjD,CAAC;QAED,MAAM,OAAO,GAAoB;YAC7B,MAAM;YACN,QAAQ;SACX,CAAC;QACF,eAAe,CAAC,OAAO,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;QAC/C,eAAe,CAAC,OAAO,EAAE,iBAAiB,EAAE,eAAe,CAAC,CAAC;QAC7D,eAAe,CAAC,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAE7C,OAAO;YACH,EAAE,EAAE,gBAAgB,CAAC,OAAO,CAAC;YAC7B,QAAQ;SACX,CAAC;IACN,CAAC;CACJ;AAnDD,0DAmDC;AAUD,SAAgB,qBAAqB,CAAC,OAAe;IACjD,OAAO,IAAA,wBAAU,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC9D,CAAC;AAED,SAAgB,gBAAgB,CAAC,KAAsB;IACnD,MAAM,SAAS,GAAG;QACd,KAAK,CAAC,QAAQ,IAAI,EAAE;QACpB,KAAK,CAAC,eAAe,IAAI,EAAE;QAC3B,KAAK,CAAC,MAAM;QACZ,KAAK,CAAC,OAAO,IAAI,EAAE;QACnB,KAAK,CAAC,QAAQ;KACjB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEb,OAAO,OAAO,qBAAqB,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;AAClE,CAAC;AAED,SAAS,oBAAoB,CAAC,MAAc;IACxC,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;QACtB,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC;IAC1D,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC1D,MAAM,QAAQ,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9B,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClD,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,OAAO,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;AAC5C,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAyB;IACjD,MAAM,UAAU,GAAG,KAAK,EAAE,IAAI,EAAE,CAAC;IAEjC,OAAO,UAAU,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;AACxE,CAAC;AAED,SAAS,eAAe,CACpB,MAAS,EACT,GAAM,EACN,KAAuB;IAEvB,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;IACxB,CAAC;AACL,CAAC"}
+{"version":3,"file":"metadata.js","sourceRoot":"","sources":["../../../src/rag/ingestion/metadata.ts"],"names":[],"mappings":";;;AAkFA,sDAEC;AAQD,4CASC;AArGD,6CAAyC;AAuBzC,MAAa,uBAAuB;IAChC,OAAO,CAAC,KAA8B;QAClC,MAAM,MAAM,GAAG,kBAAkB,CAC7B,KAAK,CAAC,QAAQ,EAAE,MAAM,IAAI,KAAK,CAAC,MAAM,IAAI,QAAQ,CACrD,IAAI,QAAQ,CAAC;QACd,MAAM,QAAQ,GAAG,kBAAkB,CAAC,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC;eACtD,qBAAqB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC5C,MAAM,QAAQ,GAAG,kBAAkB,CAAC,KAAK,CAAC,QAAQ,IAAI,KAAK,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAChF,MAAM,eAAe,GAAG,kBAAkB,CACtC,KAAK,CAAC,eAAe,IAAI,KAAK,CAAC,QAAQ,EAAE,eAAe,CAC3D,CAAC;QACF,MAAM,OAAO,GAAG,kBAAkB,CAAC,KAAK,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QAC5D,MAAM,QAAQ,GAAqB;YAC/B,MAAM;YACN,QAAQ;SACX,CAAC;QAEF,eAAe,CAAC,QAAQ,EAAE,OAAO,EAAE,kBAAkB,CACjD,KAAK,CAAC,QAAQ,EAAE,KAAK,IAAI,KAAK,CAAC,KAAK,IAAI,oBAAoB,CAAC,MAAM,CAAC,CACvE,CAAC,CAAC;QACH,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE,kBAAkB,CACpD,KAAK,CAAC,QAAQ,EAAE,QAAQ,IAAI,KAAK,CAAC,QAAQ,CAC7C,CAAC,CAAC;QACH,eAAe,CAAC,QAAQ,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;QAChD,eAAe,CAAC,QAAQ,EAAE,iBAAiB,EAAE,eAAe,CAAC,CAAC;QAC9D,eAAe,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAC9C,eAAe,CAAC,QAAQ,EAAE,WAAW,EAAE,kBAAkB,CAAC,KAAK,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;QACtF,eAAe,CAAC,QAAQ,EAAE,WAAW,EAAE,kBAAkB,CAAC,KAAK,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,CAAC;QAEtF,MAAM,GAAG,GAAG,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC,QAAQ,EAAE,GAAG,CAAC;QAC7C,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACpB,QAAQ,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC,CAAC;QAC5B,CAAC;QAED,IAAI,KAAK,CAAC,QAAQ,EAAE,KAAK,KAAK,SAAS,EAAE,CAAC;YACtC,QAAQ,CAAC,KAAK,GAAG,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC;QACjD,CAAC;QAED,MAAM,OAAO,GAAoB;YAC7B,MAAM;SACT,CAAC;QACF,eAAe,CAAC,OAAO,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC;QAC/C,eAAe,CAAC,OAAO,EAAE,iBAAiB,EAAE,eAAe,CAAC,CAAC;QAC7D,eAAe,CAAC,OAAO,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAE7C,OAAO;YACH,EAAE,EAAE,gBAAgB,CAAC,OAAO,CAAC;YAC7B,QAAQ;SACX,CAAC;IACN,CAAC;CACJ;AAlDD,0DAkDC;AASD,SAAgB,qBAAqB,CAAC,OAAe;IACjD,OAAO,IAAA,wBAAU,EAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC9D,CAAC;AAED;;;;;GAKG;AACH,SAAgB,gBAAgB,CAAC,KAAsB;IACnD,MAAM,SAAS,GAAG;QACd,KAAK,CAAC,QAAQ,IAAI,EAAE;QACpB,KAAK,CAAC,eAAe,IAAI,EAAE;QAC3B,KAAK,CAAC,MAAM;QACZ,KAAK,CAAC,OAAO,IAAI,EAAE;KACtB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEb,OAAO,OAAO,qBAAqB,CAAC,SAAS,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;AAClE,CAAC;AAED,SAAS,oBAAoB,CAAC,MAAc;IACxC,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;QACtB,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,MAAM,YAAY,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,MAAM,CAAC;IAC1D,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC1D,MAAM,QAAQ,GAAG,KAAK,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9B,IAAI,QAAQ,KAAK,SAAS,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAClD,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,OAAO,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;AAC5C,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAyB;IACjD,MAAM,UAAU,GAAG,KAAK,EAAE,IAAI,EAAE,CAAC;IAEjC,OAAO,UAAU,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;AACxE,CAAC;AAED,SAAS,eAAe,CACpB,MAAS,EACT,GAAM,EACN,KAAuB;IAEvB,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;IACxB,CAAC;AACL,CAAC"}

package/dist/rag/pipeline.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/rag/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,aAAa,EACb,WAAW,EACX,YAAY,EACZ,SAAS,EACT,cAAc,EACd,eAAe,EAClB,MAAM,SAAS,CAAC;AAEjB,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,KAAK,EACR,cAAc,EACd,YAAY,EACZ,WAAW,EACd,MAAM,cAAc,CAAC;AACtB,OAAO,KAAK,EACR,OAAO,EACV,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EACR,cAAc,EACd,kBAAkB,EACrB,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EACR,aAAa,EACb,YAAY,EACZ,WAAW,EACd,MAAM,UAAU,CAAC;AAClB,OAAO,KAAK,EACR,SAAS,EACZ,MAAM,aAAa,CAAC;AAErB;;;GAGG;AACH,MAAM,WAAW,WAAW;IACxB;;OAEG;IACH,MAAM,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IAElD;;;OAGG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC;IAE9E;;OAEG;IACH,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;CACtE;AAED,MAAM,WAAW,yBAAyB;IACtC,MAAM,EAAE,cAAc,CAAC;IACvB,UAAU,EAAE,kBAAkB,CAAC;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,aAAa,EAAE,aAAa,CAAC;IAC7B,WAAW,EAAE,WAAW,CAAC;IACzB,YAAY,EAAE,YAAY,CAAC;IAC3B,SAAS,EAAE,SAAS,CAAC;IACrB,cAAc,EAAE,cAAc,CAAC;IAC/B,SAAS,EAAE,YAAY,CAAC;IACxB,QAAQ,EAAE,WAAW,CAAC;CACzB;AAED;;;;GAIG;AACH,qBAAa,kBAAmB,YAAW,WAAW;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAqB;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;IAClC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAgB;IAC9C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAe;IAC5C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAY;IACtC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAiB;IAChD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAe;IACzC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAc;gBAE3B,OAAO,EAAE,yBAAyB;IAoBxC,MAAM,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC;IA8BjD,QAAQ,CACV,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,eAAe,GAC1B,OAAO,CAAC,cAAc,EAAE,CAAC;IAItB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,SAAS,CAAC;CAO/E"}
+{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/rag/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,aAAa,EACb,WAAW,EACX,YAAY,EACZ,SAAS,EACT,cAAc,EACd,eAAe,EAClB,MAAM,SAAS,CAAC;AAEjB,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AACtD,OAAO,KAAK,EACR,cAAc,EACd,YAAY,EACZ,WAAW,EACd,MAAM,cAAc,CAAC;AACtB,OAAO,KAAK,EACR,OAAO,EACV,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EACR,cAAc,EACd,kBAAkB,EACrB,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EACR,aAAa,EACb,YAAY,EACZ,WAAW,EACd,MAAM,UAAU,CAAC;AAClB,OAAO,KAAK,EACR,SAAS,EACZ,MAAM,aAAa,CAAC;AAErB;;;GAGG;AACH,MAAM,WAAW,WAAW;IACxB;;OAEG;IACH,MAAM,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IAElD;;;OAGG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC;IAE9E;;OAEG;IACH,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;CACtE;AAED,MAAM,WAAW,yBAAyB;IACtC,MAAM,EAAE,cAAc,CAAC;IACvB,UAAU,EAAE,kBAAkB,CAAC;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,EAAE,QAAQ,CAAC;IACnB,aAAa,EAAE,aAAa,CAAC;IAC7B,WAAW,EAAE,WAAW,CAAC;IACzB,YAAY,EAAE,YAAY,CAAC;IAC3B,SAAS,EAAE,SAAS,CAAC;IACrB,cAAc,EAAE,cAAc,CAAC;IAC/B,SAAS,EAAE,YAAY,CAAC;IACxB,QAAQ,EAAE,WAAW,CAAC;CACzB;AAED;;;;GAIG;AACH,qBAAa,kBAAmB,YAAW,WAAW;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAiB;IACxC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAqB;IAChD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAU;IAClC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAgB;IAC9C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAc;IAC1C,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAe;IAC5C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAY;IACtC,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAiB;IAChD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAe;IACzC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAc;gBAG3B,OAAO,EAAE,yBAAyB;IAoBxC,MAAM,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,YAAY,CAAC;IAoDjD,QAAQ,CACV,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,eAAe,GAC1B,OAAO,CAAC,cAAc,EAAE,CAAC;IAItB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,SAAS,CAAC;CAW/E"}

package/dist/rag/pipeline.js

@@ -19,6 +19,7 @@ class DefaultRagPipeline {
     contextBuilder;
     generator;
     verifier;
+    // TODO options没传应该要有默认的值
     constructor(options) {
         this.loader = options.loader;
         this.normalizer = options.normalizer;
@@ -41,24 +42,42 @@ class DefaultRagPipeline {
         // 统一换行、裁剪多余空白、移除无效控制字符、清理空 block，并规范 metadata/id/acl 等字段。
         // 这一步不负责解析文件格式，也不重新生成 embedding，只处理文档进入索引前的基础数据卫生。
         const normalizedDocuments = await this.normalizer.normalizeMany(documents);
-        const chunks = await this.chunker.chunkMany(normalizedDocuments);
-        const embeddedChunks = await (0, embedder_1.embedChunks)(this.embedder, chunks);
+        const skippedDocumentIds = [];
+        let documentsToIngest = normalizedDocuments;
         if (input.replaceExisting === true) {
-            await Promise.all(normalizedDocuments.map(async (document) => {
-                await this.documentStore.deleteDocument(document.id);
-                await this.vectorStore.deleteByDocument(document.id);
-                await this.keywordStore.deleteByDocument(document.id);
-            }));
+            documentsToIngest = [];
+            for (const document of normalizedDocuments) {
+                const existing = await this.documentStore.getDocument(document.id);
+                if (existing !== undefined) {
+                    // documentId 稳定（不含 checksum），checksum 相同说明内容没变：
+                    // 直接跳过，不重复 embedding，也不重写任何索引。
+                    if (existing.metadata.checksum !== undefined
+                        && existing.metadata.checksum === document.metadata.checksum) {
+                        skippedDocumentIds.push(document.id);
+                        continue;
+                    }
+                    // 内容变了：chunk id 由内容哈希派生，upsert 不会覆盖旧 chunk，
+                    // 且 in-memory 三个 store 之间没有级联删除，必须逐 store 清理旧数据，
+                    // 否则旧 chunks/vectors/keyword entries 会成为僵尸数据继续被检索到。
+                    await this.documentStore.deleteDocument(document.id);
+                    await this.vectorStore.deleteByDocument(document.id);
+                    await this.keywordStore.deleteByDocument(document.id);
+                }
+                documentsToIngest.push(document);
+            }
         }
-        await this.documentStore.upsertDocuments(normalizedDocuments);
+        const chunks = await this.chunker.chunkMany(documentsToIngest);
+        const embeddedChunks = await (0, embedder_1.embedChunks)(this.embedder, chunks);
+        await this.documentStore.upsertDocuments(documentsToIngest);
         await this.documentStore.upsertChunks(chunks);
         await this.vectorStore.upsert(embeddedChunks);
         await this.keywordStore.upsert(chunks);
         return {
-            documentCount: normalizedDocuments.length,
+            documentCount: documentsToIngest.length,
             chunkCount: chunks.length,
-            documentIds: normalizedDocuments.map((document) => document.id),
+            documentIds: documentsToIngest.map((document) => document.id),
             chunkIds: chunks.map((chunk) => chunk.id),
+            skippedDocumentIds,
         };
     }
     async retrieve(query, options) {
@@ -66,7 +85,11 @@ class DefaultRagPipeline {
     }
     async answer(query, options = {}) {
         const retrieved = await this.retrieve(query, options);
-        const context = await this.contextBuilder.build(query, retrieved, options);
+        // minScore 只在各 retriever 的原始分数尺度（cosine / TF-IDF）上有意义，
+        // 融合后的 RRF 分数量级完全不同（~0.016），透传会把结果误伤清空。
+        // ContextBuilder 的过滤阈值由独立的 minFusedScore 控制，这里不透传 minScore。
+        const { minScore: _minScore, ...contextOptions } = options;
+        const context = await this.contextBuilder.build(query, retrieved, contextOptions);
         const answer = await this.generator.generate(query, context, options);
         return await this.verifier.verify(answer, context, options);
     }

package/dist/rag/pipeline.js.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/rag/pipeline.ts"],"names":[],"mappings":";;;AAQA,oDAAoD;AA2DpD;;;;GAIG;AACH,MAAa,kBAAkB;IACV,MAAM,CAAiB;IACvB,UAAU,CAAqB;IAC/B,OAAO,CAAU;IACjB,QAAQ,CAAW;IACnB,aAAa,CAAgB;IAC7B,WAAW,CAAc;IACzB,YAAY,CAAe;IAC3B,SAAS,CAAY;IACrB,cAAc,CAAiB;IAC/B,SAAS,CAAe;IACxB,QAAQ,CAAc;IAEvC,YAAY,OAAkC;QAC1C,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QACrC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAC/B,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QACjC,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;QAC3C,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QACvC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC;QAC7C,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QAEjC,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,KAAK,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;YAC3D,MAAM,IAAI,KAAK,CACX,wDAAwD,IAAI,CAAC,QAAQ,CAAC,UAAU,iBAAiB,IAAI,CAAC,WAAW,CAAC,UAAU,GAAG,CAClI,CAAC;QACN,CAAC;IACL,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAkB;QAC3B,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAChD,+CAA+C;QAC/C,0DAA0D;QAC1D,mDAAmD;QACnD,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAC3E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,mBAAmB,CAAC,CAAC;QACjE,MAAM,cAAc,GAAG,MAAM,IAAA,sBAAW,EAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAEhE,IAAI,KAAK,CAAC,eAAe,KAAK,IAAI,EAAE,CAAC;YACjC,MAAM,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;gBACzD,MAAM,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACrD,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACrD,MAAM,IAAI,CAAC,YAAY,CAAC,gBAAgB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YAC1D,CAAC,CAAC,CAAC,CAAC;QACR,CAAC;QAED,MAAM,IAAI,CAAC,aAAa,CAAC,eAAe,CAAC,mBAAmB,CAAC,CAAC;QAC9D,MAAM,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAC9C,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;QAC9C,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAEvC,OAAO;YACH,aAAa,EAAE,mBAAmB,CAAC,MAAM;YACzC,UAAU,EAAE,MAAM,CAAC,MAAM;YACzB,WAAW,EAAE,mBAAmB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC/D,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;SAC5C,CAAC;IACN,CAAC;IAED,KAAK,CAAC,QAAQ,CACV,KAAa,EACb,OAAyB;QAEzB,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,UAAyB,EAAE;QACnD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QACtD,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,CAAC,CAAC;QAC3E,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAEtE,OAAO,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAChE,CAAC;CACJ;AA7ED,gDA6EC"}
+{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/rag/pipeline.ts"],"names":[],"mappings":";;;AAQA,oDAAoD;AA2DpD;;;;GAIG;AACH,MAAa,kBAAkB;IACV,MAAM,CAAiB;IACvB,UAAU,CAAqB;IAC/B,OAAO,CAAU;IACjB,QAAQ,CAAW;IACnB,aAAa,CAAgB;IAC7B,WAAW,CAAc;IACzB,YAAY,CAAe;IAC3B,SAAS,CAAY;IACrB,cAAc,CAAiB;IAC/B,SAAS,CAAe;IACxB,QAAQ,CAAc;IAEvC,yBAAyB;IACzB,YAAY,OAAkC;QAC1C,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QACrC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC;QAC/B,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QACjC,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;QAC3C,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,CAAC;QACvC,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,CAAC;QACzC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,CAAC;QAC7C,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,CAAC;QAEjC,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,KAAK,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;YAC3D,MAAM,IAAI,KAAK,CACX,wDAAwD,IAAI,CAAC,QAAQ,CAAC,UAAU,iBAAiB,IAAI,CAAC,WAAW,CAAC,UAAU,GAAG,CAClI,CAAC;QACN,CAAC;IACL,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAkB;QAC3B,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAChD,+CAA+C;QAC/C,0DAA0D;QAC1D,mDAAmD;QACnD,MAAM,mBAAmB,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,SAAS,CAAC,CAAC;QAE3E,MAAM,kBAAkB,GAAa,EAAE,CAAC;QACxC,IAAI,iBAAiB,GAAG,mBAAmB,CAAC;QAC5C,IAAI,KAAK,CAAC,eAAe,KAAK,IAAI,EAAE,CAAC;YACjC,iBAAiB,GAAG,EAAE,CAAC;YACvB,KAAK,MAAM,QAAQ,IAAI,mBAAmB,EAAE,CAAC;gBACzC,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,aAAa,CAAC,WAAW,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACnE,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;oBACzB,gDAAgD;oBAChD,+BAA+B;oBAC/B,IACI,QAAQ,CAAC,QAAQ,CAAC,QAAQ,KAAK,SAAS;2BACrC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,KAAK,QAAQ,CAAC,QAAQ,CAAC,QAAQ,EAC9D,CAAC;wBACC,kBAAkB,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;wBACrC,SAAS;oBACb,CAAC;oBAED,4CAA4C;oBAC5C,iDAAiD;oBACjD,oDAAoD;oBACpD,MAAM,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;oBACrD,MAAM,IAAI,CAAC,WAAW,CAAC,gBAAgB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;oBACrD,MAAM,IAAI,CAAC,YAAY,CAAC,gBAAgB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBAC1D,CAAC;gBACD,iBAAiB,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACrC,CAAC;QACL,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,iBAAiB,CAAC,CAAC;QAC/D,MAAM,cAAc,GAAG,MAAM,IAAA,sBAAW,EAAC,IAAI,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAEhE,MAAM,IAAI,CAAC,aAAa,CAAC,eAAe,CAAC,iBAAiB,CAAC,CAAC;QAC5D,MAAM,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC;QAC9C,MAAM,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;QAC9C,MAAM,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAEvC,OAAO;YACH,aAAa,EAAE,iBAAiB,CAAC,MAAM;YACvC,UAAU,EAAE,MAAM,CAAC,MAAM;YACzB,WAAW,EAAE,iBAAiB,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC7D,QAAQ,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC;YACzC,kBAAkB;SACrB,CAAC;IACN,CAAC;IAED,KAAK,CAAC,QAAQ,CACV,KAAa,EACb,OAAyB;QAEzB,OAAO,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IACzD,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,UAAyB,EAAE;QACnD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QACtD,uDAAuD;QACvD,wCAAwC;QACxC,4DAA4D;QAC5D,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,GAAG,cAAc,EAAE,GAAG,OAAO,CAAC;QAC3D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,KAAK,EAAE,SAAS,EAAE,cAAc,CAAC,CAAC;QAClF,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;QAEtE,OAAO,MAAM,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAChE,CAAC;CACJ;AAxGD,gDAwGC"}

package/dist/rag/retrieval/index.d.ts

@@ -3,6 +3,7 @@ export * from './dense-retriever';
 export * from './fusion';
 export * from './hybrid-retriever';
 export * from './http-rerank-model-client';
+export * from './parent-child-expanding-retriever';
 export * from './reranker';
 export * from './sparse-retriever';
 //# sourceMappingURL=index.d.ts.map

package/dist/rag/retrieval/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/rag/retrieval/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC;AACxB,cAAc,mBAAmB,CAAC;AAClC,cAAc,UAAU,CAAC;AACzB,cAAc,oBAAoB,CAAC;AACnC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,YAAY,CAAC;AAC3B,cAAc,oBAAoB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/rag/retrieval/index.ts"],"names":[],"mappings":"AAAA,cAAc,SAAS,CAAC;AACxB,cAAc,mBAAmB,CAAC;AAClC,cAAc,UAAU,CAAC;AACzB,cAAc,oBAAoB,CAAC;AACnC,cAAc,4BAA4B,CAAC;AAC3C,cAAc,oCAAoC,CAAC;AACnD,cAAc,YAAY,CAAC;AAC3B,cAAc,oBAAoB,CAAC"}

package/dist/rag/retrieval/index.js

@@ -19,6 +19,7 @@ __exportStar(require("./dense-retriever"), exports);
 __exportStar(require("./fusion"), exports);
 __exportStar(require("./hybrid-retriever"), exports);
 __exportStar(require("./http-rerank-model-client"), exports);
+__exportStar(require("./parent-child-expanding-retriever"), exports);
 __exportStar(require("./reranker"), exports);
 __exportStar(require("./sparse-retriever"), exports);
 //# sourceMappingURL=index.js.map

package/dist/rag/retrieval/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/rag/retrieval/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,0CAAwB;AACxB,oDAAkC;AAClC,2CAAyB;AACzB,qDAAmC;AACnC,6DAA2C;AAC3C,6CAA2B;AAC3B,qDAAmC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/rag/retrieval/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,0CAAwB;AACxB,oDAAkC;AAClC,2CAAyB;AACzB,qDAAmC;AACnC,6DAA2C;AAC3C,qEAAmD;AACnD,6CAA2B;AAC3B,qDAAmC"}

package/dist/rag/retrieval/parent-child-expanding-retriever.d.ts

@@ -0,0 +1,31 @@
+import type { RetrievedChunk, RetrieveOptions } from '../types';
+import type { DocumentStore } from '../stores';
+import type { Retriever } from './types';
+export type ParentChildExpansionStrategy = 'parent' | 'neighbors' | 'parent_or_neighbors';
+export interface ParentChildExpandingRetrieverOptions {
+    retriever: Retriever;
+    documentStore: DocumentStore;
+    strategy?: ParentChildExpansionStrategy;
+    neighborWindow?: number;
+    includeOriginalChunks?: boolean;
+    maxExpandedChunkCharacters?: number;
+}
+/**
+ * Expands child chunk hits into larger context chunks after retrieval.
+ * This keeps child chunks useful for recall while letting generation receive
+ * parent or neighboring context without changing the underlying retriever.
+ */
+export declare class ParentChildExpandingRetriever implements Retriever {
+    private readonly retriever;
+    private readonly documentStore;
+    private readonly strategy;
+    private readonly neighborWindow;
+    private readonly includeOriginalChunks;
+    private readonly maxExpandedChunkCharacters;
+    constructor(options: ParentChildExpandingRetrieverOptions);
+    retrieve(query: string, options?: RetrieveOptions): Promise<RetrievedChunk[]>;
+    private expandItem;
+    private createParentExpansion;
+    private createNeighborExpansions;
+}
+//# sourceMappingURL=parent-child-expanding-retriever.d.ts.map

package/dist/rag/retrieval/parent-child-expanding-retriever.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parent-child-expanding-retriever.d.ts","sourceRoot":"","sources":["../../../src/rag/retrieval/parent-child-expanding-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAER,cAAc,EACd,eAAe,EAClB,MAAM,UAAU,CAAC;AAClB,OAAO,KAAK,EACR,aAAa,EAEhB,MAAM,WAAW,CAAC;AAKnB,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAMzC,MAAM,MAAM,4BAA4B,GAAG,QAAQ,GAAG,WAAW,GAAG,qBAAqB,CAAC;AAE1F,MAAM,WAAW,oCAAoC;IACjD,SAAS,EAAE,SAAS,CAAC;IACrB,aAAa,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,EAAE,4BAA4B,CAAC;IACxC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,0BAA0B,CAAC,EAAE,MAAM,CAAC;CACvC;AAMD;;;;GAIG;AACH,qBAAa,6BAA8B,YAAW,SAAS;IAC3D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAY;IACtC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAgB;IAC9C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA+B;IACxD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAU;IAChD,OAAO,CAAC,QAAQ,CAAC,0BAA0B,CAAqB;gBAEpD,OAAO,EAAE,oCAAoC;IAWnD,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;YAsCrE,UAAU;IAyBxB,OAAO,CAAC,qBAAqB;IA2B7B,OAAO,CAAC,wBAAwB;CAwBnC"}

package/dist/rag/retrieval/parent-child-expanding-retriever.js

@@ -0,0 +1,194 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ParentChildExpandingRetriever = void 0;
+const utils_1 = require("../stores/utils");
+const utils_2 = require("./utils");
+/**
+ * Expands child chunk hits into larger context chunks after retrieval.
+ * This keeps child chunks useful for recall while letting generation receive
+ * parent or neighboring context without changing the underlying retriever.
+ */
+class ParentChildExpandingRetriever {
+    retriever;
+    documentStore;
+    strategy;
+    neighborWindow;
+    includeOriginalChunks;
+    maxExpandedChunkCharacters;
+    constructor(options) {
+        this.retriever = options.retriever;
+        this.documentStore = options.documentStore;
+        this.strategy = options.strategy ?? 'parent';
+        this.neighborWindow = normalizeNonNegativeInteger(options.neighborWindow, 0);
+        this.includeOriginalChunks = options.includeOriginalChunks ?? false;
+        this.maxExpandedChunkCharacters = normalizeOptionalPositiveInteger(options.maxExpandedChunkCharacters);
+    }
+    async retrieve(query, options) {
+        const recalled = await this.retriever.retrieve(query, options);
+        if (recalled.length === 0) {
+            return [];
+        }
+        const cache = createDocumentChunkCache(this.documentStore);
+        const searchOptions = (0, utils_2.toSearchOptions)(options);
+        const expanded = [];
+        const expandedByChunkId = new Map();
+        for (const item of recalled) {
+            if (this.includeOriginalChunks) {
+                appendOrMergeRetrievedChunk(expanded, expandedByChunkId, (0, utils_2.cloneRetrievedChunk)(item));
+            }
+            const expandedItems = await this.expandItem(item, cache, searchOptions);
+            if (expandedItems.length === 0) {
+                appendOrMergeRetrievedChunk(expanded, expandedByChunkId, (0, utils_2.cloneRetrievedChunk)(item));
+                continue;
+            }
+            for (const expandedItem of expandedItems) {
+                appendOrMergeRetrievedChunk(expanded, expandedByChunkId, expandedItem);
+            }
+        }
+        return expanded;
+    }
+    async expandItem(item, cache, searchOptions) {
+        const chunks = await cache.get(item.chunk.documentId);
+        if (this.strategy === 'parent') {
+            const parent = this.createParentExpansion(item, chunks, searchOptions);
+            return parent === undefined ? [] : [parent];
+        }
+        if (this.strategy === 'neighbors') {
+            return this.createNeighborExpansions(item, chunks, searchOptions);
+        }
+        const parent = this.createParentExpansion(item, chunks, searchOptions);
+        if (parent !== undefined) {
+            return [parent];
+        }
+        return this.createNeighborExpansions(item, chunks, searchOptions);
+    }
+    createParentExpansion(item, chunks, searchOptions) {
+        const parentId = item.chunk.parentId;
+        if (parentId === undefined) {
+            return undefined;
+        }
+        const parent = chunks.find((chunk) => chunk.id === parentId);
+        if (parent === undefined) {
+            return undefined;
+        }
+        if (!(0, utils_1.matchesSearchOptions)(parent, searchOptions)) {
+            return undefined;
+        }
+        if (this.maxExpandedChunkCharacters !== undefined
+            && parent.content.length > this.maxExpandedChunkCharacters) {
+            return undefined;
+        }
+        return createExpandedRetrievedChunk(parent, item, 'parent');
+    }
+    createNeighborExpansions(item, chunks, searchOptions) {
+        const candidates = getNeighborCandidates(item.chunk, chunks)
+            .filter((chunk) => (0, utils_1.matchesSearchOptions)(chunk, searchOptions))
+            .sort((left, right) => left.metadata.chunkIndex - right.metadata.chunkIndex);
+        const centerIndex = candidates.findIndex((chunk) => chunk.id === item.chunk.id);
+        if (centerIndex < 0) {
+            return [];
+        }
+        const start = Math.max(0, centerIndex - this.neighborWindow);
+        const end = Math.min(candidates.length, centerIndex + this.neighborWindow + 1);
+        return candidates
+            .slice(start, end)
+            .map((chunk) => (chunk.id === item.chunk.id
+            ? createExpandedRetrievedChunk(chunk, item, 'self')
+            : createExpandedRetrievedChunk(chunk, item, 'neighbor')));
+    }
+}
+exports.ParentChildExpandingRetriever = ParentChildExpandingRetriever;
+function createDocumentChunkCache(documentStore) {
+    const chunksByDocumentId = new Map();
+    return {
+        async get(documentId) {
+            let chunks = chunksByDocumentId.get(documentId);
+            if (chunks === undefined) {
+                chunks = documentStore.getChunksByDocument(documentId);
+                chunksByDocumentId.set(documentId, chunks);
+            }
+            return await chunks;
+        },
+    };
+}
+function getNeighborCandidates(hit, chunks) {
+    if (hit.parentId !== undefined) {
+        return chunks.filter((chunk) => chunk.id === hit.id || chunk.parentId === hit.parentId);
+    }
+    return chunks.filter((chunk) => chunk.parentId === undefined);
+}
+function createExpandedRetrievedChunk(chunk, original, expansion) {
+    const item = {
+        chunk: (0, utils_1.cloneChunk)(chunk),
+        score: original.score,
+        source: original.source,
+        metadata: createExpansionMetadata(original, expansion),
+    };
+    return item;
+}
+function createExpansionMetadata(original, expansion) {
+    const metadata = {
+        ...(original.metadata ?? {}),
+        expansion,
+        expandedFromChunkIds: [original.chunk.id],
+    };
+    assignIfDefined(metadata, 'expandedFromParentId', original.chunk.parentId);
+    return metadata;
+}
+function appendOrMergeRetrievedChunk(chunks, chunksById, item) {
+    const existing = chunksById.get(item.chunk.id);
+    if (existing === undefined) {
+        chunks.push(item);
+        chunksById.set(item.chunk.id, item);
+        return;
+    }
+    existing.score = Math.max(existing.score, item.score);
+    const metadata = mergeRetrievedMetadata(existing.metadata, item.metadata);
+    if (metadata === undefined) {
+        delete existing.metadata;
+    }
+    else {
+        existing.metadata = metadata;
+    }
+}
+function mergeRetrievedMetadata(left, right) {
+    if (left === undefined && right === undefined) {
+        return undefined;
+    }
+    const metadata = {
+        ...(left ?? {}),
+        ...(right ?? {}),
+    };
+    const expandedFromChunkIds = [
+        ...readStringArray(left?.expandedFromChunkIds),
+        ...readStringArray(right?.expandedFromChunkIds),
+    ];
+    if (expandedFromChunkIds.length > 0) {
+        metadata.expandedFromChunkIds = [...new Set(expandedFromChunkIds)];
+    }
+    return metadata;
+}
+function readStringArray(value) {
+    if (!Array.isArray(value)) {
+        return [];
+    }
+    return value.filter((item) => typeof item === 'string');
+}
+function normalizeNonNegativeInteger(value, fallback) {
+    if (value === undefined || !Number.isFinite(value) || value < 0) {
+        return fallback;
+    }
+    return Math.floor(value);
+}
+function normalizeOptionalPositiveInteger(value) {
+    if (value === undefined || !Number.isFinite(value) || value <= 0) {
+        return undefined;
+    }
+    return Math.floor(value);
+}
+function assignIfDefined(target, key, value) {
+    if (value !== undefined) {
+        target[key] = value;
+    }
+}
+//# sourceMappingURL=parent-child-expanding-retriever.js.map

package/dist/rag/retrieval/parent-child-expanding-retriever.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parent-child-expanding-retriever.js","sourceRoot":"","sources":["../../../src/rag/retrieval/parent-child-expanding-retriever.ts"],"names":[],"mappings":";;;AASA,2CAGyB;AAEzB,mCAGiB;AAiBjB;;;;GAIG;AACH,MAAa,6BAA6B;IACrB,SAAS,CAAY;IACrB,aAAa,CAAgB;IAC7B,QAAQ,CAA+B;IACvC,cAAc,CAAS;IACvB,qBAAqB,CAAU;IAC/B,0BAA0B,CAAqB;IAEhE,YAAY,OAA6C;QACrD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;QAC3C,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,QAAQ,CAAC;QAC7C,IAAI,CAAC,cAAc,GAAG,2BAA2B,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;QAC7E,IAAI,CAAC,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,IAAI,KAAK,CAAC;QACpE,IAAI,CAAC,0BAA0B,GAAG,gCAAgC,CAC9D,OAAO,CAAC,0BAA0B,CACrC,CAAC;IACN,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAyB;QACnD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC/D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,EAAE,CAAC;QACd,CAAC;QAED,MAAM,KAAK,GAAG,wBAAwB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC3D,MAAM,aAAa,GAAG,IAAA,uBAAe,EAAC,OAAO,CAAC,CAAC;QAC/C,MAAM,QAAQ,GAAqB,EAAE,CAAC;QACtC,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAA0B,CAAC;QAE5D,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC1B,IAAI,IAAI,CAAC,qBAAqB,EAAE,CAAC;gBAC7B,2BAA2B,CACvB,QAAQ,EACR,iBAAiB,EACjB,IAAA,2BAAmB,EAAC,IAAI,CAAC,CAC5B,CAAC;YACN,CAAC;YAED,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,KAAK,EAAE,aAAa,CAAC,CAAC;YACxE,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC7B,2BAA2B,CACvB,QAAQ,EACR,iBAAiB,EACjB,IAAA,2BAAmB,EAAC,IAAI,CAAC,CAC5B,CAAC;gBACF,SAAS;YACb,CAAC;YAED,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;gBACvC,2BAA2B,CAAC,QAAQ,EAAE,iBAAiB,EAAE,YAAY,CAAC,CAAC;YAC3E,CAAC;QACL,CAAC;QAED,OAAO,QAAQ,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,UAAU,CACpB,IAAoB,EACpB,KAAyB,EACzB,aAA4B;QAE5B,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAEtD,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,MAAM,EAAE,aAAa,CAAC,CAAC;YAEvE,OAAO,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAChD,CAAC;QAED,IAAI,IAAI,CAAC,QAAQ,KAAK,WAAW,EAAE,CAAC;YAChC,OAAO,IAAI,CAAC,wBAAwB,CAAC,IAAI,EAAE,MAAM,EAAE,aAAa,CAAC,CAAC;QACtE,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,MAAM,EAAE,aAAa,CAAC,CAAC;QACvE,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACvB,OAAO,CAAC,MAAM,CAAC,CAAC;QACpB,CAAC;QAED,OAAO,IAAI,CAAC,wBAAwB,CAAC,IAAI,EAAE,MAAM,EAAE,aAAa,CAAC,CAAC;IACtE,CAAC;IAEO,qBAAqB,CACzB,IAAoB,EACpB,MAAe,EACf,aAA4B;QAE5B,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;QACrC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YACzB,OAAO,SAAS,CAAC;QACrB,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC;QAC7D,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACvB,OAAO,SAAS,CAAC;QACrB,CAAC;QACD,IAAI,CAAC,IAAA,4BAAoB,EAAC,MAAM,EAAE,aAAa,CAAC,EAAE,CAAC;YAC/C,OAAO,SAAS,CAAC;QACrB,CAAC;QACD,IACI,IAAI,CAAC,0BAA0B,KAAK,SAAS;eAC1C,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,0BAA0B,EAC5D,CAAC;YACC,OAAO,SAAS,CAAC;QACrB,CAAC;QAED,OAAO,4BAA4B,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;IAChE,CAAC;IAEO,wBAAwB,CAC5B,IAAoB,EACpB,MAAe,EACf,aAA4B;QAE5B,MAAM,UAAU,GAAG,qBAAqB,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC;aACvD,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAA,4BAAoB,EAAC,KAAK,EAAE,aAAa,CAAC,CAAC;aAC7D,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,GAAG,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QACjF,MAAM,WAAW,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChF,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;YAClB,OAAO,EAAE,CAAC;QACd,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC;QAC7D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC;QAE/E,OAAO,UAAU;aACZ,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC;aACjB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CACZ,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,KAAK,CAAC,EAAE;YACtB,CAAC,CAAC,4BAA4B,CAAC,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC;YACnD,CAAC,CAAC,4BAA4B,CAAC,KAAK,EAAE,IAAI,EAAE,UAAU,CAAC,CAC9D,CAAC,CAAC;IACX,CAAC;CACJ;AArID,sEAqIC;AAED,SAAS,wBAAwB,CAAC,aAA4B;IAC1D,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAA4B,CAAC;IAE/D,OAAO;QACH,KAAK,CAAC,GAAG,CAAC,UAAkB;YACxB,IAAI,MAAM,GAAG,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAChD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gBACvB,MAAM,GAAG,aAAa,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;gBACvD,kBAAkB,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YAC/C,CAAC;YAED,OAAO,MAAM,MAAM,CAAC;QACxB,CAAC;KACJ,CAAC;AACN,CAAC;AAED,SAAS,qBAAqB,CAAC,GAAU,EAAE,MAAe;IACtD,IAAI,GAAG,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,IAAI,KAAK,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC5F,CAAC;IAED,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,4BAA4B,CACjC,KAAY,EACZ,QAAwB,EACxB,SAAyC;IAEzC,MAAM,IAAI,GAAmB;QACzB,KAAK,EAAE,IAAA,kBAAU,EAAC,KAAK,CAAC;QACxB,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,QAAQ,EAAE,uBAAuB,CAAC,QAAQ,EAAE,SAAS,CAAC;KACzD,CAAC;IAEF,OAAO,IAAI,CAAC;AAChB,CAAC;AAED,SAAS,uBAAuB,CAC5B,QAAwB,EACxB,SAAyC;IAEzC,MAAM,QAAQ,GAA4B;QACtC,GAAG,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC;QAC5B,SAAS;QACT,oBAAoB,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;KAC5C,CAAC;IAEF,eAAe,CAAC,QAAQ,EAAE,sBAAsB,EAAE,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAE3E,OAAO,QAAQ,CAAC;AACpB,CAAC;AAED,SAAS,2BAA2B,CAChC,MAAwB,EACxB,UAAuC,EACvC,IAAoB;IAEpB,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC/C,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClB,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QACpC,OAAO;IACX,CAAC;IAED,QAAQ,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,sBAAsB,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC1E,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QACzB,OAAO,QAAQ,CAAC,QAAQ,CAAC;IAC7B,CAAC;SAAM,CAAC;QACJ,QAAQ,CAAC,QAAQ,GAAG,QAAQ,CAAC;IACjC,CAAC;AACL,CAAC;AAED,SAAS,sBAAsB,CAC3B,IAAyC,EACzC,KAA0C;IAE1C,IAAI,IAAI,KAAK,SAAS,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QAC5C,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,MAAM,QAAQ,GAA4B;QACtC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;QACf,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;KACnB,CAAC;IACF,MAAM,oBAAoB,GAAG;QACzB,GAAG,eAAe,CAAC,IAAI,EAAE,oBAAoB,CAAC;QAC9C,GAAG,eAAe,CAAC,KAAK,EAAE,oBAAoB,CAAC;KAClD,CAAC;IAEF,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,QAAQ,CAAC,oBAAoB,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,oBAAoB,CAAC,CAAC,CAAC;IACvE,CAAC;IAED,OAAO,QAAQ,CAAC;AACpB,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACnC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,CAAC;IACd,CAAC;IAED,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC;AAC5E,CAAC;AAED,SAAS,2BAA2B,CAAC,KAAyB,EAAE,QAAgB;IAC5E,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QAC9D,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,gCAAgC,CAAC,KAAyB;IAC/D,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QAC/D,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,eAAe,CACpB,MAAS,EACT,GAAM,EACN,KAAuB;IAEvB,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;IACxB,CAAC;AACL,CAAC"}

package/dist/rag/stores/in-memory-keyword-store.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"in-memory-keyword-store.d.ts","sourceRoot":"","sources":["../../../src/rag/stores/in-memory-keyword-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,KAAK,EACR,YAAY,EACZ,cAAc,EACd,aAAa,EAChB,MAAM,SAAS,CAAC;AAajB,qBAAa,oBAAqB,YAAW,YAAY;IACrD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwC;IAE1D,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAWtC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAuC7E,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQzD,OAAO,CAAC,0BAA0B;CAerC"}
+{"version":3,"file":"in-memory-keyword-store.d.ts","sourceRoot":"","sources":["../../../src/rag/stores/in-memory-keyword-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,KAAK,EACR,YAAY,EACZ,cAAc,EACd,aAAa,EAChB,MAAM,SAAS,CAAC;AAiBjB,qBAAa,oBAAqB,YAAW,YAAY;IACrD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwC;IAE1D,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAWtC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAuC7E,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQzD,OAAO,CAAC,0BAA0B;CAerC"}

package/dist/rag/stores/in-memory-keyword-store.js

@@ -2,11 +2,12 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.InMemoryKeywordStore = void 0;
 const utils_1 = require("./utils");
+const keyword_tokenizer_1 = require("./keyword-tokenizer");
 class InMemoryKeywordStore {
     entries = new Map();
     async upsert(chunks) {
         for (const chunk of chunks) {
-            const terms = tokenize(chunk.content);
+            const terms = (0, keyword_tokenizer_1.tokenizeKeywordText)((0, keyword_tokenizer_1.createKeywordIndexText)(chunk));
             this.entries.set(chunk.id, {
                 chunk: (0, utils_1.cloneChunk)(chunk),
                 termCounts: countTerms(terms),
@@ -15,7 +16,7 @@ class InMemoryKeywordStore {
         }
     }
     async search(query, options = {}) {
-        const queryTerms = [...new Set(tokenize(query))];
+        const queryTerms = [...new Set((0, keyword_tokenizer_1.tokenizeKeywordText)(query))];
         if (queryTerms.length === 0) {
             return [];
         }
@@ -83,12 +84,6 @@ function scoreEntry(input) {
     }
     return score;
 }
-function tokenize(text) {
-    return text
-        .normalize('NFKC')
-        .toLowerCase()
-        .match(/[\p{L}\p{N}_-]+/gu) ?? [];
-}
 function countTerms(terms) {
     const counts = new Map();
     for (const term of terms) {

package/dist/rag/stores/in-memory-keyword-store.js.map

@@ -1 +1 @@
-{"version":3,"file":"in-memory-keyword-store.js","sourceRoot":"","sources":["../../../src/rag/stores/in-memory-keyword-store.ts"],"names":[],"mappings":";;;AAMA,mCAIiB;AAQjB,MAAa,oBAAoB;IACZ,OAAO,GAAG,IAAI,GAAG,EAA6B,CAAC;IAEhE,KAAK,CAAC,MAAM,CAAC,MAAe;QACxB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACzB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACtC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE;gBACvB,KAAK,EAAE,IAAA,kBAAU,EAAC,KAAK,CAAC;gBACxB,UAAU,EAAE,UAAU,CAAC,KAAK,CAAC;gBAC7B,MAAM,EAAE,KAAK,CAAC,MAAM;aACvB,CAAC,CAAC;QACP,CAAC;IACL,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,UAAyB,EAAE;QACnD,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACjD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,OAAO,EAAE,CAAC;QACd,CAAC;QAED,MAAM,IAAI,GAAqB,EAAE,CAAC;QAClC,MAAM,iBAAiB,GAAG,IAAI,CAAC,0BAA0B,CAAC,UAAU,CAAC,CAAC;QACtE,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAEtD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YACxC,IAAI,CAAC,IAAA,4BAAoB,EAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,CAAC;gBAC9C,SAAS;YACb,CAAC;YAED,MAAM,KAAK,GAAG,UAAU,CAAC;gBACrB,KAAK;gBACL,UAAU;gBACV,iBAAiB;gBACjB,cAAc;aACjB,CAAC,CAAC;YACH,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;gBACb,SAAS;YACb,CAAC;YAED,IAAI,CAAC,IAAI,CAAC;gBACN,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,EAAE;gBACvB,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC,UAAU;gBAClC,KAAK;gBACL,QAAQ,EAAE;oBACN,MAAM,EAAE,SAAS;oBACjB,YAAY,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;iBACxE;aACJ,CAAC,CAAC;QACP,CAAC;QAED,OAAO,IAAA,yBAAiB,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,UAAkB;QACrC,KAAK,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YACpD,IAAI,KAAK,CAAC,KAAK,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;gBACxC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACjC,CAAC;QACL,CAAC;IACL,CAAC;IAEO,0BAA0B,CAAC,UAAoB;QACnD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;QAE5C,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;gBACxC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC7B,KAAK,IAAI,CAAC,CAAC;gBACf,CAAC;YACL,CAAC;YACD,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC/B,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;CACJ;AA5ED,oDA4EC;AASD,SAAS,UAAU,CAAC,KAAsB;IACtC,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;QAClC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxD,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;YAClB,SAAS;QACb,CAAC;QAED,MAAM,EAAE,GAAG,KAAK,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,GAAG,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;QACzE,MAAM,YAAY,GAAG,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;QAC5E,KAAK,IAAI,YAAY,GAAG,GAAG,CAAC;IAChC,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC1B,OAAO,IAAI;SACN,SAAS,CAAC,MAAM,CAAC;SACjB,WAAW,EAAE;SACb,KAAK,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC;AAC1C,CAAC;AAED,SAAS,UAAU,CAAC,KAAe;IAC/B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IAEzC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC"}
+{"version":3,"file":"in-memory-keyword-store.js","sourceRoot":"","sources":["../../../src/rag/stores/in-memory-keyword-store.ts"],"names":[],"mappings":";;;AAMA,mCAIiB;AACjB,2DAG6B;AAQ7B,MAAa,oBAAoB;IACZ,OAAO,GAAG,IAAI,GAAG,EAA6B,CAAC;IAEhE,KAAK,CAAC,MAAM,CAAC,MAAe;QACxB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACzB,MAAM,KAAK,GAAG,IAAA,uCAAmB,EAAC,IAAA,0CAAsB,EAAC,KAAK,CAAC,CAAC,CAAC;YACjE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE;gBACvB,KAAK,EAAE,IAAA,kBAAU,EAAC,KAAK,CAAC;gBACxB,UAAU,EAAE,UAAU,CAAC,KAAK,CAAC;gBAC7B,MAAM,EAAE,KAAK,CAAC,MAAM;aACvB,CAAC,CAAC;QACP,CAAC;IACL,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,UAAyB,EAAE;QACnD,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,IAAA,uCAAmB,EAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC5D,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,OAAO,EAAE,CAAC;QACd,CAAC;QAED,MAAM,IAAI,GAAqB,EAAE,CAAC;QAClC,MAAM,iBAAiB,GAAG,IAAI,CAAC,0BAA0B,CAAC,UAAU,CAAC,CAAC;QACtE,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAEtD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YACxC,IAAI,CAAC,IAAA,4BAAoB,EAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,CAAC;gBAC9C,SAAS;YACb,CAAC;YAED,MAAM,KAAK,GAAG,UAAU,CAAC;gBACrB,KAAK;gBACL,UAAU;gBACV,iBAAiB;gBACjB,cAAc;aACjB,CAAC,CAAC;YACH,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;gBACb,SAAS;YACb,CAAC;YAED,IAAI,CAAC,IAAI,CAAC;gBACN,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,EAAE;gBACvB,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC,UAAU;gBAClC,KAAK;gBACL,QAAQ,EAAE;oBACN,MAAM,EAAE,SAAS;oBACjB,YAAY,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;iBACxE;aACJ,CAAC,CAAC;QACP,CAAC;QAED,OAAO,IAAA,yBAAiB,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,UAAkB;QACrC,KAAK,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YACpD,IAAI,KAAK,CAAC,KAAK,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;gBACxC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACjC,CAAC;QACL,CAAC;IACL,CAAC;IAEO,0BAA0B,CAAC,UAAoB;QACnD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;QAE5C,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;gBACxC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC7B,KAAK,IAAI,CAAC,CAAC;gBACf,CAAC;YACL,CAAC;YACD,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC/B,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;CACJ;AA5ED,oDA4EC;AASD,SAAS,UAAU,CAAC,KAAsB;IACtC,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;QAClC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxD,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;YAClB,SAAS;QACb,CAAC;QAED,MAAM,EAAE,GAAG,KAAK,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,GAAG,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;QACzE,MAAM,YAAY,GAAG,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;QAC5E,KAAK,IAAI,YAAY,GAAG,GAAG,CAAC;IAChC,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,SAAS,UAAU,CAAC,KAAe;IAC/B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IAEzC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC"}

package/dist/rag/stores/keyword-tokenizer.d.ts

@@ -0,0 +1,4 @@
+import type { Chunk } from '../types';
+export declare function createKeywordIndexText(chunk: Chunk): string;
+export declare function tokenizeKeywordText(text: string): string[];
+//# sourceMappingURL=keyword-tokenizer.d.ts.map

package/dist/rag/stores/keyword-tokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"keyword-tokenizer.d.ts","sourceRoot":"","sources":["../../../src/rag/stores/keyword-tokenizer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AAKtC,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,CAQ3D;AAED,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAS1D"}