@fllf/agent-sdk 0.1.2 → 0.1.3

package/dist/rag/retrieval/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/rag/retrieval/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,0CAAwB;AACxB,oDAAkC;AAClC,2CAAyB;AACzB,qDAAmC;AACnC,6DAA2C;AAC3C,6CAA2B;AAC3B,qDAAmC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/rag/retrieval/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;AAAA,0CAAwB;AACxB,oDAAkC;AAClC,2CAAyB;AACzB,qDAAmC;AACnC,6DAA2C;AAC3C,qEAAmD;AACnD,6CAA2B;AAC3B,qDAAmC"}

package/dist/rag/retrieval/parent-child-expanding-retriever.d.ts

@@ -0,0 +1,31 @@
+import type { RetrievedChunk, RetrieveOptions } from '../types';
+import type { DocumentStore } from '../stores';
+import type { Retriever } from './types';
+export type ParentChildExpansionStrategy = 'parent' | 'neighbors' | 'parent_or_neighbors';
+export interface ParentChildExpandingRetrieverOptions {
+    retriever: Retriever;
+    documentStore: DocumentStore;
+    strategy?: ParentChildExpansionStrategy;
+    neighborWindow?: number;
+    includeOriginalChunks?: boolean;
+    maxExpandedChunkCharacters?: number;
+}
+/**
+ * Expands child chunk hits into larger context chunks after retrieval.
+ * This keeps child chunks useful for recall while letting generation receive
+ * parent or neighboring context without changing the underlying retriever.
+ */
+export declare class ParentChildExpandingRetriever implements Retriever {
+    private readonly retriever;
+    private readonly documentStore;
+    private readonly strategy;
+    private readonly neighborWindow;
+    private readonly includeOriginalChunks;
+    private readonly maxExpandedChunkCharacters;
+    constructor(options: ParentChildExpandingRetrieverOptions);
+    retrieve(query: string, options?: RetrieveOptions): Promise<RetrievedChunk[]>;
+    private expandItem;
+    private createParentExpansion;
+    private createNeighborExpansions;
+}
+//# sourceMappingURL=parent-child-expanding-retriever.d.ts.map

package/dist/rag/retrieval/parent-child-expanding-retriever.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parent-child-expanding-retriever.d.ts","sourceRoot":"","sources":["../../../src/rag/retrieval/parent-child-expanding-retriever.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAER,cAAc,EACd,eAAe,EAClB,MAAM,UAAU,CAAC;AAClB,OAAO,KAAK,EACR,aAAa,EAEhB,MAAM,WAAW,CAAC;AAKnB,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAMzC,MAAM,MAAM,4BAA4B,GAAG,QAAQ,GAAG,WAAW,GAAG,qBAAqB,CAAC;AAE1F,MAAM,WAAW,oCAAoC;IACjD,SAAS,EAAE,SAAS,CAAC;IACrB,aAAa,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,EAAE,4BAA4B,CAAC;IACxC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,qBAAqB,CAAC,EAAE,OAAO,CAAC;IAChC,0BAA0B,CAAC,EAAE,MAAM,CAAC;CACvC;AAMD;;;;GAIG;AACH,qBAAa,6BAA8B,YAAW,SAAS;IAC3D,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAY;IACtC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAgB;IAC9C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA+B;IACxD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,QAAQ,CAAC,qBAAqB,CAAU;IAChD,OAAO,CAAC,QAAQ,CAAC,0BAA0B,CAAqB;gBAEpD,OAAO,EAAE,oCAAoC;IAWnD,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;YAsCrE,UAAU;IAyBxB,OAAO,CAAC,qBAAqB;IA2B7B,OAAO,CAAC,wBAAwB;CAwBnC"}

package/dist/rag/retrieval/parent-child-expanding-retriever.js

@@ -0,0 +1,194 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ParentChildExpandingRetriever = void 0;
+const utils_1 = require("../stores/utils");
+const utils_2 = require("./utils");
+/**
+ * Expands child chunk hits into larger context chunks after retrieval.
+ * This keeps child chunks useful for recall while letting generation receive
+ * parent or neighboring context without changing the underlying retriever.
+ */
+class ParentChildExpandingRetriever {
+    retriever;
+    documentStore;
+    strategy;
+    neighborWindow;
+    includeOriginalChunks;
+    maxExpandedChunkCharacters;
+    constructor(options) {
+        this.retriever = options.retriever;
+        this.documentStore = options.documentStore;
+        this.strategy = options.strategy ?? 'parent';
+        this.neighborWindow = normalizeNonNegativeInteger(options.neighborWindow, 0);
+        this.includeOriginalChunks = options.includeOriginalChunks ?? false;
+        this.maxExpandedChunkCharacters = normalizeOptionalPositiveInteger(options.maxExpandedChunkCharacters);
+    }
+    async retrieve(query, options) {
+        const recalled = await this.retriever.retrieve(query, options);
+        if (recalled.length === 0) {
+            return [];
+        }
+        const cache = createDocumentChunkCache(this.documentStore);
+        const searchOptions = (0, utils_2.toSearchOptions)(options);
+        const expanded = [];
+        const expandedByChunkId = new Map();
+        for (const item of recalled) {
+            if (this.includeOriginalChunks) {
+                appendOrMergeRetrievedChunk(expanded, expandedByChunkId, (0, utils_2.cloneRetrievedChunk)(item));
+            }
+            const expandedItems = await this.expandItem(item, cache, searchOptions);
+            if (expandedItems.length === 0) {
+                appendOrMergeRetrievedChunk(expanded, expandedByChunkId, (0, utils_2.cloneRetrievedChunk)(item));
+                continue;
+            }
+            for (const expandedItem of expandedItems) {
+                appendOrMergeRetrievedChunk(expanded, expandedByChunkId, expandedItem);
+            }
+        }
+        return expanded;
+    }
+    async expandItem(item, cache, searchOptions) {
+        const chunks = await cache.get(item.chunk.documentId);
+        if (this.strategy === 'parent') {
+            const parent = this.createParentExpansion(item, chunks, searchOptions);
+            return parent === undefined ? [] : [parent];
+        }
+        if (this.strategy === 'neighbors') {
+            return this.createNeighborExpansions(item, chunks, searchOptions);
+        }
+        const parent = this.createParentExpansion(item, chunks, searchOptions);
+        if (parent !== undefined) {
+            return [parent];
+        }
+        return this.createNeighborExpansions(item, chunks, searchOptions);
+    }
+    createParentExpansion(item, chunks, searchOptions) {
+        const parentId = item.chunk.parentId;
+        if (parentId === undefined) {
+            return undefined;
+        }
+        const parent = chunks.find((chunk) => chunk.id === parentId);
+        if (parent === undefined) {
+            return undefined;
+        }
+        if (!(0, utils_1.matchesSearchOptions)(parent, searchOptions)) {
+            return undefined;
+        }
+        if (this.maxExpandedChunkCharacters !== undefined
+            && parent.content.length > this.maxExpandedChunkCharacters) {
+            return undefined;
+        }
+        return createExpandedRetrievedChunk(parent, item, 'parent');
+    }
+    createNeighborExpansions(item, chunks, searchOptions) {
+        const candidates = getNeighborCandidates(item.chunk, chunks)
+            .filter((chunk) => (0, utils_1.matchesSearchOptions)(chunk, searchOptions))
+            .sort((left, right) => left.metadata.chunkIndex - right.metadata.chunkIndex);
+        const centerIndex = candidates.findIndex((chunk) => chunk.id === item.chunk.id);
+        if (centerIndex < 0) {
+            return [];
+        }
+        const start = Math.max(0, centerIndex - this.neighborWindow);
+        const end = Math.min(candidates.length, centerIndex + this.neighborWindow + 1);
+        return candidates
+            .slice(start, end)
+            .map((chunk) => (chunk.id === item.chunk.id
+            ? createExpandedRetrievedChunk(chunk, item, 'self')
+            : createExpandedRetrievedChunk(chunk, item, 'neighbor')));
+    }
+}
+exports.ParentChildExpandingRetriever = ParentChildExpandingRetriever;
+function createDocumentChunkCache(documentStore) {
+    const chunksByDocumentId = new Map();
+    return {
+        async get(documentId) {
+            let chunks = chunksByDocumentId.get(documentId);
+            if (chunks === undefined) {
+                chunks = documentStore.getChunksByDocument(documentId);
+                chunksByDocumentId.set(documentId, chunks);
+            }
+            return await chunks;
+        },
+    };
+}
+function getNeighborCandidates(hit, chunks) {
+    if (hit.parentId !== undefined) {
+        return chunks.filter((chunk) => chunk.id === hit.id || chunk.parentId === hit.parentId);
+    }
+    return chunks.filter((chunk) => chunk.parentId === undefined);
+}
+function createExpandedRetrievedChunk(chunk, original, expansion) {
+    const item = {
+        chunk: (0, utils_1.cloneChunk)(chunk),
+        score: original.score,
+        source: original.source,
+        metadata: createExpansionMetadata(original, expansion),
+    };
+    return item;
+}
+function createExpansionMetadata(original, expansion) {
+    const metadata = {
+        ...(original.metadata ?? {}),
+        expansion,
+        expandedFromChunkIds: [original.chunk.id],
+    };
+    assignIfDefined(metadata, 'expandedFromParentId', original.chunk.parentId);
+    return metadata;
+}
+function appendOrMergeRetrievedChunk(chunks, chunksById, item) {
+    const existing = chunksById.get(item.chunk.id);
+    if (existing === undefined) {
+        chunks.push(item);
+        chunksById.set(item.chunk.id, item);
+        return;
+    }
+    existing.score = Math.max(existing.score, item.score);
+    const metadata = mergeRetrievedMetadata(existing.metadata, item.metadata);
+    if (metadata === undefined) {
+        delete existing.metadata;
+    }
+    else {
+        existing.metadata = metadata;
+    }
+}
+function mergeRetrievedMetadata(left, right) {
+    if (left === undefined && right === undefined) {
+        return undefined;
+    }
+    const metadata = {
+        ...(left ?? {}),
+        ...(right ?? {}),
+    };
+    const expandedFromChunkIds = [
+        ...readStringArray(left?.expandedFromChunkIds),
+        ...readStringArray(right?.expandedFromChunkIds),
+    ];
+    if (expandedFromChunkIds.length > 0) {
+        metadata.expandedFromChunkIds = [...new Set(expandedFromChunkIds)];
+    }
+    return metadata;
+}
+function readStringArray(value) {
+    if (!Array.isArray(value)) {
+        return [];
+    }
+    return value.filter((item) => typeof item === 'string');
+}
+function normalizeNonNegativeInteger(value, fallback) {
+    if (value === undefined || !Number.isFinite(value) || value < 0) {
+        return fallback;
+    }
+    return Math.floor(value);
+}
+function normalizeOptionalPositiveInteger(value) {
+    if (value === undefined || !Number.isFinite(value) || value <= 0) {
+        return undefined;
+    }
+    return Math.floor(value);
+}
+function assignIfDefined(target, key, value) {
+    if (value !== undefined) {
+        target[key] = value;
+    }
+}
+//# sourceMappingURL=parent-child-expanding-retriever.js.map

package/dist/rag/retrieval/parent-child-expanding-retriever.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parent-child-expanding-retriever.js","sourceRoot":"","sources":["../../../src/rag/retrieval/parent-child-expanding-retriever.ts"],"names":[],"mappings":";;;AASA,2CAGyB;AAEzB,mCAGiB;AAiBjB;;;;GAIG;AACH,MAAa,6BAA6B;IACrB,SAAS,CAAY;IACrB,aAAa,CAAgB;IAC7B,QAAQ,CAA+B;IACvC,cAAc,CAAS;IACvB,qBAAqB,CAAU;IAC/B,0BAA0B,CAAqB;IAEhE,YAAY,OAA6C;QACrD,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;QAC3C,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,QAAQ,CAAC;QAC7C,IAAI,CAAC,cAAc,GAAG,2BAA2B,CAAC,OAAO,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC;QAC7E,IAAI,CAAC,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,IAAI,KAAK,CAAC;QACpE,IAAI,CAAC,0BAA0B,GAAG,gCAAgC,CAC9D,OAAO,CAAC,0BAA0B,CACrC,CAAC;IACN,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAa,EAAE,OAAyB;QACnD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QAC/D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,EAAE,CAAC;QACd,CAAC;QAED,MAAM,KAAK,GAAG,wBAAwB,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC3D,MAAM,aAAa,GAAG,IAAA,uBAAe,EAAC,OAAO,CAAC,CAAC;QAC/C,MAAM,QAAQ,GAAqB,EAAE,CAAC;QACtC,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAA0B,CAAC;QAE5D,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC1B,IAAI,IAAI,CAAC,qBAAqB,EAAE,CAAC;gBAC7B,2BAA2B,CACvB,QAAQ,EACR,iBAAiB,EACjB,IAAA,2BAAmB,EAAC,IAAI,CAAC,CAC5B,CAAC;YACN,CAAC;YAED,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,KAAK,EAAE,aAAa,CAAC,CAAC;YACxE,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAC7B,2BAA2B,CACvB,QAAQ,EACR,iBAAiB,EACjB,IAAA,2BAAmB,EAAC,IAAI,CAAC,CAC5B,CAAC;gBACF,SAAS;YACb,CAAC;YAED,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;gBACvC,2BAA2B,CAAC,QAAQ,EAAE,iBAAiB,EAAE,YAAY,CAAC,CAAC;YAC3E,CAAC;QACL,CAAC;QAED,OAAO,QAAQ,CAAC;IACpB,CAAC;IAEO,KAAK,CAAC,UAAU,CACpB,IAAoB,EACpB,KAAyB,EACzB,aAA4B;QAE5B,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QAEtD,IAAI,IAAI,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;YAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,MAAM,EAAE,aAAa,CAAC,CAAC;YAEvE,OAAO,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QAChD,CAAC;QAED,IAAI,IAAI,CAAC,QAAQ,KAAK,WAAW,EAAE,CAAC;YAChC,OAAO,IAAI,CAAC,wBAAwB,CAAC,IAAI,EAAE,MAAM,EAAE,aAAa,CAAC,CAAC;QACtE,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,qBAAqB,CAAC,IAAI,EAAE,MAAM,EAAE,aAAa,CAAC,CAAC;QACvE,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACvB,OAAO,CAAC,MAAM,CAAC,CAAC;QACpB,CAAC;QAED,OAAO,IAAI,CAAC,wBAAwB,CAAC,IAAI,EAAE,MAAM,EAAE,aAAa,CAAC,CAAC;IACtE,CAAC;IAEO,qBAAqB,CACzB,IAAoB,EACpB,MAAe,EACf,aAA4B;QAE5B,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC;QACrC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YACzB,OAAO,SAAS,CAAC;QACrB,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,QAAQ,CAAC,CAAC;QAC7D,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACvB,OAAO,SAAS,CAAC;QACrB,CAAC;QACD,IAAI,CAAC,IAAA,4BAAoB,EAAC,MAAM,EAAE,aAAa,CAAC,EAAE,CAAC;YAC/C,OAAO,SAAS,CAAC;QACrB,CAAC;QACD,IACI,IAAI,CAAC,0BAA0B,KAAK,SAAS;eAC1C,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,0BAA0B,EAC5D,CAAC;YACC,OAAO,SAAS,CAAC;QACrB,CAAC;QAED,OAAO,4BAA4B,CAAC,MAAM,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;IAChE,CAAC;IAEO,wBAAwB,CAC5B,IAAoB,EACpB,MAAe,EACf,aAA4B;QAE5B,MAAM,UAAU,GAAG,qBAAqB,CAAC,IAAI,CAAC,KAAK,EAAE,MAAM,CAAC;aACvD,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAA,4BAAoB,EAAC,KAAK,EAAE,aAAa,CAAC,CAAC;aAC7D,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,GAAG,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;QACjF,MAAM,WAAW,GAAG,UAAU,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAChF,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;YAClB,OAAO,EAAE,CAAC;QACd,CAAC;QAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,GAAG,IAAI,CAAC,cAAc,CAAC,CAAC;QAC7D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,MAAM,EAAE,WAAW,GAAG,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC,CAAC;QAE/E,OAAO,UAAU;aACZ,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC;aACjB,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CACZ,KAAK,CAAC,EAAE,KAAK,IAAI,CAAC,KAAK,CAAC,EAAE;YACtB,CAAC,CAAC,4BAA4B,CAAC,KAAK,EAAE,IAAI,EAAE,MAAM,CAAC;YACnD,CAAC,CAAC,4BAA4B,CAAC,KAAK,EAAE,IAAI,EAAE,UAAU,CAAC,CAC9D,CAAC,CAAC;IACX,CAAC;CACJ;AArID,sEAqIC;AAED,SAAS,wBAAwB,CAAC,aAA4B;IAC1D,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAA4B,CAAC;IAE/D,OAAO;QACH,KAAK,CAAC,GAAG,CAAC,UAAkB;YACxB,IAAI,MAAM,GAAG,kBAAkB,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAChD,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gBACvB,MAAM,GAAG,aAAa,CAAC,mBAAmB,CAAC,UAAU,CAAC,CAAC;gBACvD,kBAAkB,CAAC,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;YAC/C,CAAC;YAED,OAAO,MAAM,MAAM,CAAC;QACxB,CAAC;KACJ,CAAC;AACN,CAAC;AAED,SAAS,qBAAqB,CAAC,GAAU,EAAE,MAAe;IACtD,IAAI,GAAG,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QAC7B,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,IAAI,KAAK,CAAC,QAAQ,KAAK,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC5F,CAAC;IAED,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,4BAA4B,CACjC,KAAY,EACZ,QAAwB,EACxB,SAAyC;IAEzC,MAAM,IAAI,GAAmB;QACzB,KAAK,EAAE,IAAA,kBAAU,EAAC,KAAK,CAAC;QACxB,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,MAAM,EAAE,QAAQ,CAAC,MAAM;QACvB,QAAQ,EAAE,uBAAuB,CAAC,QAAQ,EAAE,SAAS,CAAC;KACzD,CAAC;IAEF,OAAO,IAAI,CAAC;AAChB,CAAC;AAED,SAAS,uBAAuB,CAC5B,QAAwB,EACxB,SAAyC;IAEzC,MAAM,QAAQ,GAA4B;QACtC,GAAG,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC;QAC5B,SAAS;QACT,oBAAoB,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;KAC5C,CAAC;IAEF,eAAe,CAAC,QAAQ,EAAE,sBAAsB,EAAE,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAE3E,OAAO,QAAQ,CAAC;AACpB,CAAC;AAED,SAAS,2BAA2B,CAChC,MAAwB,EACxB,UAAuC,EACvC,IAAoB;IAEpB,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAC/C,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClB,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QACpC,OAAO;IACX,CAAC;IAED,QAAQ,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;IACtD,MAAM,QAAQ,GAAG,sBAAsB,CAAC,QAAQ,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC1E,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;QACzB,OAAO,QAAQ,CAAC,QAAQ,CAAC;IAC7B,CAAC;SAAM,CAAC;QACJ,QAAQ,CAAC,QAAQ,GAAG,QAAQ,CAAC;IACjC,CAAC;AACL,CAAC;AAED,SAAS,sBAAsB,CAC3B,IAAyC,EACzC,KAA0C;IAE1C,IAAI,IAAI,KAAK,SAAS,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QAC5C,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,MAAM,QAAQ,GAA4B;QACtC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC;QACf,GAAG,CAAC,KAAK,IAAI,EAAE,CAAC;KACnB,CAAC;IACF,MAAM,oBAAoB,GAAG;QACzB,GAAG,eAAe,CAAC,IAAI,EAAE,oBAAoB,CAAC;QAC9C,GAAG,eAAe,CAAC,KAAK,EAAE,oBAAoB,CAAC;KAClD,CAAC;IAEF,IAAI,oBAAoB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,QAAQ,CAAC,oBAAoB,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,oBAAoB,CAAC,CAAC,CAAC;IACvE,CAAC;IAED,OAAO,QAAQ,CAAC;AACpB,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACnC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,CAAC;IACd,CAAC;IAED,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC;AAC5E,CAAC;AAED,SAAS,2BAA2B,CAAC,KAAyB,EAAE,QAAgB;IAC5E,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QAC9D,OAAO,QAAQ,CAAC;IACpB,CAAC;IAED,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,gCAAgC,CAAC,KAAyB;IAC/D,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;QAC/D,OAAO,SAAS,CAAC;IACrB,CAAC;IAED,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,eAAe,CACpB,MAAS,EACT,GAAM,EACN,KAAuB;IAEvB,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACtB,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;IACxB,CAAC;AACL,CAAC"}

package/dist/rag/stores/in-memory-keyword-store.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"in-memory-keyword-store.d.ts","sourceRoot":"","sources":["../../../src/rag/stores/in-memory-keyword-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,KAAK,EACR,YAAY,EACZ,cAAc,EACd,aAAa,EAChB,MAAM,SAAS,CAAC;AAajB,qBAAa,oBAAqB,YAAW,YAAY;IACrD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwC;IAE1D,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAWtC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAuC7E,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQzD,OAAO,CAAC,0BAA0B;CAerC"}
+{"version":3,"file":"in-memory-keyword-store.d.ts","sourceRoot":"","sources":["../../../src/rag/stores/in-memory-keyword-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AACtC,OAAO,KAAK,EACR,YAAY,EACZ,cAAc,EACd,aAAa,EAChB,MAAM,SAAS,CAAC;AAiBjB,qBAAa,oBAAqB,YAAW,YAAY;IACrD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAwC;IAE1D,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAWtC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAuC7E,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQzD,OAAO,CAAC,0BAA0B;CAerC"}

package/dist/rag/stores/in-memory-keyword-store.js

@@ -2,11 +2,12 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.InMemoryKeywordStore = void 0;
 const utils_1 = require("./utils");
+const keyword_tokenizer_1 = require("./keyword-tokenizer");
 class InMemoryKeywordStore {
     entries = new Map();
     async upsert(chunks) {
         for (const chunk of chunks) {
-            const terms = tokenize(chunk.content);
+            const terms = (0, keyword_tokenizer_1.tokenizeKeywordText)((0, keyword_tokenizer_1.createKeywordIndexText)(chunk));
             this.entries.set(chunk.id, {
                 chunk: (0, utils_1.cloneChunk)(chunk),
                 termCounts: countTerms(terms),
@@ -15,7 +16,7 @@ class InMemoryKeywordStore {
         }
     }
     async search(query, options = {}) {
-        const queryTerms = [...new Set(tokenize(query))];
+        const queryTerms = [...new Set((0, keyword_tokenizer_1.tokenizeKeywordText)(query))];
         if (queryTerms.length === 0) {
             return [];
         }
@@ -83,12 +84,6 @@ function scoreEntry(input) {
     }
     return score;
 }
-function tokenize(text) {
-    return text
-        .normalize('NFKC')
-        .toLowerCase()
-        .match(/[\p{L}\p{N}_-]+/gu) ?? [];
-}
 function countTerms(terms) {
     const counts = new Map();
     for (const term of terms) {

package/dist/rag/stores/in-memory-keyword-store.js.map

@@ -1 +1 @@
-{"version":3,"file":"in-memory-keyword-store.js","sourceRoot":"","sources":["../../../src/rag/stores/in-memory-keyword-store.ts"],"names":[],"mappings":";;;AAMA,mCAIiB;AAQjB,MAAa,oBAAoB;IACZ,OAAO,GAAG,IAAI,GAAG,EAA6B,CAAC;IAEhE,KAAK,CAAC,MAAM,CAAC,MAAe;QACxB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACzB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACtC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE;gBACvB,KAAK,EAAE,IAAA,kBAAU,EAAC,KAAK,CAAC;gBACxB,UAAU,EAAE,UAAU,CAAC,KAAK,CAAC;gBAC7B,MAAM,EAAE,KAAK,CAAC,MAAM;aACvB,CAAC,CAAC;QACP,CAAC;IACL,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,UAAyB,EAAE;QACnD,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACjD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,OAAO,EAAE,CAAC;QACd,CAAC;QAED,MAAM,IAAI,GAAqB,EAAE,CAAC;QAClC,MAAM,iBAAiB,GAAG,IAAI,CAAC,0BAA0B,CAAC,UAAU,CAAC,CAAC;QACtE,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAEtD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YACxC,IAAI,CAAC,IAAA,4BAAoB,EAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,CAAC;gBAC9C,SAAS;YACb,CAAC;YAED,MAAM,KAAK,GAAG,UAAU,CAAC;gBACrB,KAAK;gBACL,UAAU;gBACV,iBAAiB;gBACjB,cAAc;aACjB,CAAC,CAAC;YACH,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;gBACb,SAAS;YACb,CAAC;YAED,IAAI,CAAC,IAAI,CAAC;gBACN,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,EAAE;gBACvB,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC,UAAU;gBAClC,KAAK;gBACL,QAAQ,EAAE;oBACN,MAAM,EAAE,SAAS;oBACjB,YAAY,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;iBACxE;aACJ,CAAC,CAAC;QACP,CAAC;QAED,OAAO,IAAA,yBAAiB,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,UAAkB;QACrC,KAAK,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YACpD,IAAI,KAAK,CAAC,KAAK,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;gBACxC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACjC,CAAC;QACL,CAAC;IACL,CAAC;IAEO,0BAA0B,CAAC,UAAoB;QACnD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;QAE5C,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;gBACxC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC7B,KAAK,IAAI,CAAC,CAAC;gBACf,CAAC;YACL,CAAC;YACD,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC/B,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;CACJ;AA5ED,oDA4EC;AASD,SAAS,UAAU,CAAC,KAAsB;IACtC,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;QAClC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxD,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;YAClB,SAAS;QACb,CAAC;QAED,MAAM,EAAE,GAAG,KAAK,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,GAAG,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;QACzE,MAAM,YAAY,GAAG,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;QAC5E,KAAK,IAAI,YAAY,GAAG,GAAG,CAAC;IAChC,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC1B,OAAO,IAAI;SACN,SAAS,CAAC,MAAM,CAAC;SACjB,WAAW,EAAE;SACb,KAAK,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC;AAC1C,CAAC;AAED,SAAS,UAAU,CAAC,KAAe;IAC/B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IAEzC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC"}
+{"version":3,"file":"in-memory-keyword-store.js","sourceRoot":"","sources":["../../../src/rag/stores/in-memory-keyword-store.ts"],"names":[],"mappings":";;;AAMA,mCAIiB;AACjB,2DAG6B;AAQ7B,MAAa,oBAAoB;IACZ,OAAO,GAAG,IAAI,GAAG,EAA6B,CAAC;IAEhE,KAAK,CAAC,MAAM,CAAC,MAAe;QACxB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACzB,MAAM,KAAK,GAAG,IAAA,uCAAmB,EAAC,IAAA,0CAAsB,EAAC,KAAK,CAAC,CAAC,CAAC;YACjE,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE;gBACvB,KAAK,EAAE,IAAA,kBAAU,EAAC,KAAK,CAAC;gBACxB,UAAU,EAAE,UAAU,CAAC,KAAK,CAAC;gBAC7B,MAAM,EAAE,KAAK,CAAC,MAAM;aACvB,CAAC,CAAC;QACP,CAAC;IACL,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,UAAyB,EAAE;QACnD,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,IAAA,uCAAmB,EAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC5D,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,OAAO,EAAE,CAAC;QACd,CAAC;QAED,MAAM,IAAI,GAAqB,EAAE,CAAC;QAClC,MAAM,iBAAiB,GAAG,IAAI,CAAC,0BAA0B,CAAC,UAAU,CAAC,CAAC;QACtE,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAEtD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YACxC,IAAI,CAAC,IAAA,4BAAoB,EAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,EAAE,CAAC;gBAC9C,SAAS;YACb,CAAC;YAED,MAAM,KAAK,GAAG,UAAU,CAAC;gBACrB,KAAK;gBACL,UAAU;gBACV,iBAAiB;gBACjB,cAAc;aACjB,CAAC,CAAC;YACH,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;gBACb,SAAS;YACb,CAAC;YAED,IAAI,CAAC,IAAI,CAAC;gBACN,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,EAAE;gBACvB,UAAU,EAAE,KAAK,CAAC,KAAK,CAAC,UAAU;gBAClC,KAAK;gBACL,QAAQ,EAAE;oBACN,MAAM,EAAE,SAAS;oBACjB,YAAY,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;iBACxE;aACJ,CAAC,CAAC;QACP,CAAC;QAED,OAAO,IAAA,yBAAiB,EAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,UAAkB;QACrC,KAAK,MAAM,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YACpD,IAAI,KAAK,CAAC,KAAK,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;gBACxC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACjC,CAAC;QACL,CAAC;IACL,CAAC;IAEO,0BAA0B,CAAC,UAAoB;QACnD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;QAE5C,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC5B,IAAI,KAAK,GAAG,CAAC,CAAC;YACd,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;gBACxC,IAAI,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;oBAC7B,KAAK,IAAI,CAAC,CAAC;gBACf,CAAC;YACL,CAAC;YACD,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC/B,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;CACJ;AA5ED,oDA4EC;AASD,SAAS,UAAU,CAAC,KAAsB;IACtC,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;QAClC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxD,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;YAClB,SAAS;QACb,CAAC;QAED,MAAM,EAAE,GAAG,KAAK,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,GAAG,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;QACzE,MAAM,YAAY,GAAG,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;QAC5E,KAAK,IAAI,YAAY,GAAG,GAAG,CAAC;IAChC,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,SAAS,UAAU,CAAC,KAAe;IAC/B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IAEzC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC"}

package/dist/rag/stores/keyword-tokenizer.d.ts

@@ -0,0 +1,4 @@
+import type { Chunk } from '../types';
+export declare function createKeywordIndexText(chunk: Chunk): string;
+export declare function tokenizeKeywordText(text: string): string[];
+//# sourceMappingURL=keyword-tokenizer.d.ts.map

package/dist/rag/stores/keyword-tokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"keyword-tokenizer.d.ts","sourceRoot":"","sources":["../../../src/rag/stores/keyword-tokenizer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,UAAU,CAAC;AAKtC,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,KAAK,GAAG,MAAM,CAQ3D;AAED,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAS1D"}

package/dist/rag/stores/keyword-tokenizer.js

@@ -0,0 +1,113 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createKeywordIndexText = createKeywordIndexText;
+exports.tokenizeKeywordText = tokenizeKeywordText;
+const RAW_TOKEN_PATTERN = /[\p{L}\p{N}_-]+/gu;
+const CJK_CHARACTER_PATTERN = /[\u3400-\u4DBF\u4E00-\u9FFF\uF900-\uFAFF]/u;
+function createKeywordIndexText(chunk) {
+    return [
+        chunk.metadata.title,
+        chunk.metadata.headingPath?.join(' '),
+        chunk.content,
+    ]
+        .filter((part) => part !== undefined && part.trim().length > 0)
+        .join('\n');
+}
+function tokenizeKeywordText(text) {
+    const rawTokens = text.normalize('NFKC').match(RAW_TOKEN_PATTERN) ?? [];
+    const terms = [];
+    for (const rawToken of rawTokens) {
+        terms.push(...createTokenTerms(rawToken));
+    }
+    return terms;
+}
+function createTokenTerms(rawToken) {
+    const terms = new Set();
+    addTerm(terms, rawToken);
+    for (const part of splitMixedToken(rawToken)) {
+        addTerm(terms, part.value);
+        if (part.kind === 'cjk') {
+            addCjkNgrams(terms, part.value);
+        }
+        else {
+            addIdentifierTerms(terms, part.value);
+        }
+    }
+    return [...terms];
+}
+function splitMixedToken(token) {
+    const parts = [];
+    let current = '';
+    let currentKind;
+    for (const character of Array.from(token)) {
+        const kind = isCjkCharacter(character) ? 'cjk' : 'identifier';
+        if (currentKind !== undefined && currentKind !== kind) {
+            parts.push({
+                value: current,
+                kind: currentKind,
+            });
+            current = '';
+        }
+        current += character;
+        currentKind = kind;
+    }
+    if (current.length > 0 && currentKind !== undefined) {
+        parts.push({
+            value: current,
+            kind: currentKind,
+        });
+    }
+    return parts;
+}
+function addIdentifierTerms(terms, value) {
+    const separatorParts = value
+        .split(/[_-]+/u)
+        .filter((part) => part.length > 0);
+    for (const part of separatorParts) {
+        addTerm(terms, part);
+        addCaseSplitTerms(terms, part);
+    }
+}
+function addCaseSplitTerms(terms, value) {
+    const parts = splitIdentifierCase(value);
+    if (parts.length <= 1) {
+        return;
+    }
+    for (const part of parts) {
+        addTerm(terms, part);
+    }
+    for (let index = 0; index < parts.length - 1; index += 1) {
+        const current = parts[index];
+        const next = parts[index + 1];
+        if (current !== undefined && next !== undefined && isAcronym(next)) {
+            addTerm(terms, `${current}${next}`);
+        }
+    }
+}
+function splitIdentifierCase(value) {
+    return value.match(/[A-Z]?[a-z]+|[A-Z]+(?![a-z])|\d+/g) ?? [value];
+}
+function addCjkNgrams(terms, value) {
+    const characters = Array.from(value);
+    for (const size of [2, 3]) {
+        if (characters.length < size) {
+            continue;
+        }
+        for (let index = 0; index <= characters.length - size; index += 1) {
+            addTerm(terms, characters.slice(index, index + size).join(''));
+        }
+    }
+}
+function addTerm(terms, value) {
+    const normalized = value.toLowerCase().trim();
+    if (normalized.length > 0) {
+        terms.add(normalized);
+    }
+}
+function isCjkCharacter(character) {
+    return CJK_CHARACTER_PATTERN.test(character);
+}
+function isAcronym(value) {
+    return /^[A-Z0-9]{2,}$/u.test(value);
+}
+//# sourceMappingURL=keyword-tokenizer.js.map

package/dist/rag/stores/keyword-tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"keyword-tokenizer.js","sourceRoot":"","sources":["../../../src/rag/stores/keyword-tokenizer.ts"],"names":[],"mappings":";;AAKA,wDAQC;AAED,kDASC;AAtBD,MAAM,iBAAiB,GAAG,mBAAmB,CAAC;AAC9C,MAAM,qBAAqB,GAAG,4CAA4C,CAAC;AAE3E,SAAgB,sBAAsB,CAAC,KAAY;IAC/C,OAAO;QACH,KAAK,CAAC,QAAQ,CAAC,KAAK;QACpB,KAAK,CAAC,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC;QACrC,KAAK,CAAC,OAAO;KAChB;SACI,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;SAC9E,IAAI,CAAC,IAAI,CAAC,CAAC;AACpB,CAAC;AAED,SAAgB,mBAAmB,CAAC,IAAY;IAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC,IAAI,EAAE,CAAC;IACxE,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC9C,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,SAAS,gBAAgB,CAAC,QAAgB;IACtC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAEhC,OAAO,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IAEzB,KAAK,MAAM,IAAI,IAAI,eAAe,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC3C,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QAE3B,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YACtB,YAAY,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QACpC,CAAC;aAAM,CAAC;YACJ,kBAAkB,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1C,CAAC;IACL,CAAC;IAED,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC;AACtB,CAAC;AAED,SAAS,eAAe,CAAC,KAAa;IAClC,MAAM,KAAK,GAAyD,EAAE,CAAC;IACvE,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,WAA6C,CAAC;IAElD,KAAK,MAAM,SAAS,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC;QAC9D,IAAI,WAAW,KAAK,SAAS,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;YACpD,KAAK,CAAC,IAAI,CAAC;gBACP,KAAK,EAAE,OAAO;gBACd,IAAI,EAAE,WAAW;aACpB,CAAC,CAAC;YACH,OAAO,GAAG,EAAE,CAAC;QACjB,CAAC;QAED,OAAO,IAAI,SAAS,CAAC;QACrB,WAAW,GAAG,IAAI,CAAC;IACvB,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;QAClD,KAAK,CAAC,IAAI,CAAC;YACP,KAAK,EAAE,OAAO;YACd,IAAI,EAAE,WAAW;SACpB,CAAC,CAAC;IACP,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAkB,EAAE,KAAa;IACzD,MAAM,cAAc,GAAG,KAAK;SACvB,KAAK,CAAC,QAAQ,CAAC;SACf,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAEvC,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;QAChC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QACrB,iBAAiB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IACnC,CAAC;AACL,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAkB,EAAE,KAAa;IACxD,MAAM,KAAK,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;IACzC,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACpB,OAAO;IACX,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IACzB,CAAC;IAED,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACvD,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;QAC7B,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QAC9B,IAAI,OAAO,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;YACjE,OAAO,CAAC,KAAK,EAAE,GAAG,OAAO,GAAG,IAAI,EAAE,CAAC,CAAC;QACxC,CAAC;IACL,CAAC;AACL,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAa;IACtC,OAAO,KAAK,CAAC,KAAK,CAAC,mCAAmC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACvE,CAAC;AAED,SAAS,YAAY,CAAC,KAAkB,EAAE,KAAa;IACnD,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAErC,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QACxB,IAAI,UAAU,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;YAC3B,SAAS;QACb,CAAC;QAED,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,UAAU,CAAC,MAAM,GAAG,IAAI,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YAChE,OAAO,CAAC,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QACnE,CAAC;IACL,CAAC;AACL,CAAC;AAED,SAAS,OAAO,CAAC,KAAkB,EAAE,KAAa;IAC9C,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;IAC9C,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAC1B,CAAC;AACL,CAAC;AAED,SAAS,cAAc,CAAC,SAAiB;IACrC,OAAO,qBAAqB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;AACjD,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC5B,OAAO,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACzC,CAAC"}

package/dist/rag/stores/postgres/pg-vector-store.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pg-vector-store.d.ts","sourceRoot":"","sources":["../../../../src/rag/stores/postgres/pg-vector-store.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACR,aAAa,EAChB,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EACR,cAAc,EACd,aAAa,EACb,WAAW,EACd,MAAM,UAAU,CAAC;AAOlB,OAAO,KAAK,EAER,uBAAuB,EAE1B,MAAM,SAAS,CAAC;AAEjB,MAAM,WAAW,oBAAqB,SAAQ,uBAAuB;IACjE,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,qBAAa,aAAc,YAAW,WAAW;IAC7C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAwB;IAC3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;gBAEhC,OAAO,EAAE,oBAAoB;IAMnC,MAAM,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IA4B7C,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IA2ChF,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAM5D"}
+{"version":3,"file":"pg-vector-store.d.ts","sourceRoot":"","sources":["../../../../src/rag/stores/postgres/pg-vector-store.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACR,aAAa,EAChB,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EACR,cAAc,EACd,aAAa,EACb,WAAW,EACd,MAAM,UAAU,CAAC;AAUlB,OAAO,KAAK,EAER,uBAAuB,EAE1B,MAAM,SAAS,CAAC;AAEjB,MAAM,WAAW,oBAAqB,SAAQ,uBAAuB;IACjE,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,qBAAa,aAAc,YAAW,WAAW;IAC7C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAwB;IAC3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;gBAEhC,OAAO,EAAE,oBAAoB;IAMnC,MAAM,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAmC7C,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAoDhF,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAM5D"}

package/dist/rag/stores/postgres/pg-vector-store.js

@@ -13,23 +13,29 @@ class PgVectorStore {
         this.dimensions = (0, embeddings_1.assertValidEmbeddingDimensions)(options.dimensions);
     }
     async upsert(items) {
+        // 先整体校验维度再分批写入，避免批次中途失败留下半写状态。
         for (const item of items) {
             (0, embeddings_1.assertEmbeddingDimensions)(item.embedding, this.dimensions, `chunk ${item.chunk.id} embedding`);
+        }
+        for (const batch of (0, sql_1.chunkBatches)((0, sql_1.dedupeByKey)(items, (item) => item.chunk.id))) {
             await this.db.query(`
 INSERT INTO ${this.tables.embeddings} (
     chunk_id, document_id, embedding, updated_at
-) VALUES (
-    $1, $2, $3::vector, now()
-)
+) VALUES
+${(0, sql_1.createMultiRowValuesSql)({
+                rowCount: batch.length,
+                casts: ['', '', '::vector'],
+                rowSuffix: ', now()',
+            })}
 ON CONFLICT (chunk_id) DO UPDATE SET
     document_id = EXCLUDED.document_id,
     embedding = EXCLUDED.embedding,
     updated_at = now()
-`.trim(), [
+`.trim(), batch.flatMap((item) => [
                 item.chunk.id,
                 item.chunk.documentId,
                 (0, sql_1.toVectorLiteral)(item.embedding),
-            ]);
+            ]));
         }
     }
     async search(vector, options = {}) {
@@ -43,6 +49,9 @@ ON CONFLICT (chunk_id) DO UPDATE SET
             ? ''
             : `${where.clause.length === 0 ? 'WHERE' : 'AND'} 1 - (e.embedding <=> $1::vector) >= $${where.values.length + 2}`;
         const limitPlaceholder = `$${where.values.length + (options.minScore === undefined ? 2 : 3)}`;
+        // ORDER BY 只保留向量距离表达式：一旦追加 document_id / chunk_id 等次级排序键，
+        // planner 就无法走 HNSW 索引，会退化成全表扫描。
+        // 确定性 tie-break（同分按 documentId/chunkId）挪到 Node 侧对返回的 topK 结果做稳定排序。
         const result = await this.db.query(`
 SELECT
     e.chunk_id,
@@ -52,7 +61,7 @@ FROM ${this.tables.embeddings} e
 JOIN ${this.tables.chunks} c ON c.id = e.chunk_id
 ${where.clause}
 ${minScoreClause}
-ORDER BY e.embedding <=> $1::vector ASC, e.document_id ASC, e.chunk_id ASC
+ORDER BY e.embedding <=> $1::vector ASC
 LIMIT ${limitPlaceholder}
 `.trim(), [
             (0, sql_1.toVectorLiteral)(vector),
@@ -60,14 +69,18 @@ LIMIT ${limitPlaceholder}
             ...(options.minScore === undefined ? [] : [options.minScore]),
             topK,
         ]);
-        return result.rows.map((row) => ({
+        return result.rows
+            .map((row) => ({
             chunkId: String(row.chunk_id),
             documentId: String(row.document_id),
             score: Number(row.score ?? 0),
             metadata: {
                 source: 'vector',
             },
-        }));
+        }))
+            .sort((left, right) => (right.score - left.score
+            || left.documentId.localeCompare(right.documentId)
+            || left.chunkId.localeCompare(right.chunkId)));
     }
     async deleteByDocument(documentId) {
         await this.db.query(`DELETE FROM ${this.tables.embeddings} WHERE document_id = $1`, [documentId]);

package/dist/rag/stores/postgres/pg-vector-store.js.map

@@ -1 +1 @@
-{"version":3,"file":"pg-vector-store.js","sourceRoot":"","sources":["../../../../src/rag/stores/postgres/pg-vector-store.ts"],"names":[],"mappings":";;;AAAA,iDAG0B;AAS1B,+BAKe;AAWf,MAAa,aAAa;IACb,UAAU,CAAS;IACX,EAAE,CAAwB;IAC1B,MAAM,CAAqB;IAE5C,YAAY,OAA6B;QACrC,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAA,+BAAyB,EAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAC7D,IAAI,CAAC,UAAU,GAAG,IAAA,2CAA8B,EAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAsB;QAC/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,IAAA,sCAAyB,EACrB,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,UAAU,EACf,SAAS,IAAI,CAAC,KAAK,CAAC,EAAE,YAAY,CACrC,CAAC;YACF,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CACf;cACF,IAAI,CAAC,MAAM,CAAC,UAAU;;;;;;;;;CASnC,CAAC,IAAI,EAAE,EACQ;gBACI,IAAI,CAAC,KAAK,CAAC,EAAE;gBACb,IAAI,CAAC,KAAK,CAAC,UAAU;gBACrB,IAAA,qBAAe,EAAC,IAAI,CAAC,SAAS,CAAC;aAClC,CACJ,CAAC;QACN,CAAC;IACL,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,MAAgB,EAAE,UAAyB,EAAE;QACtD,IAAA,sCAAyB,EAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,iBAAiB,CAAC,CAAC;QACtE,MAAM,IAAI,GAAG,IAAA,mBAAa,EAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;YACb,OAAO,EAAE,CAAC;QACd,CAAC;QAED,MAAM,KAAK,GAAG,IAAA,2BAAqB,EAAC,OAAO,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;QACrD,MAAM,cAAc,GAAG,OAAO,CAAC,QAAQ,KAAK,SAAS;YACjD,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,yCAAyC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvH,MAAM,gBAAgB,GAAG,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9F,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAC9B;;;;;OAKL,IAAI,CAAC,MAAM,CAAC,UAAU;OACtB,IAAI,CAAC,MAAM,CAAC,MAAM;EACvB,KAAK,CAAC,MAAM;EACZ,cAAc;;QAER,gBAAgB;CACvB,CAAC,IAAI,EAAE,EACI;YACI,IAAA,qBAAe,EAAC,MAAM,CAAC;YACvB,GAAG,KAAK,CAAC,MAAM;YACf,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAC7D,IAAI;SACP,CACJ,CAAC;QAEF,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAC7B,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC7B,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC;YACnC,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC;YAC7B,QAAQ,EAAE;gBACN,MAAM,EAAE,QAAQ;aACnB;SACJ,CAAC,CAAC,CAAC;IACR,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,UAAkB;QACrC,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CACf,eAAe,IAAI,CAAC,MAAM,CAAC,UAAU,yBAAyB,EAC9D,CAAC,UAAU,CAAC,CACf,CAAC;IACN,CAAC;CACJ;AAxFD,sCAwFC"}
+{"version":3,"file":"pg-vector-store.js","sourceRoot":"","sources":["../../../../src/rag/stores/postgres/pg-vector-store.ts"],"names":[],"mappings":";;;AAAA,iDAG0B;AAS1B,+BAQe;AAWf,MAAa,aAAa;IACb,UAAU,CAAS;IACX,EAAE,CAAwB;IAC1B,MAAM,CAAqB;IAE5C,YAAY,OAA6B;QACrC,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAA,+BAAyB,EAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAC7D,IAAI,CAAC,UAAU,GAAG,IAAA,2CAA8B,EAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAsB;QAC/B,+BAA+B;QAC/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,IAAA,sCAAyB,EACrB,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,UAAU,EACf,SAAS,IAAI,CAAC,KAAK,CAAC,EAAE,YAAY,CACrC,CAAC;QACN,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,IAAA,kBAAY,EAAC,IAAA,iBAAW,EAAC,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;YAC5E,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CACf;cACF,IAAI,CAAC,MAAM,CAAC,UAAU;;;EAGlC,IAAA,6BAAuB,EAAC;gBACtB,QAAQ,EAAE,KAAK,CAAC,MAAM;gBACtB,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,UAAU,CAAC;gBAC3B,SAAS,EAAE,SAAS;aACvB,CAAC;;;;;CAKD,CAAC,IAAI,EAAE,EACQ,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;gBACpB,IAAI,CAAC,KAAK,CAAC,EAAE;gBACb,IAAI,CAAC,KAAK,CAAC,UAAU;gBACrB,IAAA,qBAAe,EAAC,IAAI,CAAC,SAAS,CAAC;aAClC,CAAC,CACL,CAAC;QACN,CAAC;IACL,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,MAAgB,EAAE,UAAyB,EAAE;QACtD,IAAA,sCAAyB,EAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,iBAAiB,CAAC,CAAC;QACtE,MAAM,IAAI,GAAG,IAAA,mBAAa,EAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;YACb,OAAO,EAAE,CAAC;QACd,CAAC;QAED,MAAM,KAAK,GAAG,IAAA,2BAAqB,EAAC,OAAO,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;QACrD,MAAM,cAAc,GAAG,OAAO,CAAC,QAAQ,KAAK,SAAS;YACjD,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,yCAAyC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvH,MAAM,gBAAgB,GAAG,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9F,0DAA0D;QAC1D,iCAAiC;QACjC,mEAAmE;QACnE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAC9B;;;;;OAKL,IAAI,CAAC,MAAM,CAAC,UAAU;OACtB,IAAI,CAAC,MAAM,CAAC,MAAM;EACvB,KAAK,CAAC,MAAM;EACZ,cAAc;;QAER,gBAAgB;CACvB,CAAC,IAAI,EAAE,EACI;YACI,IAAA,qBAAe,EAAC,MAAM,CAAC;YACvB,GAAG,KAAK,CAAC,MAAM;YACf,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAC7D,IAAI;SACP,CACJ,CAAC;QAEF,OAAO,MAAM,CAAC,IAAI;aACb,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACX,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC7B,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC;YACnC,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC;YAC7B,QAAQ,EAAE;gBACN,MAAM,EAAE,QAAQ;aACnB;SACJ,CAAC,CAAC;aACF,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CACnB,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK;eACrB,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,KAAK,CAAC,UAAU,CAAC;eAC/C,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAC/C,CAAC,CAAC;IACX,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,UAAkB;QACrC,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CACf,eAAe,IAAI,CAAC,MAAM,CAAC,UAAU,yBAAyB,EAC9D,CAAC,UAAU,CAAC,CACf,CAAC;IACN,CAAC;CACJ;AAxGD,sCAwGC"}

package/dist/rag/stores/postgres/postgres-document-store.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"postgres-document-store.d.ts","sourceRoot":"","sources":["../../../../src/rag/stores/postgres/postgres-document-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,KAAK,EAIL,WAAW,EACd,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EACR,aAAa,EAChB,MAAM,UAAU,CAAC;AAYlB,OAAO,KAAK,EAER,uBAAuB,EAE1B,MAAM,SAAS,CAAC;AAEjB,qBAAa,qBAAsB,YAAW,aAAa;IACvD,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAwB;IAC3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;gBAEhC,OAAO,EAAE,uBAAuB;IAKtC,eAAe,CAAC,SAAS,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IA8CxD,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAsD5C,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAoB1C,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAczD,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOjD,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC;CAS1E"}
+{"version":3,"file":"postgres-document-store.d.ts","sourceRoot":"","sources":["../../../../src/rag/stores/postgres/postgres-document-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,KAAK,EAIL,WAAW,EACd,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EACR,aAAa,EAChB,MAAM,UAAU,CAAC;AAelB,OAAO,KAAK,EAER,uBAAuB,EAE1B,MAAM,SAAS,CAAC;AAWjB,qBAAa,qBAAsB,YAAW,aAAa;IACvD,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAwB;IAC3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;gBAEhC,OAAO,EAAE,uBAAuB;IAKtC,eAAe,CAAC,SAAS,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAiDxD,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAwD5C,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAoB1C,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAczD,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOjD,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC;CAS1E"}

package/dist/rag/stores/postgres/postgres-document-store.js

@@ -2,6 +2,14 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.PostgresDocumentStore = void 0;
 const sql_1 = require("./sql");
+// 逐行 INSERT 在大批量入库时网络往返开销是主要瓶颈，
+// 这里统一走 multi-row VALUES 批量插入，ON CONFLICT 语义与逐行版本一致。
+const DOCUMENT_COLUMN_CASTS = [
+    '', '', '::jsonb', '::jsonb', '', '', '', '', '', '::text[]', '', '', '', '',
+];
+const CHUNK_COLUMN_CASTS = [
+    '', '', '', '', '', '::jsonb', '', '', '', '', '::text[]', '', '', '', '', '::text[]', '::jsonb',
+];
 class PostgresDocumentStore {
     db;
     tables;
@@ -10,14 +18,17 @@ class PostgresDocumentStore {
         this.tables = (0, sql_1.resolvePostgresTableNames)(options.tablePrefix);
     }
     async upsertDocuments(documents) {
-        for (const document of documents) {
+        for (const batch of (0, sql_1.chunkBatches)((0, sql_1.dedupeByKey)(documents, (document) => document.id))) {
             await this.db.query(`
 INSERT INTO ${this.tables.documents} (
     id, content, metadata, blocks, source, title, mime_type, tenant_id, knowledge_base_id,
     acl, version, checksum, created_at_meta, updated_at_meta, updated_at
-) VALUES (
-    $1, $2, $3::jsonb, $4::jsonb, $5, $6, $7, $8, $9, $10::text[], $11, $12, $13, $14, now()
-)
+) VALUES
+${(0, sql_1.createMultiRowValuesSql)({
+                rowCount: batch.length,
+                casts: DOCUMENT_COLUMN_CASTS,
+                rowSuffix: ', now()',
+            })}
 ON CONFLICT (id) DO UPDATE SET
     content = EXCLUDED.content,
     metadata = EXCLUDED.metadata,
@@ -33,7 +44,7 @@ ON CONFLICT (id) DO UPDATE SET
     created_at_meta = EXCLUDED.created_at_meta,
     updated_at_meta = EXCLUDED.updated_at_meta,
     updated_at = now()
-`.trim(), [
+`.trim(), batch.flatMap((document) => [
                 document.id,
                 document.content,
                 (0, sql_1.createJson)(document.metadata),
@@ -48,20 +59,22 @@ ON CONFLICT (id) DO UPDATE SET
                 document.metadata.checksum ?? null,
                 document.metadata.createdAt ?? null,
                 document.metadata.updatedAt ?? null,
-            ]);
+            ]));
         }
     }
     async upsertChunks(chunks) {
-        for (const chunk of chunks) {
+        for (const batch of (0, sql_1.chunkBatches)((0, sql_1.dedupeByKey)(chunks, (chunk) => chunk.id))) {
             await this.db.query(`
 INSERT INTO ${this.tables.chunks} (
     id, document_id, parent_id, content, kind, metadata, source, title, tenant_id,
     knowledge_base_id, heading_path, page, start_offset, end_offset, chunk_index, acl, extra,
     updated_at
-) VALUES (
-    $1, $2, $3, $4, $5, $6::jsonb, $7, $8, $9, $10, $11::text[], $12, $13, $14, $15, $16::text[],
-    $17::jsonb, now()
-)
+) VALUES
+${(0, sql_1.createMultiRowValuesSql)({
+                rowCount: batch.length,
+                casts: CHUNK_COLUMN_CASTS,
+                rowSuffix: ', now()',
+            })}
 ON CONFLICT (id) DO UPDATE SET
     document_id = EXCLUDED.document_id,
     parent_id = EXCLUDED.parent_id,
@@ -80,7 +93,7 @@ ON CONFLICT (id) DO UPDATE SET
     acl = EXCLUDED.acl,
     extra = EXCLUDED.extra,
     updated_at = now()
-`.trim(), [
+`.trim(), batch.flatMap((chunk) => [
                 chunk.id,
                 chunk.documentId,
                 chunk.parentId ?? null,
@@ -98,7 +111,7 @@ ON CONFLICT (id) DO UPDATE SET
                 chunk.metadata.chunkIndex,
                 chunk.metadata.acl ?? null,
                 (0, sql_1.createOptionalJson)(chunk.metadata.extra),
-            ]);
+            ]));
         }
     }
     async getChunks(ids) {