@fllf/agent-sdk 0.1.1 → 0.1.3

package/dist/rag/stores/keyword-tokenizer.js

@@ -0,0 +1,113 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createKeywordIndexText = createKeywordIndexText;
+exports.tokenizeKeywordText = tokenizeKeywordText;
+const RAW_TOKEN_PATTERN = /[\p{L}\p{N}_-]+/gu;
+const CJK_CHARACTER_PATTERN = /[\u3400-\u4DBF\u4E00-\u9FFF\uF900-\uFAFF]/u;
+function createKeywordIndexText(chunk) {
+    return [
+        chunk.metadata.title,
+        chunk.metadata.headingPath?.join(' '),
+        chunk.content,
+    ]
+        .filter((part) => part !== undefined && part.trim().length > 0)
+        .join('\n');
+}
+function tokenizeKeywordText(text) {
+    const rawTokens = text.normalize('NFKC').match(RAW_TOKEN_PATTERN) ?? [];
+    const terms = [];
+    for (const rawToken of rawTokens) {
+        terms.push(...createTokenTerms(rawToken));
+    }
+    return terms;
+}
+function createTokenTerms(rawToken) {
+    const terms = new Set();
+    addTerm(terms, rawToken);
+    for (const part of splitMixedToken(rawToken)) {
+        addTerm(terms, part.value);
+        if (part.kind === 'cjk') {
+            addCjkNgrams(terms, part.value);
+        }
+        else {
+            addIdentifierTerms(terms, part.value);
+        }
+    }
+    return [...terms];
+}
+function splitMixedToken(token) {
+    const parts = [];
+    let current = '';
+    let currentKind;
+    for (const character of Array.from(token)) {
+        const kind = isCjkCharacter(character) ? 'cjk' : 'identifier';
+        if (currentKind !== undefined && currentKind !== kind) {
+            parts.push({
+                value: current,
+                kind: currentKind,
+            });
+            current = '';
+        }
+        current += character;
+        currentKind = kind;
+    }
+    if (current.length > 0 && currentKind !== undefined) {
+        parts.push({
+            value: current,
+            kind: currentKind,
+        });
+    }
+    return parts;
+}
+function addIdentifierTerms(terms, value) {
+    const separatorParts = value
+        .split(/[_-]+/u)
+        .filter((part) => part.length > 0);
+    for (const part of separatorParts) {
+        addTerm(terms, part);
+        addCaseSplitTerms(terms, part);
+    }
+}
+function addCaseSplitTerms(terms, value) {
+    const parts = splitIdentifierCase(value);
+    if (parts.length <= 1) {
+        return;
+    }
+    for (const part of parts) {
+        addTerm(terms, part);
+    }
+    for (let index = 0; index < parts.length - 1; index += 1) {
+        const current = parts[index];
+        const next = parts[index + 1];
+        if (current !== undefined && next !== undefined && isAcronym(next)) {
+            addTerm(terms, `${current}${next}`);
+        }
+    }
+}
+function splitIdentifierCase(value) {
+    return value.match(/[A-Z]?[a-z]+|[A-Z]+(?![a-z])|\d+/g) ?? [value];
+}
+function addCjkNgrams(terms, value) {
+    const characters = Array.from(value);
+    for (const size of [2, 3]) {
+        if (characters.length < size) {
+            continue;
+        }
+        for (let index = 0; index <= characters.length - size; index += 1) {
+            addTerm(terms, characters.slice(index, index + size).join(''));
+        }
+    }
+}
+function addTerm(terms, value) {
+    const normalized = value.toLowerCase().trim();
+    if (normalized.length > 0) {
+        terms.add(normalized);
+    }
+}
+function isCjkCharacter(character) {
+    return CJK_CHARACTER_PATTERN.test(character);
+}
+function isAcronym(value) {
+    return /^[A-Z0-9]{2,}$/u.test(value);
+}
+//# sourceMappingURL=keyword-tokenizer.js.map

package/dist/rag/stores/keyword-tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"keyword-tokenizer.js","sourceRoot":"","sources":["../../../src/rag/stores/keyword-tokenizer.ts"],"names":[],"mappings":";;AAKA,wDAQC;AAED,kDASC;AAtBD,MAAM,iBAAiB,GAAG,mBAAmB,CAAC;AAC9C,MAAM,qBAAqB,GAAG,4CAA4C,CAAC;AAE3E,SAAgB,sBAAsB,CAAC,KAAY;IAC/C,OAAO;QACH,KAAK,CAAC,QAAQ,CAAC,KAAK;QACpB,KAAK,CAAC,QAAQ,CAAC,WAAW,EAAE,IAAI,CAAC,GAAG,CAAC;QACrC,KAAK,CAAC,OAAO;KAChB;SACI,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC;SAC9E,IAAI,CAAC,IAAI,CAAC,CAAC;AACpB,CAAC;AAED,SAAgB,mBAAmB,CAAC,IAAY;IAC5C,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,iBAAiB,CAAC,IAAI,EAAE,CAAC;IACxE,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QAC/B,KAAK,CAAC,IAAI,CAAC,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC,CAAC;IAC9C,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,SAAS,gBAAgB,CAAC,QAAgB;IACtC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAEhC,OAAO,CAAC,KAAK,EAAE,QAAQ,CAAC,CAAC;IAEzB,KAAK,MAAM,IAAI,IAAI,eAAe,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC3C,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QAE3B,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;YACtB,YAAY,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QACpC,CAAC;aAAM,CAAC;YACJ,kBAAkB,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QAC1C,CAAC;IACL,CAAC;IAED,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC;AACtB,CAAC;AAED,SAAS,eAAe,CAAC,KAAa;IAClC,MAAM,KAAK,GAAyD,EAAE,CAAC;IACvE,IAAI,OAAO,GAAG,EAAE,CAAC;IACjB,IAAI,WAA6C,CAAC;IAElD,KAAK,MAAM,SAAS,IAAI,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACxC,MAAM,IAAI,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,YAAY,CAAC;QAC9D,IAAI,WAAW,KAAK,SAAS,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;YACpD,KAAK,CAAC,IAAI,CAAC;gBACP,KAAK,EAAE,OAAO;gBACd,IAAI,EAAE,WAAW;aACpB,CAAC,CAAC;YACH,OAAO,GAAG,EAAE,CAAC;QACjB,CAAC;QAED,OAAO,IAAI,SAAS,CAAC;QACrB,WAAW,GAAG,IAAI,CAAC;IACvB,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,KAAK,SAAS,EAAE,CAAC;QAClD,KAAK,CAAC,IAAI,CAAC;YACP,KAAK,EAAE,OAAO;YACd,IAAI,EAAE,WAAW;SACpB,CAAC,CAAC;IACP,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAkB,EAAE,KAAa;IACzD,MAAM,cAAc,GAAG,KAAK;SACvB,KAAK,CAAC,QAAQ,CAAC;SACf,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAEvC,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;QAChC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QACrB,iBAAiB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IACnC,CAAC;AACL,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAkB,EAAE,KAAa;IACxD,MAAM,KAAK,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;IACzC,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QACpB,OAAO;IACX,CAAC;IAED,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IACzB,CAAC;IAED,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACvD,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;QAC7B,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;QAC9B,IAAI,OAAO,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS,IAAI,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC;YACjE,OAAO,CAAC,KAAK,EAAE,GAAG,OAAO,GAAG,IAAI,EAAE,CAAC,CAAC;QACxC,CAAC;IACL,CAAC;AACL,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAa;IACtC,OAAO,KAAK,CAAC,KAAK,CAAC,mCAAmC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACvE,CAAC;AAED,SAAS,YAAY,CAAC,KAAkB,EAAE,KAAa;IACnD,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAErC,KAAK,MAAM,IAAI,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;QACxB,IAAI,UAAU,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;YAC3B,SAAS;QACb,CAAC;QAED,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,IAAI,UAAU,CAAC,MAAM,GAAG,IAAI,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;YAChE,OAAO,CAAC,KAAK,EAAE,UAAU,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QACnE,CAAC;IACL,CAAC;AACL,CAAC;AAED,SAAS,OAAO,CAAC,KAAkB,EAAE,KAAa;IAC9C,MAAM,UAAU,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC,IAAI,EAAE,CAAC;IAC9C,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACxB,KAAK,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IAC1B,CAAC;AACL,CAAC;AAED,SAAS,cAAc,CAAC,SAAiB;IACrC,OAAO,qBAAqB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;AACjD,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC5B,OAAO,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;AACzC,CAAC"}

package/dist/rag/stores/postgres/pg-vector-store.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pg-vector-store.d.ts","sourceRoot":"","sources":["../../../../src/rag/stores/postgres/pg-vector-store.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACR,aAAa,EAChB,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EACR,cAAc,EACd,aAAa,EACb,WAAW,EACd,MAAM,UAAU,CAAC;AAOlB,OAAO,KAAK,EAER,uBAAuB,EAE1B,MAAM,SAAS,CAAC;AAEjB,MAAM,WAAW,oBAAqB,SAAQ,uBAAuB;IACjE,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,qBAAa,aAAc,YAAW,WAAW;IAC7C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAwB;IAC3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;gBAEhC,OAAO,EAAE,oBAAoB;IAMnC,MAAM,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IA4B7C,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IA2ChF,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAM5D"}
+{"version":3,"file":"pg-vector-store.d.ts","sourceRoot":"","sources":["../../../../src/rag/stores/postgres/pg-vector-store.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACR,aAAa,EAChB,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EACR,cAAc,EACd,aAAa,EACb,WAAW,EACd,MAAM,UAAU,CAAC;AAUlB,OAAO,KAAK,EAER,uBAAuB,EAE1B,MAAM,SAAS,CAAC;AAEjB,MAAM,WAAW,oBAAqB,SAAQ,uBAAuB;IACjE,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,qBAAa,aAAc,YAAW,WAAW;IAC7C,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAwB;IAC3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;gBAEhC,OAAO,EAAE,oBAAoB;IAMnC,MAAM,CAAC,KAAK,EAAE,aAAa,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAmC7C,MAAM,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAoDhF,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAM5D"}

package/dist/rag/stores/postgres/pg-vector-store.js

@@ -13,23 +13,29 @@ class PgVectorStore {
         this.dimensions = (0, embeddings_1.assertValidEmbeddingDimensions)(options.dimensions);
     }
     async upsert(items) {
+        // 先整体校验维度再分批写入，避免批次中途失败留下半写状态。
         for (const item of items) {
             (0, embeddings_1.assertEmbeddingDimensions)(item.embedding, this.dimensions, `chunk ${item.chunk.id} embedding`);
+        }
+        for (const batch of (0, sql_1.chunkBatches)((0, sql_1.dedupeByKey)(items, (item) => item.chunk.id))) {
             await this.db.query(`
 INSERT INTO ${this.tables.embeddings} (
     chunk_id, document_id, embedding, updated_at
-) VALUES (
-    $1, $2, $3::vector, now()
-)
+) VALUES
+${(0, sql_1.createMultiRowValuesSql)({
+                rowCount: batch.length,
+                casts: ['', '', '::vector'],
+                rowSuffix: ', now()',
+            })}
 ON CONFLICT (chunk_id) DO UPDATE SET
     document_id = EXCLUDED.document_id,
     embedding = EXCLUDED.embedding,
     updated_at = now()
-`.trim(), [
+`.trim(), batch.flatMap((item) => [
                 item.chunk.id,
                 item.chunk.documentId,
                 (0, sql_1.toVectorLiteral)(item.embedding),
-            ]);
+            ]));
         }
     }
     async search(vector, options = {}) {
@@ -43,6 +49,9 @@ ON CONFLICT (chunk_id) DO UPDATE SET
             ? ''
             : `${where.clause.length === 0 ? 'WHERE' : 'AND'} 1 - (e.embedding <=> $1::vector) >= $${where.values.length + 2}`;
         const limitPlaceholder = `$${where.values.length + (options.minScore === undefined ? 2 : 3)}`;
+        // ORDER BY 只保留向量距离表达式：一旦追加 document_id / chunk_id 等次级排序键，
+        // planner 就无法走 HNSW 索引，会退化成全表扫描。
+        // 确定性 tie-break（同分按 documentId/chunkId）挪到 Node 侧对返回的 topK 结果做稳定排序。
         const result = await this.db.query(`
 SELECT
     e.chunk_id,
@@ -52,7 +61,7 @@ FROM ${this.tables.embeddings} e
 JOIN ${this.tables.chunks} c ON c.id = e.chunk_id
 ${where.clause}
 ${minScoreClause}
-ORDER BY e.embedding <=> $1::vector ASC, e.document_id ASC, e.chunk_id ASC
+ORDER BY e.embedding <=> $1::vector ASC
 LIMIT ${limitPlaceholder}
 `.trim(), [
             (0, sql_1.toVectorLiteral)(vector),
@@ -60,14 +69,18 @@ LIMIT ${limitPlaceholder}
             ...(options.minScore === undefined ? [] : [options.minScore]),
             topK,
         ]);
-        return result.rows.map((row) => ({
+        return result.rows
+            .map((row) => ({
             chunkId: String(row.chunk_id),
             documentId: String(row.document_id),
             score: Number(row.score ?? 0),
             metadata: {
                 source: 'vector',
             },
-        }));
+        }))
+            .sort((left, right) => (right.score - left.score
+            || left.documentId.localeCompare(right.documentId)
+            || left.chunkId.localeCompare(right.chunkId)));
     }
     async deleteByDocument(documentId) {
         await this.db.query(`DELETE FROM ${this.tables.embeddings} WHERE document_id = $1`, [documentId]);

package/dist/rag/stores/postgres/pg-vector-store.js.map

@@ -1 +1 @@
-{"version":3,"file":"pg-vector-store.js","sourceRoot":"","sources":["../../../../src/rag/stores/postgres/pg-vector-store.ts"],"names":[],"mappings":";;;AAAA,iDAG0B;AAS1B,+BAKe;AAWf,MAAa,aAAa;IACb,UAAU,CAAS;IACX,EAAE,CAAwB;IAC1B,MAAM,CAAqB;IAE5C,YAAY,OAA6B;QACrC,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAA,+BAAyB,EAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAC7D,IAAI,CAAC,UAAU,GAAG,IAAA,2CAA8B,EAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAsB;QAC/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,IAAA,sCAAyB,EACrB,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,UAAU,EACf,SAAS,IAAI,CAAC,KAAK,CAAC,EAAE,YAAY,CACrC,CAAC;YACF,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CACf;cACF,IAAI,CAAC,MAAM,CAAC,UAAU;;;;;;;;;CASnC,CAAC,IAAI,EAAE,EACQ;gBACI,IAAI,CAAC,KAAK,CAAC,EAAE;gBACb,IAAI,CAAC,KAAK,CAAC,UAAU;gBACrB,IAAA,qBAAe,EAAC,IAAI,CAAC,SAAS,CAAC;aAClC,CACJ,CAAC;QACN,CAAC;IACL,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,MAAgB,EAAE,UAAyB,EAAE;QACtD,IAAA,sCAAyB,EAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,iBAAiB,CAAC,CAAC;QACtE,MAAM,IAAI,GAAG,IAAA,mBAAa,EAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;YACb,OAAO,EAAE,CAAC;QACd,CAAC;QAED,MAAM,KAAK,GAAG,IAAA,2BAAqB,EAAC,OAAO,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;QACrD,MAAM,cAAc,GAAG,OAAO,CAAC,QAAQ,KAAK,SAAS;YACjD,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,yCAAyC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvH,MAAM,gBAAgB,GAAG,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9F,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAC9B;;;;;OAKL,IAAI,CAAC,MAAM,CAAC,UAAU;OACtB,IAAI,CAAC,MAAM,CAAC,MAAM;EACvB,KAAK,CAAC,MAAM;EACZ,cAAc;;QAER,gBAAgB;CACvB,CAAC,IAAI,EAAE,EACI;YACI,IAAA,qBAAe,EAAC,MAAM,CAAC;YACvB,GAAG,KAAK,CAAC,MAAM;YACf,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAC7D,IAAI;SACP,CACJ,CAAC;QAEF,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAC7B,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC7B,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC;YACnC,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC;YAC7B,QAAQ,EAAE;gBACN,MAAM,EAAE,QAAQ;aACnB;SACJ,CAAC,CAAC,CAAC;IACR,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,UAAkB;QACrC,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CACf,eAAe,IAAI,CAAC,MAAM,CAAC,UAAU,yBAAyB,EAC9D,CAAC,UAAU,CAAC,CACf,CAAC;IACN,CAAC;CACJ;AAxFD,sCAwFC"}
+{"version":3,"file":"pg-vector-store.js","sourceRoot":"","sources":["../../../../src/rag/stores/postgres/pg-vector-store.ts"],"names":[],"mappings":";;;AAAA,iDAG0B;AAS1B,+BAQe;AAWf,MAAa,aAAa;IACb,UAAU,CAAS;IACX,EAAE,CAAwB;IAC1B,MAAM,CAAqB;IAE5C,YAAY,OAA6B;QACrC,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAA,+BAAyB,EAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAC7D,IAAI,CAAC,UAAU,GAAG,IAAA,2CAA8B,EAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAsB;QAC/B,+BAA+B;QAC/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,IAAA,sCAAyB,EACrB,IAAI,CAAC,SAAS,EACd,IAAI,CAAC,UAAU,EACf,SAAS,IAAI,CAAC,KAAK,CAAC,EAAE,YAAY,CACrC,CAAC;QACN,CAAC;QAED,KAAK,MAAM,KAAK,IAAI,IAAA,kBAAY,EAAC,IAAA,iBAAW,EAAC,KAAK,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;YAC5E,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CACf;cACF,IAAI,CAAC,MAAM,CAAC,UAAU;;;EAGlC,IAAA,6BAAuB,EAAC;gBACtB,QAAQ,EAAE,KAAK,CAAC,MAAM;gBACtB,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,UAAU,CAAC;gBAC3B,SAAS,EAAE,SAAS;aACvB,CAAC;;;;;CAKD,CAAC,IAAI,EAAE,EACQ,KAAK,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC;gBACpB,IAAI,CAAC,KAAK,CAAC,EAAE;gBACb,IAAI,CAAC,KAAK,CAAC,UAAU;gBACrB,IAAA,qBAAe,EAAC,IAAI,CAAC,SAAS,CAAC;aAClC,CAAC,CACL,CAAC;QACN,CAAC;IACL,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,MAAgB,EAAE,UAAyB,EAAE;QACtD,IAAA,sCAAyB,EAAC,MAAM,EAAE,IAAI,CAAC,UAAU,EAAE,iBAAiB,CAAC,CAAC;QACtE,MAAM,IAAI,GAAG,IAAA,mBAAa,EAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;YACb,OAAO,EAAE,CAAC;QACd,CAAC;QAED,MAAM,KAAK,GAAG,IAAA,2BAAqB,EAAC,OAAO,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;QACrD,MAAM,cAAc,GAAG,OAAO,CAAC,QAAQ,KAAK,SAAS;YACjD,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,yCAAyC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvH,MAAM,gBAAgB,GAAG,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC9F,0DAA0D;QAC1D,iCAAiC;QACjC,mEAAmE;QACnE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAC9B;;;;;OAKL,IAAI,CAAC,MAAM,CAAC,UAAU;OACtB,IAAI,CAAC,MAAM,CAAC,MAAM;EACvB,KAAK,CAAC,MAAM;EACZ,cAAc;;QAER,gBAAgB;CACvB,CAAC,IAAI,EAAE,EACI;YACI,IAAA,qBAAe,EAAC,MAAM,CAAC;YACvB,GAAG,KAAK,CAAC,MAAM;YACf,GAAG,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAC7D,IAAI;SACP,CACJ,CAAC;QAEF,OAAO,MAAM,CAAC,IAAI;aACb,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACX,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC7B,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC;YACnC,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC;YAC7B,QAAQ,EAAE;gBACN,MAAM,EAAE,QAAQ;aACnB;SACJ,CAAC,CAAC;aACF,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CACnB,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK;eACrB,IAAI,CAAC,UAAU,CAAC,aAAa,CAAC,KAAK,CAAC,UAAU,CAAC;eAC/C,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAC/C,CAAC,CAAC;IACX,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,UAAkB;QACrC,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CACf,eAAe,IAAI,CAAC,MAAM,CAAC,UAAU,yBAAyB,EAC9D,CAAC,UAAU,CAAC,CACf,CAAC;IACN,CAAC;CACJ;AAxGD,sCAwGC"}

package/dist/rag/stores/postgres/postgres-document-store.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"postgres-document-store.d.ts","sourceRoot":"","sources":["../../../../src/rag/stores/postgres/postgres-document-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,KAAK,EAIL,WAAW,EACd,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EACR,aAAa,EAChB,MAAM,UAAU,CAAC;AAYlB,OAAO,KAAK,EAER,uBAAuB,EAE1B,MAAM,SAAS,CAAC;AAEjB,qBAAa,qBAAsB,YAAW,aAAa;IACvD,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAwB;IAC3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;gBAEhC,OAAO,EAAE,uBAAuB;IAKtC,eAAe,CAAC,SAAS,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IA8CxD,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAsD5C,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAoB1C,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAczD,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOjD,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC;CAS1E"}
+{"version":3,"file":"postgres-document-store.d.ts","sourceRoot":"","sources":["../../../../src/rag/stores/postgres/postgres-document-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,KAAK,EAIL,WAAW,EACd,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EACR,aAAa,EAChB,MAAM,UAAU,CAAC;AAelB,OAAO,KAAK,EAER,uBAAuB,EAE1B,MAAM,SAAS,CAAC;AAWjB,qBAAa,qBAAsB,YAAW,aAAa;IACvD,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAwB;IAC3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;gBAEhC,OAAO,EAAE,uBAAuB;IAKtC,eAAe,CAAC,SAAS,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAiDxD,YAAY,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAwD5C,SAAS,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAoB1C,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC;IAczD,cAAc,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAOjD,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,GAAG,SAAS,CAAC;CAS1E"}

package/dist/rag/stores/postgres/postgres-document-store.js

@@ -2,6 +2,14 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.PostgresDocumentStore = void 0;
 const sql_1 = require("./sql");
+// 逐行 INSERT 在大批量入库时网络往返开销是主要瓶颈，
+// 这里统一走 multi-row VALUES 批量插入，ON CONFLICT 语义与逐行版本一致。
+const DOCUMENT_COLUMN_CASTS = [
+    '', '', '::jsonb', '::jsonb', '', '', '', '', '', '::text[]', '', '', '', '',
+];
+const CHUNK_COLUMN_CASTS = [
+    '', '', '', '', '', '::jsonb', '', '', '', '', '::text[]', '', '', '', '', '::text[]', '::jsonb',
+];
 class PostgresDocumentStore {
     db;
     tables;
@@ -10,14 +18,17 @@ class PostgresDocumentStore {
         this.tables = (0, sql_1.resolvePostgresTableNames)(options.tablePrefix);
     }
     async upsertDocuments(documents) {
-        for (const document of documents) {
+        for (const batch of (0, sql_1.chunkBatches)((0, sql_1.dedupeByKey)(documents, (document) => document.id))) {
             await this.db.query(`
 INSERT INTO ${this.tables.documents} (
     id, content, metadata, blocks, source, title, mime_type, tenant_id, knowledge_base_id,
     acl, version, checksum, created_at_meta, updated_at_meta, updated_at
-) VALUES (
-    $1, $2, $3::jsonb, $4::jsonb, $5, $6, $7, $8, $9, $10::text[], $11, $12, $13, $14, now()
-)
+) VALUES
+${(0, sql_1.createMultiRowValuesSql)({
+                rowCount: batch.length,
+                casts: DOCUMENT_COLUMN_CASTS,
+                rowSuffix: ', now()',
+            })}
 ON CONFLICT (id) DO UPDATE SET
     content = EXCLUDED.content,
     metadata = EXCLUDED.metadata,
@@ -33,7 +44,7 @@ ON CONFLICT (id) DO UPDATE SET
     created_at_meta = EXCLUDED.created_at_meta,
     updated_at_meta = EXCLUDED.updated_at_meta,
     updated_at = now()
-`.trim(), [
+`.trim(), batch.flatMap((document) => [
                 document.id,
                 document.content,
                 (0, sql_1.createJson)(document.metadata),
@@ -48,20 +59,22 @@ ON CONFLICT (id) DO UPDATE SET
                 document.metadata.checksum ?? null,
                 document.metadata.createdAt ?? null,
                 document.metadata.updatedAt ?? null,
-            ]);
+            ]));
         }
     }
     async upsertChunks(chunks) {
-        for (const chunk of chunks) {
+        for (const batch of (0, sql_1.chunkBatches)((0, sql_1.dedupeByKey)(chunks, (chunk) => chunk.id))) {
             await this.db.query(`
 INSERT INTO ${this.tables.chunks} (
     id, document_id, parent_id, content, kind, metadata, source, title, tenant_id,
     knowledge_base_id, heading_path, page, start_offset, end_offset, chunk_index, acl, extra,
     updated_at
-) VALUES (
-    $1, $2, $3, $4, $5, $6::jsonb, $7, $8, $9, $10, $11::text[], $12, $13, $14, $15, $16::text[],
-    $17::jsonb, now()
-)
+) VALUES
+${(0, sql_1.createMultiRowValuesSql)({
+                rowCount: batch.length,
+                casts: CHUNK_COLUMN_CASTS,
+                rowSuffix: ', now()',
+            })}
 ON CONFLICT (id) DO UPDATE SET
     document_id = EXCLUDED.document_id,
     parent_id = EXCLUDED.parent_id,
@@ -80,7 +93,7 @@ ON CONFLICT (id) DO UPDATE SET
     acl = EXCLUDED.acl,
     extra = EXCLUDED.extra,
     updated_at = now()
-`.trim(), [
+`.trim(), batch.flatMap((chunk) => [
                 chunk.id,
                 chunk.documentId,
                 chunk.parentId ?? null,
@@ -98,7 +111,7 @@ ON CONFLICT (id) DO UPDATE SET
                 chunk.metadata.chunkIndex,
                 chunk.metadata.acl ?? null,
                 (0, sql_1.createOptionalJson)(chunk.metadata.extra),
-            ]);
+            ]));
         }
     }
     async getChunks(ids) {

package/dist/rag/stores/postgres/postgres-document-store.js.map

@@ -1 +1 @@
-{"version":3,"file":"postgres-document-store.js","sourceRoot":"","sources":["../../../../src/rag/stores/postgres/postgres-document-store.ts"],"names":[],"mappings":";;;AAUA,+BAUe;AAOf,MAAa,qBAAqB;IACb,EAAE,CAAwB;IAC1B,MAAM,CAAqB;IAE5C,YAAY,OAAgC;QACxC,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAA,+BAAyB,EAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,SAAwB;QAC1C,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;YAC/B,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CACf;cACF,IAAI,CAAC,MAAM,CAAC,SAAS;;;;;;;;;;;;;;;;;;;;;CAqBlC,CAAC,IAAI,EAAE,EACQ;gBACI,QAAQ,CAAC,EAAE;gBACX,QAAQ,CAAC,OAAO;gBAChB,IAAA,gBAAU,EAAC,QAAQ,CAAC,QAAQ,CAAC;gBAC7B,IAAA,wBAAkB,EAAC,QAAQ,CAAC,MAAM,CAAC;gBACnC,QAAQ,CAAC,QAAQ,CAAC,MAAM;gBACxB,QAAQ,CAAC,QAAQ,CAAC,KAAK,IAAI,IAAI;gBAC/B,QAAQ,CAAC,QAAQ,CAAC,QAAQ,IAAI,IAAI;gBAClC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,IAAI,IAAI;gBAClC,QAAQ,CAAC,QAAQ,CAAC,eAAe,IAAI,IAAI;gBACzC,QAAQ,CAAC,QAAQ,CAAC,GAAG,IAAI,IAAI;gBAC7B,QAAQ,CAAC,QAAQ,CAAC,OAAO,IAAI,IAAI;gBACjC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,IAAI,IAAI;gBAClC,QAAQ,CAAC,QAAQ,CAAC,SAAS,IAAI,IAAI;gBACnC,QAAQ,CAAC,QAAQ,CAAC,SAAS,IAAI,IAAI;aACtC,CACJ,CAAC;QACN,CAAC;IACL,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,MAAe;QAC9B,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACzB,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CACf;cACF,IAAI,CAAC,MAAM,CAAC,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;CA0B/B,CAAC,IAAI,EAAE,EACQ;gBACI,KAAK,CAAC,EAAE;gBACR,KAAK,CAAC,UAAU;gBAChB,KAAK,CAAC,QAAQ,IAAI,IAAI;gBACtB,KAAK,CAAC,OAAO;gBACb,KAAK,CAAC,IAAI;gBACV,IAAA,gBAAU,EAAC,KAAK,CAAC,QAAQ,CAAC;gBAC1B,KAAK,CAAC,QAAQ,CAAC,MAAM;gBACrB,KAAK,CAAC,QAAQ,CAAC,KAAK,IAAI,IAAI;gBAC5B,KAAK,CAAC,QAAQ,CAAC,QAAQ,IAAI,IAAI;gBAC/B,KAAK,CAAC,QAAQ,CAAC,eAAe,IAAI,IAAI;gBACtC,KAAK,CAAC,QAAQ,CAAC,WAAW,IAAI,IAAI;gBAClC,KAAK,CAAC,QAAQ,CAAC,IAAI,IAAI,IAAI;gBAC3B,KAAK,CAAC,QAAQ,CAAC,WAAW,IAAI,IAAI;gBAClC,KAAK,CAAC,QAAQ,CAAC,SAAS,IAAI,IAAI;gBAChC,KAAK,CAAC,QAAQ,CAAC,UAAU;gBACzB,KAAK,CAAC,QAAQ,CAAC,GAAG,IAAI,IAAI;gBAC1B,IAAA,wBAAkB,EAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC;aAC3C,CACJ,CAAC;QACN,CAAC;IACL,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,GAAa;QACzB,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnB,OAAO,EAAE,CAAC;QACd,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAC9B;;OAEL,IAAI,CAAC,MAAM,CAAC,MAAM;;CAExB,CAAC,IAAI,EAAE,EACI,CAAC,GAAG,CAAC,CACR,CAAC;QACF,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAExF,OAAO,GAAG;aACL,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;aAC/B,MAAM,CAAC,CAAC,KAAK,EAAkB,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IAChE,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,UAAkB;QACxC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAC9B;;OAEL,IAAI,CAAC,MAAM,CAAC,MAAM;;;CAGxB,CAAC,IAAI,EAAE,EACI,CAAC,UAAU,CAAC,CACf,CAAC;QAEF,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,UAAkB;QACnC,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CACf,eAAe,IAAI,CAAC,MAAM,CAAC,SAAS,gBAAgB,EACpD,CAAC,UAAU,CAAC,CACf,CAAC;IACN,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,UAAkB;QAChC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAC9B,iBAAiB,IAAI,CAAC,MAAM,CAAC,SAAS,gBAAgB,EACtD,CAAC,UAAU,CAAC,CACf,CAAC;QACF,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAE3B,OAAO,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;IACjE,CAAC;CACJ;AA/JD,sDA+JC;AAED,SAAS,gBAAgB,CAAC,GAA4B;IAClD,MAAM,QAAQ,GAAG,IAAA,oBAAc,EAAC,GAAG,CAAC,QAAQ,CAA0C,CAAC;IACvF,MAAM,gBAAgB,GAAqB;QACvC,MAAM,EAAE,IAAA,wBAAkB,EAAC,GAAG,CAAC,MAAM,CAAC,IAAI,QAAQ,EAAE,MAAM,IAAI,QAAQ;KACzE,CAAC;IAEF,IAAA,qBAAe,EAAC,gBAAgB,EAAE,OAAO,EAAE,IAAA,wBAAkB,EAAC,GAAG,CAAC,KAAK,CAAC,IAAI,QAAQ,EAAE,KAAK,CAAC,CAAC;IAC7F,IAAA,qBAAe,EAAC,gBAAgB,EAAE,UAAU,EAAE,IAAA,wBAAkB,EAAC,GAAG,CAAC,SAAS,CAAC,IAAI,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACvG,IAAA,qBAAe,EAAC,gBAAgB,EAAE,UAAU,EAAE,IAAA,wBAAkB,EAAC,GAAG,CAAC,SAAS,CAAC,IAAI,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACvG,IAAA,qBAAe,EACX,gBAAgB,EAChB,iBAAiB,EACjB,IAAA,wBAAkB,EAAC,GAAG,CAAC,iBAAiB,CAAC,IAAI,QAAQ,EAAE,eAAe,CACzE,CAAC;IACF,IAAA,qBAAe,EAAC,gBAAgB,EAAE,SAAS,EAAE,IAAA,wBAAkB,EAAC,GAAG,CAAC,OAAO,CAAC,IAAI,QAAQ,EAAE,OAAO,CAAC,CAAC;IACnG,IAAA,qBAAe,EAAC,gBAAgB,EAAE,UAAU,EAAE,IAAA,wBAAkB,EAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACtG,IAAA,qBAAe,EACX,gBAAgB,EAChB,WAAW,EACX,IAAA,wBAAkB,EAAC,GAAG,CAAC,eAAe,CAAC,IAAI,QAAQ,EAAE,SAAS,CACjE,CAAC;IACF,IAAA,qBAAe,EACX,gBAAgB,EAChB,WAAW,EACX,IAAA,wBAAkB,EAAC,GAAG,CAAC,eAAe,CAAC,IAAI,QAAQ,EAAE,SAAS,CACjE,CAAC;IACF,IAAA,qBAAe,EAAC,gBAAgB,EAAE,KAAK,EAAE,IAAA,6BAAuB,EAAC,GAAG,CAAC,GAAG,CAAC,IAAI,QAAQ,EAAE,GAAG,CAAC,CAAC;IAC5F,IAAA,qBAAe,EAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IAE5D,MAAM,QAAQ,GAAgB;QAC1B,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QAClB,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;QAClC,QAAQ,EAAE,gBAAgB;KAC7B,CAAC;IACF,MAAM,MAAM,GAAG,IAAA,mBAAa,EAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACzC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACvB,QAAQ,CAAC,MAAM,GAAG,MAAyB,CAAC;IAChD,CAAC;IAED,OAAO,QAAQ,CAAC;AACpB,CAAC;AAED,SAAS,UAAU,CAAC,GAA4B;IAC5C,MAAM,KAAK,GAAU;QACjB,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC;QACnC,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;QAClC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAc;QACnC,QAAQ,EAAE,IAAA,wBAAkB,EAAC,GAAG,CAAC;KACpC,CAAC;IAEF,IAAA,qBAAe,EAAC,KAAK,EAAE,UAAU,EAAE,IAAA,wBAAkB,EAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;IAEtE,OAAO,KAAK,CAAC;AACjB,CAAC"}
+{"version":3,"file":"postgres-document-store.js","sourceRoot":"","sources":["../../../../src/rag/stores/postgres/postgres-document-store.ts"],"names":[],"mappings":";;;AAUA,+BAae;AAOf,gCAAgC;AAChC,qDAAqD;AACrD,MAAM,qBAAqB,GAAG;IAC1B,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE,SAAS,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE;CACtE,CAAC;AACX,MAAM,kBAAkB,GAAG;IACvB,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,UAAU,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,EAAE,UAAU,EAAE,SAAS;CAC1F,CAAC;AAEX,MAAa,qBAAqB;IACb,EAAE,CAAwB;IAC1B,MAAM,CAAqB;IAE5C,YAAY,OAAgC;QACxC,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAA,+BAAyB,EAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,SAAwB;QAC1C,KAAK,MAAM,KAAK,IAAI,IAAA,kBAAY,EAAC,IAAA,iBAAW,EAAC,SAAS,EAAE,CAAC,QAAQ,EAAE,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;YAClF,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CACf;cACF,IAAI,CAAC,MAAM,CAAC,SAAS;;;;EAIjC,IAAA,6BAAuB,EAAC;gBACtB,QAAQ,EAAE,KAAK,CAAC,MAAM;gBACtB,KAAK,EAAE,qBAAqB;gBAC5B,SAAS,EAAE,SAAS;aACvB,CAAC;;;;;;;;;;;;;;;;CAgBD,CAAC,IAAI,EAAE,EACQ,KAAK,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC;gBACxB,QAAQ,CAAC,EAAE;gBACX,QAAQ,CAAC,OAAO;gBAChB,IAAA,gBAAU,EAAC,QAAQ,CAAC,QAAQ,CAAC;gBAC7B,IAAA,wBAAkB,EAAC,QAAQ,CAAC,MAAM,CAAC;gBACnC,QAAQ,CAAC,QAAQ,CAAC,MAAM;gBACxB,QAAQ,CAAC,QAAQ,CAAC,KAAK,IAAI,IAAI;gBAC/B,QAAQ,CAAC,QAAQ,CAAC,QAAQ,IAAI,IAAI;gBAClC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,IAAI,IAAI;gBAClC,QAAQ,CAAC,QAAQ,CAAC,eAAe,IAAI,IAAI;gBACzC,QAAQ,CAAC,QAAQ,CAAC,GAAG,IAAI,IAAI;gBAC7B,QAAQ,CAAC,QAAQ,CAAC,OAAO,IAAI,IAAI;gBACjC,QAAQ,CAAC,QAAQ,CAAC,QAAQ,IAAI,IAAI;gBAClC,QAAQ,CAAC,QAAQ,CAAC,SAAS,IAAI,IAAI;gBACnC,QAAQ,CAAC,QAAQ,CAAC,SAAS,IAAI,IAAI;aACtC,CAAC,CACL,CAAC;QACN,CAAC;IACL,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,MAAe;QAC9B,KAAK,MAAM,KAAK,IAAI,IAAA,kBAAY,EAAC,IAAA,iBAAW,EAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;YACzE,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CACf;cACF,IAAI,CAAC,MAAM,CAAC,MAAM;;;;;EAK9B,IAAA,6BAAuB,EAAC;gBACtB,QAAQ,EAAE,KAAK,CAAC,MAAM;gBACtB,KAAK,EAAE,kBAAkB;gBACzB,SAAS,EAAE,SAAS;aACvB,CAAC;;;;;;;;;;;;;;;;;;;CAmBD,CAAC,IAAI,EAAE,EACQ,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC;gBACrB,KAAK,CAAC,EAAE;gBACR,KAAK,CAAC,UAAU;gBAChB,KAAK,CAAC,QAAQ,IAAI,IAAI;gBACtB,KAAK,CAAC,OAAO;gBACb,KAAK,CAAC,IAAI;gBACV,IAAA,gBAAU,EAAC,KAAK,CAAC,QAAQ,CAAC;gBAC1B,KAAK,CAAC,QAAQ,CAAC,MAAM;gBACrB,KAAK,CAAC,QAAQ,CAAC,KAAK,IAAI,IAAI;gBAC5B,KAAK,CAAC,QAAQ,CAAC,QAAQ,IAAI,IAAI;gBAC/B,KAAK,CAAC,QAAQ,CAAC,eAAe,IAAI,IAAI;gBACtC,KAAK,CAAC,QAAQ,CAAC,WAAW,IAAI,IAAI;gBAClC,KAAK,CAAC,QAAQ,CAAC,IAAI,IAAI,IAAI;gBAC3B,KAAK,CAAC,QAAQ,CAAC,WAAW,IAAI,IAAI;gBAClC,KAAK,CAAC,QAAQ,CAAC,SAAS,IAAI,IAAI;gBAChC,KAAK,CAAC,QAAQ,CAAC,UAAU;gBACzB,KAAK,CAAC,QAAQ,CAAC,GAAG,IAAI,IAAI;gBAC1B,IAAA,wBAAkB,EAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC;aAC3C,CAAC,CACL,CAAC;QACN,CAAC;IACL,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,GAAa;QACzB,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnB,OAAO,EAAE,CAAC;QACd,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAC9B;;OAEL,IAAI,CAAC,MAAM,CAAC,MAAM;;CAExB,CAAC,IAAI,EAAE,EACI,CAAC,GAAG,CAAC,CACR,CAAC;QACF,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAExF,OAAO,GAAG;aACL,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;aAC/B,MAAM,CAAC,CAAC,KAAK,EAAkB,EAAE,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC;IAChE,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,UAAkB;QACxC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAC9B;;OAEL,IAAI,CAAC,MAAM,CAAC,MAAM;;;CAGxB,CAAC,IAAI,EAAE,EACI,CAAC,UAAU,CAAC,CACf,CAAC;QAEF,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,UAAkB;QACnC,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CACf,eAAe,IAAI,CAAC,MAAM,CAAC,SAAS,gBAAgB,EACpD,CAAC,UAAU,CAAC,CACf,CAAC;IACN,CAAC;IAED,KAAK,CAAC,WAAW,CAAC,UAAkB;QAChC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAC9B,iBAAiB,IAAI,CAAC,MAAM,CAAC,SAAS,gBAAgB,EACtD,CAAC,UAAU,CAAC,CACf,CAAC;QACF,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAE3B,OAAO,GAAG,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC;IACjE,CAAC;CACJ;AApKD,sDAoKC;AAED,SAAS,gBAAgB,CAAC,GAA4B;IAClD,MAAM,QAAQ,GAAG,IAAA,oBAAc,EAAC,GAAG,CAAC,QAAQ,CAA0C,CAAC;IACvF,MAAM,gBAAgB,GAAqB;QACvC,MAAM,EAAE,IAAA,wBAAkB,EAAC,GAAG,CAAC,MAAM,CAAC,IAAI,QAAQ,EAAE,MAAM,IAAI,QAAQ;KACzE,CAAC;IAEF,IAAA,qBAAe,EAAC,gBAAgB,EAAE,OAAO,EAAE,IAAA,wBAAkB,EAAC,GAAG,CAAC,KAAK,CAAC,IAAI,QAAQ,EAAE,KAAK,CAAC,CAAC;IAC7F,IAAA,qBAAe,EAAC,gBAAgB,EAAE,UAAU,EAAE,IAAA,wBAAkB,EAAC,GAAG,CAAC,SAAS,CAAC,IAAI,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACvG,IAAA,qBAAe,EAAC,gBAAgB,EAAE,UAAU,EAAE,IAAA,wBAAkB,EAAC,GAAG,CAAC,SAAS,CAAC,IAAI,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACvG,IAAA,qBAAe,EACX,gBAAgB,EAChB,iBAAiB,EACjB,IAAA,wBAAkB,EAAC,GAAG,CAAC,iBAAiB,CAAC,IAAI,QAAQ,EAAE,eAAe,CACzE,CAAC;IACF,IAAA,qBAAe,EAAC,gBAAgB,EAAE,SAAS,EAAE,IAAA,wBAAkB,EAAC,GAAG,CAAC,OAAO,CAAC,IAAI,QAAQ,EAAE,OAAO,CAAC,CAAC;IACnG,IAAA,qBAAe,EAAC,gBAAgB,EAAE,UAAU,EAAE,IAAA,wBAAkB,EAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,QAAQ,EAAE,QAAQ,CAAC,CAAC;IACtG,IAAA,qBAAe,EACX,gBAAgB,EAChB,WAAW,EACX,IAAA,wBAAkB,EAAC,GAAG,CAAC,eAAe,CAAC,IAAI,QAAQ,EAAE,SAAS,CACjE,CAAC;IACF,IAAA,qBAAe,EACX,gBAAgB,EAChB,WAAW,EACX,IAAA,wBAAkB,EAAC,GAAG,CAAC,eAAe,CAAC,IAAI,QAAQ,EAAE,SAAS,CACjE,CAAC;IACF,IAAA,qBAAe,EAAC,gBAAgB,EAAE,KAAK,EAAE,IAAA,6BAAuB,EAAC,GAAG,CAAC,GAAG,CAAC,IAAI,QAAQ,EAAE,GAAG,CAAC,CAAC;IAC5F,IAAA,qBAAe,EAAC,gBAAgB,EAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,CAAC;IAE5D,MAAM,QAAQ,GAAgB;QAC1B,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QAClB,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;QAClC,QAAQ,EAAE,gBAAgB;KAC7B,CAAC;IACF,MAAM,MAAM,GAAG,IAAA,mBAAa,EAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACzC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACvB,QAAQ,CAAC,MAAM,GAAG,MAAyB,CAAC;IAChD,CAAC;IAED,OAAO,QAAQ,CAAC;AACpB,CAAC;AAED,SAAS,UAAU,CAAC,GAA4B;IAC5C,MAAM,KAAK,GAAU;QACjB,EAAE,EAAE,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QAClB,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC;QACnC,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;QAClC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,IAAI,CAAc;QACnC,QAAQ,EAAE,IAAA,wBAAkB,EAAC,GAAG,CAAC;KACpC,CAAC;IAEF,IAAA,qBAAe,EAAC,KAAK,EAAE,UAAU,EAAE,IAAA,wBAAkB,EAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC;IAEtE,OAAO,KAAK,CAAC;AACjB,CAAC"}

package/dist/rag/stores/postgres/postgres-keyword-store.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"postgres-keyword-store.d.ts","sourceRoot":"","sources":["../../../../src/rag/stores/postgres/postgres-keyword-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,KAAK,EACR,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EACR,YAAY,EACZ,cAAc,EACd,aAAa,EAChB,MAAM,UAAU,CAAC;AAQlB,OAAO,KAAK,EAER,uBAAuB,EAE1B,MAAM,SAAS,CAAC;AAUjB,qBAAa,oBAAqB,YAAW,YAAY;IACrD,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAwB;IAC3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;gBAEhC,OAAO,EAAE,uBAAuB;IAKtC,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IA4BtC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAwC7E,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;YAO3C,mBAAmB;YA8BnB,0BAA0B;YAuB1B,YAAY;CAQ7B"}
+{"version":3,"file":"postgres-keyword-store.d.ts","sourceRoot":"","sources":["../../../../src/rag/stores/postgres/postgres-keyword-store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,KAAK,EACR,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EACR,YAAY,EACZ,cAAc,EACd,aAAa,EAChB,MAAM,UAAU,CAAC;AAWlB,OAAO,KAAK,EAER,uBAAuB,EAE1B,MAAM,SAAS,CAAC;AAcjB,qBAAa,oBAAqB,YAAW,YAAY;IACrD,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAwB;IAC3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;gBAEhC,OAAO,EAAE,uBAAuB;IAKtC,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAkCtC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,GAAE,aAAkB,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IAwC7E,gBAAgB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;YAO3C,mBAAmB;YA8BnB,0BAA0B;YAuB1B,YAAY;CAQ7B"}

package/dist/rag/stores/postgres/postgres-keyword-store.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.PostgresKeywordStore = void 0;
 const sql_1 = require("./sql");
+const keyword_tokenizer_1 = require("../keyword-tokenizer");
 class PostgresKeywordStore {
     db;
     tables;
@@ -10,31 +11,36 @@ class PostgresKeywordStore {
         this.tables = (0, sql_1.resolvePostgresTableNames)(options.tablePrefix);
     }
     async upsert(chunks) {
-        for (const chunk of chunks) {
-            const terms = tokenize(chunk.content);
-            await this.db.query(`
-INSERT INTO ${this.tables.keywordEntries} (
-    chunk_id, document_id, terms, term_counts, length, updated_at
-) VALUES (
-    $1, $2, $3::text[], $4::jsonb, $5, now()
-)
-ON CONFLICT (chunk_id) DO UPDATE SET
-    document_id = EXCLUDED.document_id,
-    terms = EXCLUDED.terms,
-    term_counts = EXCLUDED.term_counts,
-    length = EXCLUDED.length,
-    updated_at = now()
-`.trim(), [
-                chunk.id,
-                chunk.documentId,
-                [...new Set(terms)],
-                (0, sql_1.createJson)(Object.fromEntries(countTerms(terms))),
-                terms.length,
-            ]);
+        for (const batch of (0, sql_1.chunkBatches)((0, sql_1.dedupeByKey)(chunks, (chunk) => chunk.id))) {
+            await this.db.query(`
+INSERT INTO ${this.tables.keywordEntries} (
+    chunk_id, document_id, terms, term_counts, length, updated_at
+) VALUES
+${(0, sql_1.createMultiRowValuesSql)({
+                rowCount: batch.length,
+                casts: ['', '', '::text[]', '::jsonb', ''],
+                rowSuffix: ', now()',
+            })}
+ON CONFLICT (chunk_id) DO UPDATE SET
+    document_id = EXCLUDED.document_id,
+    terms = EXCLUDED.terms,
+    term_counts = EXCLUDED.term_counts,
+    length = EXCLUDED.length,
+    updated_at = now()
+`.trim(), batch.flatMap((chunk) => {
+                const terms = (0, keyword_tokenizer_1.tokenizeKeywordText)((0, keyword_tokenizer_1.createKeywordIndexText)(chunk));
+                return [
+                    chunk.id,
+                    chunk.documentId,
+                    [...new Set(terms)],
+                    (0, sql_1.createJson)(Object.fromEntries(countTerms(terms))),
+                    terms.length,
+                ];
+            }));
         }
     }
     async search(query, options = {}) {
-        const queryTerms = [...new Set(tokenize(query))];
+        const queryTerms = [...new Set((0, keyword_tokenizer_1.tokenizeKeywordText)(query))];
         const topK = (0, sql_1.normalizeTopK)(options.topK);
         if (queryTerms.length === 0 || topK === 0) {
             return [];
@@ -73,17 +79,17 @@ ON CONFLICT (chunk_id) DO UPDATE SET
     }
     async loadMatchingEntries(queryTerms, options) {
         const where = (0, sql_1.buildChunkSearchWhere)(options, 2, 'c');
-        const result = await this.db.query(`
-SELECT
-    k.chunk_id,
-    k.document_id,
-    k.term_counts,
-    k.length,
-    c.content
-FROM ${this.tables.keywordEntries} k
-JOIN ${this.tables.chunks} c ON c.id = k.chunk_id
-WHERE k.terms && $1::text[]
-${where.clause.length > 0 ? `AND ${where.clause.slice('WHERE '.length)}` : ''}
+        const result = await this.db.query(`
+SELECT
+    k.chunk_id,
+    k.document_id,
+    k.term_counts,
+    k.length,
+    c.content
+FROM ${this.tables.keywordEntries} k
+JOIN ${this.tables.chunks} c ON c.id = k.chunk_id
+WHERE k.terms && $1::text[]
+${where.clause.length > 0 ? `AND ${where.clause.slice('WHERE '.length)}` : ''}
 `.trim(), [queryTerms, ...where.values]);
         return result.rows.map((row) => ({
             chunkId: String(row.chunk_id),
@@ -94,10 +100,10 @@ ${where.clause.length > 0 ? `AND ${where.clause.slice('WHERE '.length)}` : ''}
         }));
     }
     async calculateDocumentFrequency(queryTerms) {
-        const result = await this.db.query(`
-SELECT terms
-FROM ${this.tables.keywordEntries}
-WHERE terms && $1::text[]
+        const result = await this.db.query(`
+SELECT terms
+FROM ${this.tables.keywordEntries}
+WHERE terms && $1::text[]
 `.trim(), [queryTerms]);
         const frequency = new Map(queryTerms.map((term) => [term, 0]));
         for (const row of result.rows) {
@@ -131,12 +137,6 @@ function scoreEntry(input) {
     }
     return score;
 }
-function tokenize(text) {
-    return text
-        .normalize('NFKC')
-        .toLowerCase()
-        .match(/[\p{L}\p{N}_-]+/gu) ?? [];
-}
 function countTerms(terms) {
     const counts = new Map();
     for (const term of terms) {

package/dist/rag/stores/postgres/postgres-keyword-store.js.map

@@ -1 +1 @@
-{"version":3,"file":"postgres-keyword-store.js","sourceRoot":"","sources":["../../../../src/rag/stores/postgres/postgres-keyword-store.ts"],"names":[],"mappings":";;;AAQA,+BAMe;AAef,MAAa,oBAAoB;IACZ,EAAE,CAAwB;IAC1B,MAAM,CAAqB;IAE5C,YAAY,OAAgC;QACxC,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAA,+BAAyB,EAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,MAAe;QACxB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YACzB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACtC,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CACf;cACF,IAAI,CAAC,MAAM,CAAC,cAAc;;;;;;;;;;;CAWvC,CAAC,IAAI,EAAE,EACQ;gBACI,KAAK,CAAC,EAAE;gBACR,KAAK,CAAC,UAAU;gBAChB,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;gBACnB,IAAA,gBAAU,EAAC,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;gBACjD,KAAK,CAAC,MAAM;aACf,CACJ,CAAC;QACN,CAAC;IACL,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,UAAyB,EAAE;QACnD,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACjD,MAAM,IAAI,GAAG,IAAA,mBAAa,EAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;YACxC,OAAO,EAAE,CAAC;QACd,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACpE,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,0BAA0B,CAAC,UAAU,CAAC,CAAC;QAC5E,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QACjD,MAAM,IAAI,GAAG,OAAO;aACf,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACb,KAAK;YACL,KAAK,EAAE,UAAU,CAAC;gBACd,KAAK;gBACL,UAAU;gBACV,iBAAiB;gBACjB,cAAc;aACjB,CAAC;SACL,CAAC,CAAC;aACF,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;aAChC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC,QAAQ,CAAC;aAClF,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CACnB,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK;eACrB,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC;eAC3D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAC3D,CAAC;aACD,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAEpB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACvB,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO;YAC3B,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU;YACjC,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,QAAQ,EAAE;gBACN,MAAM,EAAE,SAAS;gBACjB,YAAY,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;aAC7E;SACJ,CAAC,CAAC,CAAC;IACR,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,UAAkB;QACrC,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CACf,eAAe,IAAI,CAAC,MAAM,CAAC,cAAc,yBAAyB,EAClE,CAAC,UAAU,CAAC,CACf,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAC7B,UAAoB,EACpB,OAAsB;QAEtB,MAAM,KAAK,GAAG,IAAA,2BAAqB,EAAC,OAAO,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;QACrD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAC9B;;;;;;;OAOL,IAAI,CAAC,MAAM,CAAC,cAAc;OAC1B,IAAI,CAAC,MAAM,CAAC,MAAM;;EAEvB,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;CAC5E,CAAC,IAAI,EAAE,EACI,CAAC,UAAU,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,CAChC,CAAC;QAEF,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAC7B,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC7B,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC;YACnC,UAAU,EAAE,gBAAgB,CAAC,GAAG,CAAC,WAAW,CAAC;YAC7C,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,CAAC;YAC/B,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;SACrC,CAAC,CAAC,CAAC;IACR,CAAC;IAEO,KAAK,CAAC,0BAA0B,CAAC,UAAoB;QACzD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAC9B;;OAEL,IAAI,CAAC,MAAM,CAAC,cAAc;;CAEhC,CAAC,IAAI,EAAE,EACI,CAAC,UAAU,CAAC,CACf,CAAC;QACF,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAE/D,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAC5B,MAAM,KAAK,GAAG,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACzC,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;gBAC5B,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;oBACvB,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBACxD,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAEO,KAAK,CAAC,YAAY;QACtB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAC9B,iCAAiC,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAChE,CAAC;QACF,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;QAEjD,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAC9B,CAAC;CACJ;AAjJD,oDAiJC;AAED,SAAS,UAAU,CAAC,KAKnB;IACG,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;QAClC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxD,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;YAClB,SAAS;QACb,CAAC;QAED,MAAM,EAAE,GAAG,KAAK,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,GAAG,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;QACzE,MAAM,YAAY,GAAG,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;QAC5E,KAAK,IAAI,YAAY,GAAG,GAAG,CAAC;IAChC,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY;IAC1B,OAAO,IAAI;SACN,SAAS,CAAC,MAAM,CAAC;SACjB,WAAW,EAAE;SACb,KAAK,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC;AAC1C,CAAC;AAED,SAAS,UAAU,CAAC,KAAe;IAC/B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACpC,MAAM,MAAM,GAAG,IAAA,oBAAc,EAAC,KAAK,CAAC,IAAI,EAAE,CAAC;IAC3C,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACjD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC5B,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC5B,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACnC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,CAAC;IACd,CAAC;IAED,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC;AAC5E,CAAC"}
+{"version":3,"file":"postgres-keyword-store.js","sourceRoot":"","sources":["../../../../src/rag/stores/postgres/postgres-keyword-store.ts"],"names":[],"mappings":";;;AAQA,+BASe;AAMf,4DAG8B;AAU9B,MAAa,oBAAoB;IACZ,EAAE,CAAwB;IAC1B,MAAM,CAAqB;IAE5C,YAAY,OAAgC;QACxC,IAAI,CAAC,EAAE,GAAG,OAAO,CAAC,EAAE,CAAC;QACrB,IAAI,CAAC,MAAM,GAAG,IAAA,+BAAyB,EAAC,OAAO,CAAC,WAAW,CAAC,CAAC;IACjE,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,MAAe;QACxB,KAAK,MAAM,KAAK,IAAI,IAAA,kBAAY,EAAC,IAAA,iBAAW,EAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC;YACzE,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CACf;cACF,IAAI,CAAC,MAAM,CAAC,cAAc;;;EAGtC,IAAA,6BAAuB,EAAC;gBACtB,QAAQ,EAAE,KAAK,CAAC,MAAM;gBACtB,KAAK,EAAE,CAAC,EAAE,EAAE,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,EAAE,CAAC;gBAC1C,SAAS,EAAE,SAAS;aACvB,CAAC;;;;;;;CAOD,CAAC,IAAI,EAAE,EACQ,KAAK,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;gBACpB,MAAM,KAAK,GAAG,IAAA,uCAAmB,EAAC,IAAA,0CAAsB,EAAC,KAAK,CAAC,CAAC,CAAC;gBAEjE,OAAO;oBACH,KAAK,CAAC,EAAE;oBACR,KAAK,CAAC,UAAU;oBAChB,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;oBACnB,IAAA,gBAAU,EAAC,MAAM,CAAC,WAAW,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;oBACjD,KAAK,CAAC,MAAM;iBACf,CAAC;YACN,CAAC,CAAC,CACL,CAAC;QACN,CAAC;IACL,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,UAAyB,EAAE;QACnD,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,IAAA,uCAAmB,EAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC5D,MAAM,IAAI,GAAG,IAAA,mBAAa,EAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;YACxC,OAAO,EAAE,CAAC;QACd,CAAC;QAED,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACpE,MAAM,iBAAiB,GAAG,MAAM,IAAI,CAAC,0BAA0B,CAAC,UAAU,CAAC,CAAC;QAC5E,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QACjD,MAAM,IAAI,GAAG,OAAO;aACf,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YACb,KAAK;YACL,KAAK,EAAE,UAAU,CAAC;gBACd,KAAK;gBACL,UAAU;gBACV,iBAAiB;gBACjB,cAAc;aACjB,CAAC;SACL,CAAC,CAAC;aACF,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;aAChC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC,QAAQ,CAAC;aAClF,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,CACnB,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK;eACrB,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC;eAC3D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,CAC3D,CAAC;aACD,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;QAEpB,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YACvB,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO;YAC3B,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU;YACjC,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,QAAQ,EAAE;gBACN,MAAM,EAAE,SAAS;gBACjB,YAAY,EAAE,UAAU,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;aAC7E;SACJ,CAAC,CAAC,CAAC;IACR,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,UAAkB;QACrC,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CACf,eAAe,IAAI,CAAC,MAAM,CAAC,cAAc,yBAAyB,EAClE,CAAC,UAAU,CAAC,CACf,CAAC;IACN,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAC7B,UAAoB,EACpB,OAAsB;QAEtB,MAAM,KAAK,GAAG,IAAA,2BAAqB,EAAC,OAAO,EAAE,CAAC,EAAE,GAAG,CAAC,CAAC;QACrD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAC9B;;;;;;;OAOL,IAAI,CAAC,MAAM,CAAC,cAAc;OAC1B,IAAI,CAAC,MAAM,CAAC,MAAM;;EAEvB,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE;CAC5E,CAAC,IAAI,EAAE,EACI,CAAC,UAAU,EAAE,GAAG,KAAK,CAAC,MAAM,CAAC,CAChC,CAAC;QAEF,OAAO,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YAC7B,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;YAC7B,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,CAAC;YACnC,UAAU,EAAE,gBAAgB,CAAC,GAAG,CAAC,WAAW,CAAC;YAC7C,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,IAAI,CAAC,CAAC;YAC/B,OAAO,EAAE,MAAM,CAAC,GAAG,CAAC,OAAO,IAAI,EAAE,CAAC;SACrC,CAAC,CAAC,CAAC;IACR,CAAC;IAEO,KAAK,CAAC,0BAA0B,CAAC,UAAoB;QACzD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAC9B;;OAEL,IAAI,CAAC,MAAM,CAAC,cAAc;;CAEhC,CAAC,IAAI,EAAE,EACI,CAAC,UAAU,CAAC,CACf,CAAC;QACF,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAE/D,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;YAC5B,MAAM,KAAK,GAAG,eAAe,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YACzC,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;gBAC5B,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;oBACvB,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;gBACxD,CAAC;YACL,CAAC;QACL,CAAC;QAED,OAAO,SAAS,CAAC;IACrB,CAAC;IAEO,KAAK,CAAC,YAAY;QACtB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,KAAK,CAC9B,iCAAiC,IAAI,CAAC,MAAM,CAAC,cAAc,EAAE,CAChE,CAAC;QACF,MAAM,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;QAEjD,OAAO,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAC9B,CAAC;CACJ;AAvJD,oDAuJC;AAED,SAAS,UAAU,CAAC,KAKnB;IACG,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,MAAM,IAAI,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;QAClC,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxD,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;YAClB,SAAS;QACb,CAAC;QAED,MAAM,EAAE,GAAG,KAAK,CAAC,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,cAAc,GAAG,EAAE,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC;QACzE,MAAM,YAAY,GAAG,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;QAC5E,KAAK,IAAI,YAAY,GAAG,GAAG,CAAC;IAChC,CAAC;IAED,OAAO,KAAK,CAAC;AACjB,CAAC;AAED,SAAS,UAAU,CAAC,KAAe;IAC/B,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACvB,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAClD,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACpC,MAAM,MAAM,GAAG,IAAA,oBAAc,EAAC,KAAK,CAAC,IAAI,EAAE,CAAC;IAC3C,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;QACjD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;YAC5B,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC5B,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,SAAS,eAAe,CAAC,KAAc;IACnC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,CAAC;IACd,CAAC;IAED,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,EAAkB,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC;AAC5E,CAAC"}

package/dist/rag/stores/postgres/sql.d.ts

@@ -16,6 +16,28 @@ export interface SqlWhere {
 export declare function buildChunkSearchWhere(options: SearchOptions, startIndex?: number, alias?: string): SqlWhere;
 export declare function assignIfDefined<T extends object, K extends keyof T>(target: T, key: K, value: T[K] | undefined): void;
 export declare function normalizeTopK(topK: number | undefined): number;
+/**
+ * 批量 upsert 的单批行数上限。500 行是往返次数与单语句参数量之间的平衡：
+ * 最宽的表（chunks，17 列）一批也只有 8500 个参数，远低于 Postgres 65535 的协议上限。
+ */
+export declare const UPSERT_BATCH_SIZE = 500;
+export declare function chunkBatches<T>(items: readonly T[], batchSize?: number): T[][];
+/**
+ * 同一条 multi-row INSERT 里出现重复冲突键时，Postgres 会报
+ * "ON CONFLICT DO UPDATE command cannot affect row a second time"。
+ * 逐行 INSERT 的旧语义是"后写覆盖先写"，这里按 key 去重保留最后一条以保持语义一致。
+ */
+export declare function dedupeByKey<T>(items: readonly T[], keyOf: (item: T) => string): T[];
+/**
+ * 生成 multi-row VALUES 的占位符片段，例如：
+ * ($1, $2::jsonb, now()),\n($3, $4::jsonb, now())。
+ * casts 按列声明类型转换后缀（'' 表示无转换），rowSuffix 用于追加 now() 这类字面量列。
+ */
+export declare function createMultiRowValuesSql(input: {
+    rowCount: number;
+    casts: readonly string[];
+    rowSuffix?: string;
+}): string;
 export declare function rowToChunkMetadata(row: Record<string, unknown>): ChunkMetadata;
 export declare function readOptionalString(value: unknown): string | undefined;
 export declare function readOptionalNumber(value: unknown): number | undefined;

package/dist/rag/stores/postgres/sql.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"sql.d.ts","sourceRoot":"","sources":["../../../../src/rag/stores/postgres/sql.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,aAAa,EAChB,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EACR,aAAa,EAChB,MAAM,UAAU,CAAC;AAClB,OAAO,KAAK,EACR,kBAAkB,EACrB,MAAM,SAAS,CAAC;AAKjB,wBAAgB,yBAAyB,CAAC,WAAW,SAAuB,GAAG,kBAAkB,CAShG;AAED,wBAAgB,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAI1D;AAED,wBAAgB,UAAU,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAEjD;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAEhE;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAIlF;AAED,wBAAgB,aAAa,CAAC,CAAC,GAAG,OAAO,EAAE,KAAK,EAAE,OAAO,GAAG,CAAC,EAAE,GAAG,SAAS,CAI1E;AAED,wBAAgB,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAMhD;AAED,wBAAgB,eAAe,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAEjE;AAED,MAAM,WAAW,QAAQ;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,OAAO,EAAE,CAAC;CACrB;AAED,wBAAgB,qBAAqB,CACjC,OAAO,EAAE,aAAa,EACtB,UAAU,SAAI,EACd,KAAK,SAAM,GACZ,QAAQ,CAsCV;AA4CD,wBAAgB,eAAe,CAAC,CAAC,SAAS,MAAM,EAAE,CAAC,SAAS,MAAM,CAAC,EAC/D,MAAM,EAAE,CAAC,EACT,GAAG,EAAE,CAAC,EACN,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,GACxB,IAAI,CAIN;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAS9D;AAED,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,aAAa,CAiB9E;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAErE;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAWrE;AAED,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,EAAE,GAAG,SAAS,CAM5E"}
+{"version":3,"file":"sql.d.ts","sourceRoot":"","sources":["../../../../src/rag/stores/postgres/sql.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACR,aAAa,EAChB,MAAM,aAAa,CAAC;AACrB,OAAO,KAAK,EACR,aAAa,EAChB,MAAM,UAAU,CAAC;AAClB,OAAO,KAAK,EACR,kBAAkB,EACrB,MAAM,SAAS,CAAC;AAKjB,wBAAgB,yBAAyB,CAAC,WAAW,SAAuB,GAAG,kBAAkB,CAShG;AAED,wBAAgB,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAI1D;AAED,wBAAgB,UAAU,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAEjD;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAEhE;AAED,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,CAIlF;AAED,wBAAgB,aAAa,CAAC,CAAC,GAAG,OAAO,EAAE,KAAK,EAAE,OAAO,GAAG,CAAC,EAAE,GAAG,SAAS,CAI1E;AAED,wBAAgB,QAAQ,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAMhD;AAED,wBAAgB,eAAe,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAEjE;AAED,MAAM,WAAW,QAAQ;IACrB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,OAAO,EAAE,CAAC;CACrB;AAED,wBAAgB,qBAAqB,CACjC,OAAO,EAAE,aAAa,EACtB,UAAU,SAAI,EACd,KAAK,SAAM,GACZ,QAAQ,CAsCV;AA4CD,wBAAgB,eAAe,CAAC,CAAC,SAAS,MAAM,EAAE,CAAC,SAAS,MAAM,CAAC,EAC/D,MAAM,EAAE,CAAC,EACT,GAAG,EAAE,CAAC,EACN,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,SAAS,GACxB,IAAI,CAIN;AAED,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,CAS9D;AAED;;;GAGG;AACH,eAAO,MAAM,iBAAiB,MAAM,CAAC;AAErC,wBAAgB,YAAY,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,SAAS,SAAoB,GAAG,CAAC,EAAE,EAAE,CAOzF;AAED;;;;GAIG;AACH,wBAAgB,WAAW,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,MAAM,GAAG,CAAC,EAAE,CAOnF;AAED;;;;GAIG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE;IAC3C,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,SAAS,MAAM,EAAE,CAAC;IACzB,SAAS,CAAC,EAAE,MAAM,CAAC;CACtB,GAAG,MAAM,CAWT;AAED,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,aAAa,CAiB9E;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAErE;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,SAAS,CAWrE;AAED,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,EAAE,GAAG,SAAS,CAM5E"}

package/dist/rag/stores/postgres/sql.js

@@ -1,5 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.UPSERT_BATCH_SIZE = void 0;
 exports.resolvePostgresTableNames = resolvePostgresTableNames;
 exports.quoteIdentifier = quoteIdentifier;
 exports.createJson = createJson;
@@ -11,6 +12,9 @@ exports.toVectorLiteral = toVectorLiteral;
 exports.buildChunkSearchWhere = buildChunkSearchWhere;
 exports.assignIfDefined = assignIfDefined;
 exports.normalizeTopK = normalizeTopK;
+exports.chunkBatches = chunkBatches;
+exports.dedupeByKey = dedupeByKey;
+exports.createMultiRowValuesSql = createMultiRowValuesSql;
 exports.rowToChunkMetadata = rowToChunkMetadata;
 exports.readOptionalString = readOptionalString;
 exports.readOptionalNumber = readOptionalNumber;
@@ -131,6 +135,44 @@ function normalizeTopK(topK) {
     }
     return Math.floor(topK);
 }
+/**
+ * 批量 upsert 的单批行数上限。500 行是往返次数与单语句参数量之间的平衡：
+ * 最宽的表（chunks，17 列）一批也只有 8500 个参数，远低于 Postgres 65535 的协议上限。
+ */
+exports.UPSERT_BATCH_SIZE = 500;
+function chunkBatches(items, batchSize = exports.UPSERT_BATCH_SIZE) {
+    const batches = [];
+    for (let start = 0; start < items.length; start += batchSize) {
+        batches.push(items.slice(start, start + batchSize));
+    }
+    return batches;
+}
+/**
+ * 同一条 multi-row INSERT 里出现重复冲突键时，Postgres 会报
+ * "ON CONFLICT DO UPDATE command cannot affect row a second time"。
+ * 逐行 INSERT 的旧语义是"后写覆盖先写"，这里按 key 去重保留最后一条以保持语义一致。
+ */
+function dedupeByKey(items, keyOf) {
+    const byKey = new Map();
+    for (const item of items) {
+        byKey.set(keyOf(item), item);
+    }
+    return [...byKey.values()];
+}
+/**
+ * 生成 multi-row VALUES 的占位符片段，例如：
+ * ($1, $2::jsonb, now()),\n($3, $4::jsonb, now())。
+ * casts 按列声明类型转换后缀（'' 表示无转换），rowSuffix 用于追加 now() 这类字面量列。
+ */
+function createMultiRowValuesSql(input) {
+    const width = input.casts.length;
+    const rows = [];
+    for (let rowIndex = 0; rowIndex < input.rowCount; rowIndex += 1) {
+        const placeholders = input.casts.map((cast, columnIndex) => `$${rowIndex * width + columnIndex + 1}${cast}`);
+        rows.push(`(${placeholders.join(', ')}${input.rowSuffix ?? ''})`);
+    }
+    return rows.join(',\n');
+}
 function rowToChunkMetadata(row) {
     const metadata = {
         source: String(row.source ?? 'inline'),