@customize-agent/knowledge 4.0.1 → 4.0.2

package/dist/chunking/bge-tokenizer.d.ts

@@ -0,0 +1,10 @@
+export declare class BgeTokenizer {
+    private readonly vocab;
+    private readonly unkToken;
+    constructor(tokenizerPath?: string | undefined);
+    countTokens(text: string): number;
+    encode(text: string): string[];
+    private preTokenize;
+    private wordPiece;
+    private static resolveTokenizerPath;
+}

package/dist/chunking/bge-tokenizer.js

@@ -0,0 +1,68 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { fileURLToPath } from 'node:url';
+export class BgeTokenizer {
+    vocab;
+    unkToken;
+    constructor(tokenizerPath = BgeTokenizer.resolveTokenizerPath()) {
+        if (!tokenizerPath)
+            throw new Error('BGE tokenizer.json 缺失，无法执行真实 Token 计数');
+        const raw = JSON.parse(fs.readFileSync(tokenizerPath, 'utf8'));
+        this.vocab = new Set(Object.keys(raw.model?.vocab ?? {}));
+        this.unkToken = raw.model?.unk_token ?? '[UNK]';
+        if (this.vocab.size === 0)
+            throw new Error('BGE tokenizer vocab 为空，无法执行真实 Token 计数');
+    }
+    countTokens(text) {
+        return this.encode(text).length;
+    }
+    encode(text) {
+        const tokens = [];
+        for (const token of this.preTokenize(text))
+            tokens.push(...this.wordPiece(token));
+        return tokens;
+    }
+    preTokenize(text) {
+        const normalized = Array.from(text, char => {
+            const code = char.charCodeAt(0);
+            return code < 32 || code === 127 ? ' ' : char;
+        }).join('')
+            .replace(/([\p{Script=Han}])/gu, ' $1 ')
+            .normalize('NFKC');
+        return normalized.match(/[\p{Script=Han}]|[\p{Letter}\p{Number}_]+|[^\s\p{Letter}\p{Number}_]/gu) ?? [];
+    }
+    wordPiece(token) {
+        if (this.vocab.has(token))
+            return [token];
+        const pieces = [];
+        let start = 0;
+        while (start < token.length) {
+            let end = token.length;
+            let current;
+            while (start < end) {
+                const piece = `${start > 0 ? '##' : ''}${token.slice(start, end)}`;
+                if (this.vocab.has(piece)) {
+                    current = piece;
+                    break;
+                }
+                end -= 1;
+            }
+            if (!current)
+                return Array.from(token).map(() => this.unkToken);
+            pieces.push(current);
+            start = end;
+        }
+        return pieces;
+    }
+    static resolveTokenizerPath() {
+        const currentDir = path.dirname(fileURLToPath(import.meta.url));
+        const candidates = [
+            process.env.CUSTOMIZE_BGE_TOKENIZER_PATH,
+            process.env.KB_BGE_TOKENIZER_PATH,
+            path.resolve(process.cwd(), 'packages', 'knowledge', 'models', 'bge-small-zh-v1.5', 'tokenizer.json'),
+            path.resolve(process.cwd(), 'models', 'bge-small-zh-v1.5', 'tokenizer.json'),
+            path.resolve(currentDir, '..', '..', 'models', 'bge-small-zh-v1.5', 'tokenizer.json'),
+        ].filter(Boolean);
+        return candidates.find(candidate => fs.existsSync(candidate));
+    }
+}

package/dist/chunking/text-chunker.d.ts

@@ -14,17 +14,27 @@ export interface ChunkConfig {
     headerInjection: boolean;
 }
 export declare class TextChunker {
+    private readonly tokenizer;
     chunk(text: string, file: ClassifiedFile, metadata?: Record<string, unknown>): TextChunk[];
     private createCandidates;
     private createTextCandidates;
     private createTableCandidates;
     private createDataCandidates;
     private createCodeCandidates;
+    private normalizeCodeLanguage;
+    private splitCodeByLanguage;
+    private collectIndentSensitiveBlocks;
+    private collectBraceBalancedBlocks;
+    private splitCodeByStructuralFallback;
     private splitIntoSections;
     private mergeLeadingHeader;
     private recursiveSplit;
     private mergeParts;
     private splitByWindow;
+    private splitBySentenceBoundary;
+    private isMarkdownTable;
+    private extractMarkdownTableRowRange;
+    private splitMarkdownTable;
     private enforceCandidateLimit;
     private withHeader;
     private createChunk;

package/dist/chunking/text-chunker.js

@@ -1,3 +1,4 @@
+import { BgeTokenizer } from './bge-tokenizer.js';
 const DEFAULT_CONFIGS = {
     document: { maxChunkSize: 800, overlap: 100, headerInjection: true },
     spreadsheet: { maxChunkSize: 1000, overlap: 120, headerInjection: true },
@@ -18,7 +19,18 @@ const RECURSIVE_SEPARATORS = [
     /(?<=[，、])\s*/u,
     /\s+/u,
 ];
+const LANGUAGE_ROUTER = {
+    typescript: { delimiters: /\n(?=(?:export\s+)?(?:async\s+)?(?:class|function|interface|type|const|let)\s)/u, blockStart: /\{\s*$/u, indentSensitive: false },
+    javascript: { delimiters: /\n(?=(?:export\s+)?(?:async\s+)?(?:class|function|const|let)\s)/u, blockStart: /\{\s*$/u, indentSensitive: false },
+    python: { delimiters: /\n(?=(?:class|def)\s)/u, blockStart: /:\s*$/u, indentSensitive: true },
+    go: { delimiters: /\n(?=(?:func|type|struct|interface)\s)/u, blockStart: /\{\s*$/u, indentSensitive: false },
+    java: { delimiters: /\n(?=(?:public|protected|private|static|final|abstract|\s)*(?:class|interface|enum|(?:\w|<|>|\[|\])+\s+\w+\s*\())/u, blockStart: /\{\s*$/u, indentSensitive: false },
+    csharp: { delimiters: /\n(?=(?:public|protected|private|internal|static|sealed|abstract|\s)*(?:class|interface|enum|(?:\w|<|>|\[|\])+\s+\w+\s*\())/u, blockStart: /\{\s*$/u, indentSensitive: false },
+    cpp: { delimiters: /\n(?=(?:class|struct|namespace|template)\s|[\w:*&<>]+\s+\w+\s*\()/u, blockStart: /\{\s*$/u, indentSensitive: false },
+    c: { delimiters: /\n(?=(?:struct|enum)\s|[\w*]+\s+\w+\s*\()/u, blockStart: /\{\s*$/u, indentSensitive: false },
+};
 export class TextChunker {
+    tokenizer = new BgeTokenizer();
     chunk(text, file, metadata = {}) {
         const source = text.trim();
         if (source.length === 0)
@@ -34,7 +46,7 @@ export class TextChunker {
         if (file.category === 'data')
             return this.createDataCandidates(text, config);
         if (file.category === 'code')
-            return this.createCodeCandidates(text, config);
+            return this.createCodeCandidates(text, file, config);
         return this.createTextCandidates(text, file.category, config);
     }
     createTextCandidates(text, category, config) {
@@ -62,43 +74,23 @@ export class TextChunker {
         return candidates;
     }
     createTableCandidates(text, config) {
-        const lines = text.split(/\r?\n/u).map(line => line.trim()).filter(Boolean);
-        const headerLines = lines.filter(line => /^(文件|类型|Sheet|表格|列|Columns?)[:：]/iu.test(line));
-        const dataLines = lines.filter(line => !headerLines.includes(line));
-        if (dataLines.length === 0)
-            return this.createTextCandidates(text, 'spreadsheet', config);
-        const header = headerLines.join('\n');
-        const candidates = [];
-        let rowStart = 0;
-        let parentIndex = 0;
-        while (rowStart < dataLines.length) {
-            const rows = [];
-            while (rowStart + rows.length < dataLines.length) {
-                const nextLine = dataLines[rowStart + rows.length];
-                if (!nextLine)
-                    break;
-                const nextRows = [...rows, nextLine];
-                const candidate = [header, `行范围: ${rowStart + 1}-${rowStart + nextRows.length}`, ...nextRows].filter(Boolean).join('\n');
-                if (rows.length > 0 && this.estimateTokens(candidate) > config.maxChunkSize)
-                    break;
-                rows.push(nextLine);
-            }
-            const textChunk = [header, `行范围: ${rowStart + 1}-${rowStart + rows.length}`, ...rows].filter(Boolean).join('\n');
-            candidates.push({
-                text: textChunk,
-                startChar: text.indexOf(rows[0] ?? ''),
-                endChar: text.indexOf(rows.at(-1) ?? '') + (rows.at(-1)?.length ?? 0),
-                sectionTitle: '表格数据',
-                kind: 'table',
-                parentId: `table-${parentIndex}`,
-                parentIndex,
-                childIndex: 0,
-                rowRange: `${rowStart + 1}-${rowStart + rows.length}`,
+        if (this.isMarkdownTable(text)) {
+            return this.splitMarkdownTable(text, config.maxChunkSize).map((part, index) => {
+                const startChar = Math.max(0, text.indexOf(part.slice(0, 40)));
+                return {
+                    text: part,
+                    startChar,
+                    endChar: startChar + part.length,
+                    sectionTitle: this.extractSectionTitle(part) ?? '表格数据',
+                    kind: 'table',
+                    parentId: `table-${index}`,
+                    parentIndex: index,
+                    childIndex: 0,
+                    rowRange: this.extractMarkdownTableRowRange(part),
+                };
             });
-            rowStart += Math.max(1, rows.length);
-            parentIndex += 1;
         }
-        return candidates;
+        return this.createTextCandidates(text, 'spreadsheet', config);
     }
     createDataCandidates(text, config) {
         const sections = text.split(/\n(?=[\w.[\]-]+[:：]\s)|\n{2,}/u).map(part => part.trim()).filter(Boolean);
@@ -114,19 +106,89 @@ export class TextChunker {
             childIndex: 0,
         }));
     }
-    createCodeCandidates(text, config) {
-        const blocks = text.split(/\n(?=(?:export\s+)?(?:async\s+)?(?:function|class|interface|type|const|let|var)\s)/u).map(part => part.trim()).filter(Boolean);
+    createCodeCandidates(text, file, config) {
+        const language = this.normalizeCodeLanguage(file.format);
+        const languageConfig = LANGUAGE_ROUTER[language];
+        const blocks = languageConfig
+            ? this.splitCodeByLanguage(text, languageConfig)
+            : this.splitCodeByStructuralFallback(text);
         const parts = blocks.length > 1 ? blocks : this.recursiveSplit(text, config.maxChunkSize);
-        return this.mergeParts(parts, config.maxChunkSize, config.overlap).map((part, index) => ({
-            text: part,
-            startChar: Math.max(0, text.indexOf(part.slice(0, 40))),
-            endChar: Math.max(0, text.indexOf(part.slice(0, 40))) + part.length,
-            sectionTitle: this.extractSectionTitle(part),
-            kind: 'code',
-            parentId: `code-${index}`,
-            parentIndex: index,
-            childIndex: 0,
-        }));
+        return this.mergeParts(parts, config.maxChunkSize, config.overlap).map((part, index) => {
+            const startChar = Math.max(0, text.indexOf(part.slice(0, 40)));
+            return {
+                text: part,
+                startChar,
+                endChar: startChar + part.length,
+                sectionTitle: this.extractSectionTitle(part),
+                kind: 'code',
+                parentId: `code-${language}-${index}`,
+                parentIndex: index,
+                childIndex: 0,
+            };
+        });
+    }
+    normalizeCodeLanguage(format) {
+        const aliases = {
+            ts: 'typescript', tsx: 'typescript', typescript: 'typescript',
+            js: 'javascript', jsx: 'javascript', javascript: 'javascript',
+            py: 'python', python: 'python',
+            golang: 'go', go: 'go',
+            java: 'java', cs: 'csharp', csharp: 'csharp',
+            cpp: 'cpp', cxx: 'cpp', cc: 'cpp', hpp: 'cpp',
+            c: 'c', h: 'c',
+        };
+        return aliases[format.toLowerCase()] ?? format.toLowerCase();
+    }
+    splitCodeByLanguage(text, config) {
+        const raw = text.split(config.delimiters).map(part => part.trim()).filter(Boolean);
+        if (raw.length <= 1)
+            return raw;
+        return raw.flatMap(block => config.indentSensitive ? this.collectIndentSensitiveBlocks(block, config) : this.collectBraceBalancedBlocks(block, config));
+    }
+    collectIndentSensitiveBlocks(block, config) {
+        const lines = block.split(/\r?\n/u);
+        const result = [];
+        let current = [];
+        let baseIndent;
+        for (const line of lines) {
+            const indent = line.match(/^\s*/u)?.[0].length ?? 0;
+            if (current.length > 0 && baseIndent != null && indent <= baseIndent && config.blockStart.test(current[0] ?? '') && line.trim()) {
+                result.push(current.join('\n').trim());
+                current = [];
+                baseIndent = undefined;
+            }
+            if (current.length === 0)
+                baseIndent = indent;
+            current.push(line);
+        }
+        if (current.length > 0)
+            result.push(current.join('\n').trim());
+        return result.filter(Boolean);
+    }
+    collectBraceBalancedBlocks(block, _config) {
+        const lines = block.split(/\r?\n/u);
+        const result = [];
+        let current = [];
+        let depth = 0;
+        for (const line of lines) {
+            current.push(line);
+            depth += (line.match(/\{/gu) ?? []).length;
+            depth -= (line.match(/\}/gu) ?? []).length;
+            if (current.length > 1 && depth <= 0) {
+                result.push(current.join('\n').trim());
+                current = [];
+                depth = 0;
+            }
+        }
+        if (current.length > 0)
+            result.push(current.join('\n').trim());
+        return result.filter(Boolean);
+    }
+    splitCodeByStructuralFallback(text) {
+        const blocks = this.collectBraceBalancedBlocks(text, { delimiters: /\n/u, blockStart: /\{\s*$/u, indentSensitive: false });
+        if (blocks.length > 1)
+            return blocks;
+        return text.split(/\n(?=\S)/u).map(part => part.trim()).filter(Boolean);
     }
     splitIntoSections(text, category) {
         const pattern = category === 'cad' || category === 'diagram'
@@ -154,11 +216,13 @@ export class TextChunker {
     recursiveSplit(text, maxTokens, separatorIndex = 0) {
         if (this.estimateTokens(text) <= maxTokens)
             return [text.trim()].filter(Boolean);
+        if (this.isMarkdownTable(text))
+            return this.splitMarkdownTable(text, maxTokens);
         if (separatorIndex >= RECURSIVE_SEPARATORS.length)
-            return this.splitByWindow(text, maxTokens);
+            return this.splitBySentenceBoundary(text, maxTokens);
         const separator = RECURSIVE_SEPARATORS[separatorIndex];
         if (!separator)
-            return this.splitByWindow(text, maxTokens);
+            return this.splitBySentenceBoundary(text, maxTokens);
         const parts = text.split(separator).map(part => part.trim()).filter(Boolean);
         if (parts.length <= 1)
             return this.recursiveSplit(text, maxTokens, separatorIndex + 1);
@@ -183,8 +247,11 @@ export class TextChunker {
         return chunks.flatMap(chunk => this.estimateTokens(chunk) > maxTokens ? this.splitByWindow(chunk, maxTokens, overlapTokens) : [chunk]);
     }
     splitByWindow(text, maxTokens, overlapTokens = 0) {
-        const maxChars = Math.max(200, maxTokens * 4);
-        const overlapChars = Math.max(0, overlapTokens * 4);
+        const tokens = this.tokenizer.encode(text);
+        if (tokens.length <= maxTokens)
+            return [text.trim()].filter(Boolean);
+        const maxChars = Math.max(200, Math.ceil(text.length * (maxTokens / Math.max(1, tokens.length))));
+        const overlapChars = Math.max(0, Math.ceil(text.length * (overlapTokens / Math.max(1, tokens.length))));
         const step = Math.max(1, maxChars - overlapChars);
         const chunks = [];
         for (let start = 0; start < text.length; start += step) {
@@ -194,6 +261,44 @@ export class TextChunker {
         }
         return chunks.filter(Boolean);
     }
+    splitBySentenceBoundary(text, maxTokens) {
+        const units = text.split(/(?<=[。？！；;.!?])\s+|(?<=[。？！；;.!?])/u).map(part => part.trim()).filter(Boolean);
+        if (units.length <= 1)
+            return this.splitByWindow(text, maxTokens);
+        return this.mergeParts(units, maxTokens, 0);
+    }
+    isMarkdownTable(text) {
+        const lines = text.trim().split(/\r?\n/u);
+        return lines.length >= 3 && lines.some(line => /^\s*\|?\s*:?-{3,}:?\s*\|/u.test(line));
+    }
+    extractMarkdownTableRowRange(text) {
+        const lines = text.trim().split(/\r?\n/u).filter(line => /^\s*\|/u.test(line));
+        const rowCount = Math.max(0, lines.length - 2);
+        return rowCount > 0 ? `1-${rowCount}` : undefined;
+    }
+    splitMarkdownTable(text, maxTokens) {
+        const lines = text.trim().split(/\r?\n/u).filter(Boolean);
+        const separatorIndex = lines.findIndex(line => /^\s*\|?\s*:?-{3,}:?\s*\|/u.test(line));
+        if (separatorIndex <= 0)
+            return this.splitBySentenceBoundary(text, maxTokens);
+        const header = lines.slice(0, separatorIndex + 1);
+        const rows = lines.slice(separatorIndex + 1);
+        const chunks = [];
+        let current = [];
+        for (const row of rows) {
+            const candidate = [...header, ...current, row].join('\n');
+            if (current.length > 0 && this.estimateTokens(candidate) > maxTokens) {
+                chunks.push([...header, ...current].join('\n'));
+                current = [row];
+            }
+            else {
+                current.push(row);
+            }
+        }
+        if (current.length > 0)
+            chunks.push([...header, ...current].join('\n'));
+        return chunks.flatMap(chunk => this.estimateTokens(chunk) > maxTokens ? this.splitByWindow(chunk, maxTokens) : [chunk]);
+    }
     enforceCandidateLimit(candidates, config) {
         return candidates.flatMap(candidate => {
             if (this.estimateTokens(candidate.text) <= config.maxChunkSize)
@@ -263,6 +368,6 @@ export class TextChunker {
         return text.slice(Math.max(0, text.length - chars));
     }
     estimateTokens(text) {
-        return Math.max(1, Math.ceil(text.length / 4));
+        return Math.max(1, this.tokenizer.countTokens(text));
     }
 }

package/dist/classification/classifier.js

@@ -76,7 +76,6 @@ export class FileClassifier {
             ['.html', 'web', 'html'], ['.htm', 'web', 'html'], ['.xhtml', 'web', 'html'], ['.css', 'web', 'stylesheet'], ['.scss', 'web', 'stylesheet'], ['.sass', 'web', 'stylesheet'], ['.less', 'web', 'stylesheet'],
             ['.hbs', 'web', 'template'], ['.ejs', 'web', 'template'], ['.pug', 'web', 'template'], ['.j2', 'web', 'template'], ['.jinja2', 'web', 'template'],
             ['.drawio', 'diagram', 'drawio'], ['.dio', 'diagram', 'drawio'], ['.vsdx', 'diagram', 'visio'], ['.vdx', 'diagram', 'visio'], ['.puml', 'diagram', 'plantuml'], ['.plantuml', 'diagram', 'plantuml'], ['.mmd', 'diagram', 'mermaid'], ['.mermaid', 'diagram', 'mermaid'], ['.excalidraw', 'diagram', 'excalidraw'],
-            ['.zip', 'archive', 'zip'], ['.jar', 'archive', 'zip'], ['.war', 'archive', 'zip'], ['.apk', 'archive', 'zip'], ['.tar', 'archive', 'tar'], ['.gz', 'archive', 'other'], ['.tgz', 'archive', 'tar'], ['.bz2', 'archive', 'other'], ['.rar', 'archive', 'other'], ['.7z', 'archive', 'other'],
         ];
         return new Map(entries.map(([ext, category, format]) => [ext, [category, format]]));
     }

package/dist/core/index-state-store.d.ts

@@ -1,6 +1,18 @@
 import type { TextChunk } from '../chunking/text-chunker.js';
 import type { FileCategory, IndexStateRecord } from '../types.js';
+export type KnowledgeJobStatus = 'PENDING' | 'PARSING' | 'CHUNKING' | 'INDEXING' | 'SUCCESS' | 'ERROR';
+export interface KnowledgeIndexJob {
+    id: string;
+    relativePath: string;
+    status: KnowledgeJobStatus;
+    percent: number;
+    message: string;
+    errorMessage?: string;
+    createdAt: number;
+    updatedAt: number;
+}
 export interface StoredChunk {
+    rowid: number;
     id: string;
     relativePath: string;
     chunkIndex: number;
@@ -34,6 +46,18 @@ export interface StoredParentChunk {
     metadataJson?: string;
     createdAt: number;
 }
+export interface StoredDocumentChunk {
+    id: string;
+    relativePath: string;
+    content: string;
+    category: FileCategory;
+    format: string;
+    collectionName: string;
+    parentCount: number;
+    chunkCount: number;
+    metadataJson?: string;
+    createdAt: number;
+}
 export interface FileHashRecord {
     contentHash: string;
     filePath: string;
@@ -68,6 +92,22 @@ export declare class IndexStateStore {
     upsertRecord(record: IndexStateRecord): void;
     updateVerified(relativePath: string, mtime: number): void;
     listRecords(): IndexStateRecord[];
+    enqueueIndexJob(job: {
+        id: string;
+        relativePath: string;
+        message?: string;
+    }): KnowledgeIndexJob;
+    updateIndexJob(id: string, patch: {
+        status?: KnowledgeJobStatus;
+        percent?: number;
+        message?: string;
+        errorMessage?: string;
+    }): void;
+    getIndexJob(id: string): KnowledgeIndexJob | undefined;
+    listPendingIndexJobs(limit?: number): KnowledgeIndexJob[];
+    countPendingIndexJobs(): number;
+    listActiveIndexJobsByPath(relativePath: string): KnowledgeIndexJob[];
+    listIndexJobsByPrefix(prefix: string): KnowledgeIndexJob[];
     replaceChunks(relativePath: string, chunks: TextChunk[], file: {
         category: FileCategory;
         format: string;
@@ -78,9 +118,12 @@ export declare class IndexStateStore {
         relativePath?: string;
         limit?: number;
     }): StoredChunk[];
+    getChunkByRowid(rowid: number): StoredChunk | undefined;
+    getChunksByRowids(rowids: number[]): StoredChunk[];
     getContextChunks(relativePath: string, chunkIndex: number, window?: number): StoredChunk[];
     listParentChunks(relativePath: string): StoredParentChunk[];
     getParentChunk(relativePath: string, parentId: string): StoredParentChunk | undefined;
+    getDocumentChunk(relativePath: string): StoredDocumentChunk | undefined;
     getChunksByParent(relativePath: string, parentId: string, limit?: number): StoredChunk[];
     searchChunks(query: string, limit?: number): ChunkSearchResult[];
     private searchChunksFts;
@@ -127,11 +170,15 @@ export declare class IndexStateStore {
     private rowToFileHash;
     private rowToRelationship;
     private rowToParentChunk;
+    private rowToDocumentChunk;
     private splitParentGroups;
+    private rowToJob;
     private metadataString;
     private rowToChunk;
     private expandSearchTerms;
     private toFtsQuery;
+    private mergeKeywordResults;
+    private chineseNgrams;
     private bm25ToPositiveScore;
     private scoreChunkDetailed;
     private countOccurrences;