@customize-agent/knowledge 1.0.1 → 2.1.0

package/dist/chunking/text-chunker.d.ts

@@ -15,9 +15,20 @@ export interface ChunkConfig {
 }
 export declare class TextChunker {
     chunk(text: string, file: ClassifiedFile, metadata?: Record<string, unknown>): TextChunk[];
+    private createCandidates;
+    private createTextCandidates;
+    private createTableCandidates;
+    private createDataCandidates;
+    private createCodeCandidates;
+    private splitIntoSections;
+    private mergeLeadingHeader;
+    private recursiveSplit;
+    private mergeParts;
+    private splitByWindow;
+    private enforceCandidateLimit;
     private withHeader;
-    private splitBySemanticBoundary;
     private createChunk;
+    private kindForCategory;
     private extractSectionTitle;
     private takeOverlap;
     private estimateTokens;

package/dist/chunking/text-chunker.js

@@ -1,75 +1,253 @@
 const DEFAULT_CONFIGS = {
     document: { maxChunkSize: 800, overlap: 100, headerInjection: true },
-    spreadsheet: { maxChunkSize: 1000, overlap: 200, headerInjection: true },
+    spreadsheet: { maxChunkSize: 1000, overlap: 120, headerInjection: true },
     image: { maxChunkSize: 512, overlap: 0, headerInjection: true },
-    cad: { maxChunkSize: 600, overlap: 100, headerInjection: true },
-    code: { maxChunkSize: 1000, overlap: 200, headerInjection: true },
-    data: { maxChunkSize: 600, overlap: 100, headerInjection: true },
+    cad: { maxChunkSize: 600, overlap: 80, headerInjection: true },
+    code: { maxChunkSize: 1000, overlap: 120, headerInjection: true },
+    data: { maxChunkSize: 600, overlap: 80, headerInjection: true },
     web: { maxChunkSize: 800, overlap: 100, headerInjection: true },
     diagram: { maxChunkSize: 512, overlap: 0, headerInjection: true },
     archive: { maxChunkSize: 500, overlap: 50, headerInjection: false },
     other: { maxChunkSize: 500, overlap: 50, headerInjection: false },
 };
+const RECURSIVE_SEPARATORS = [
+    /\n(?=#{1,6}\s)/u,
+    /\n{2,}/u,
+    /\n(?=(?:第[一二三四五六七八九十百千万\d]+[章节条]|[一二三四五六七八九十]+、|\d+[.)、]))/u,
+    /(?<=[。！？；])\s*/u,
+    /(?<=[，、])\s*/u,
+    /\s+/u,
+];
 export class TextChunker {
     chunk(text, file, metadata = {}) {
-        if (text.trim().length === 0)
+        const source = text.trim();
+        if (source.length === 0)
             return [];
         const config = DEFAULT_CONFIGS[file.category];
-        const normalized = this.withHeader(text, file, config);
-        const paragraphs = this.splitBySemanticBoundary(normalized, file.category);
+        const normalized = this.withHeader(source, file, config);
+        const candidates = this.enforceCandidateLimit(this.createCandidates(normalized, file, config), config);
+        return candidates.map((candidate, index) => this.createChunk(index, candidate, file, metadata));
+    }
+    createCandidates(text, file, config) {
+        if (file.category === 'spreadsheet')
+            return this.createTableCandidates(text, config);
+        if (file.category === 'data')
+            return this.createDataCandidates(text, config);
+        if (file.category === 'code')
+            return this.createCodeCandidates(text, config);
+        return this.createTextCandidates(text, file.category, config);
+    }
+    createTextCandidates(text, category, config) {
+        const sections = this.splitIntoSections(text, category);
+        const candidates = [];
+        sections.forEach((section, parentIndex) => {
+            const parentId = `p${parentIndex}`;
+            const parts = this.mergeLeadingHeader(this.recursiveSplit(section.text, config.maxChunkSize));
+            const merged = this.mergeParts(parts, config.maxChunkSize, config.overlap);
+            merged.forEach((part, childIndex) => {
+                const localStart = section.text.indexOf(part.replace(/^\s+/u, '').slice(0, 40));
+                const startChar = section.startChar + Math.max(0, localStart);
+                candidates.push({
+                    text: part,
+                    startChar,
+                    endChar: startChar + part.length,
+                    sectionTitle: section.title,
+                    kind: this.kindForCategory(category),
+                    parentId,
+                    parentIndex,
+                    childIndex,
+                });
+            });
+        });
+        return candidates;
+    }
+    createTableCandidates(text, config) {
+        const lines = text.split(/\r?\n/u).map(line => line.trim()).filter(Boolean);
+        const headerLines = lines.filter(line => /^(文件|类型|Sheet|表格|列|Columns?)[:：]/iu.test(line));
+        const dataLines = lines.filter(line => !headerLines.includes(line));
+        if (dataLines.length === 0)
+            return this.createTextCandidates(text, 'spreadsheet', config);
+        const header = headerLines.join('\n');
+        const candidates = [];
+        let rowStart = 0;
+        let parentIndex = 0;
+        while (rowStart < dataLines.length) {
+            const rows = [];
+            while (rowStart + rows.length < dataLines.length) {
+                const nextLine = dataLines[rowStart + rows.length];
+                if (!nextLine)
+                    break;
+                const nextRows = [...rows, nextLine];
+                const candidate = [header, `行范围: ${rowStart + 1}-${rowStart + nextRows.length}`, ...nextRows].filter(Boolean).join('\n');
+                if (rows.length > 0 && this.estimateTokens(candidate) > config.maxChunkSize)
+                    break;
+                rows.push(nextLine);
+            }
+            const textChunk = [header, `行范围: ${rowStart + 1}-${rowStart + rows.length}`, ...rows].filter(Boolean).join('\n');
+            candidates.push({
+                text: textChunk,
+                startChar: text.indexOf(rows[0] ?? ''),
+                endChar: text.indexOf(rows.at(-1) ?? '') + (rows.at(-1)?.length ?? 0),
+                sectionTitle: '表格数据',
+                kind: 'table',
+                parentId: `table-${parentIndex}`,
+                parentIndex,
+                childIndex: 0,
+                rowRange: `${rowStart + 1}-${rowStart + rows.length}`,
+            });
+            rowStart += Math.max(1, rows.length);
+            parentIndex += 1;
+        }
+        return candidates;
+    }
+    createDataCandidates(text, config) {
+        const sections = text.split(/\n(?=[\w.[\]-]+[:：]\s)|\n{2,}/u).map(part => part.trim()).filter(Boolean);
+        const parts = sections.length > 1 ? sections : this.recursiveSplit(text, config.maxChunkSize);
+        return this.mergeParts(parts, config.maxChunkSize, config.overlap).map((part, index) => ({
+            text: part,
+            startChar: Math.max(0, text.indexOf(part.slice(0, 40))),
+            endChar: Math.max(0, text.indexOf(part.slice(0, 40))) + part.length,
+            sectionTitle: this.extractSectionTitle(part),
+            kind: 'data',
+            parentId: `data-${index}`,
+            parentIndex: index,
+            childIndex: 0,
+        }));
+    }
+    createCodeCandidates(text, config) {
+        const blocks = text.split(/\n(?=(?:export\s+)?(?:async\s+)?(?:function|class|interface|type|const|let|var)\s)/u).map(part => part.trim()).filter(Boolean);
+        const parts = blocks.length > 1 ? blocks : this.recursiveSplit(text, config.maxChunkSize);
+        return this.mergeParts(parts, config.maxChunkSize, config.overlap).map((part, index) => ({
+            text: part,
+            startChar: Math.max(0, text.indexOf(part.slice(0, 40))),
+            endChar: Math.max(0, text.indexOf(part.slice(0, 40))) + part.length,
+            sectionTitle: this.extractSectionTitle(part),
+            kind: 'code',
+            parentId: `code-${index}`,
+            parentIndex: index,
+            childIndex: 0,
+        }));
+    }
+    splitIntoSections(text, category) {
+        const pattern = category === 'cad' || category === 'diagram'
+            ? /\n{2,}/u
+            : /\n(?=#{1,6}\s)|\n{2,}/u;
+        const rawSections = this.mergeLeadingHeader(text.split(pattern).map(part => part.trim()).filter(Boolean));
+        let cursor = 0;
+        return rawSections.map(section => {
+            const startChar = Math.max(cursor, text.indexOf(section, cursor));
+            cursor = startChar + section.length;
+            return { text: section, startChar, title: this.extractSectionTitle(section) };
+        });
+    }
+    mergeLeadingHeader(sections) {
+        if (sections.length < 2)
+            return sections;
+        const first = sections[0];
+        if (!first)
+            return sections;
+        const isHeader = /^文件[:：].+\n类型[:：]/u.test(first) && this.estimateTokens(first) < 80;
+        if (!isHeader)
+            return sections;
+        return [`${first}\n\n${sections[1]}`, ...sections.slice(2)];
+    }
+    recursiveSplit(text, maxTokens, separatorIndex = 0) {
+        if (this.estimateTokens(text) <= maxTokens)
+            return [text.trim()].filter(Boolean);
+        if (separatorIndex >= RECURSIVE_SEPARATORS.length)
+            return this.splitByWindow(text, maxTokens);
+        const separator = RECURSIVE_SEPARATORS[separatorIndex];
+        if (!separator)
+            return this.splitByWindow(text, maxTokens);
+        const parts = text.split(separator).map(part => part.trim()).filter(Boolean);
+        if (parts.length <= 1)
+            return this.recursiveSplit(text, maxTokens, separatorIndex + 1);
+        return parts.flatMap(part => this.recursiveSplit(part, maxTokens, separatorIndex + 1));
+    }
+    mergeParts(parts, maxTokens, overlapTokens) {
         const chunks = [];
         let buffer = '';
-        let chunkStart = 0;
-        let cursor = 0;
-        for (const paragraph of paragraphs) {
-            const candidate = buffer.length === 0 ? paragraph : `${buffer}\n\n${paragraph}`;
-            if (this.estimateTokens(candidate) > config.maxChunkSize && buffer.length > 0) {
-                chunks.push(this.createChunk(chunks.length, buffer, chunkStart, cursor, metadata));
-                const overlapText = this.takeOverlap(buffer, config.overlap);
-                buffer = overlapText.length > 0 ? `${overlapText}\n\n${paragraph}` : paragraph;
-                chunkStart = Math.max(0, cursor - overlapText.length);
+        for (const part of parts) {
+            const candidate = buffer ? `${buffer}\n${part}` : part;
+            if (buffer && this.estimateTokens(candidate) > maxTokens) {
+                chunks.push(buffer);
+                const overlap = this.takeOverlap(buffer, overlapTokens);
+                buffer = overlap ? `${overlap}\n${part}` : part;
             }
             else {
                 buffer = candidate;
             }
-            cursor += paragraph.length + 2;
         }
-        if (buffer.trim().length > 0) {
-            chunks.push(this.createChunk(chunks.length, buffer, chunkStart, normalized.length, metadata));
+        if (buffer.trim())
+            chunks.push(buffer);
+        return chunks.flatMap(chunk => this.estimateTokens(chunk) > maxTokens ? this.splitByWindow(chunk, maxTokens, overlapTokens) : [chunk]);
+    }
+    splitByWindow(text, maxTokens, overlapTokens = 0) {
+        const maxChars = Math.max(200, maxTokens * 4);
+        const overlapChars = Math.max(0, overlapTokens * 4);
+        const step = Math.max(1, maxChars - overlapChars);
+        const chunks = [];
+        for (let start = 0; start < text.length; start += step) {
+            chunks.push(text.slice(start, start + maxChars).trim());
+            if (start + maxChars >= text.length)
+                break;
         }
-        return chunks;
+        return chunks.filter(Boolean);
+    }
+    enforceCandidateLimit(candidates, config) {
+        return candidates.flatMap(candidate => {
+            if (this.estimateTokens(candidate.text) <= config.maxChunkSize)
+                return [candidate];
+            return this.splitByWindow(candidate.text, config.maxChunkSize, config.overlap).map((text, index) => ({
+                ...candidate,
+                text,
+                childIndex: candidate.childIndex + index,
+                startChar: candidate.startChar + Math.max(0, candidate.text.indexOf(text.slice(0, 40))),
+                endChar: candidate.startChar + Math.max(0, candidate.text.indexOf(text.slice(0, 40))) + text.length,
+                rowRange: candidate.rowRange ? `${candidate.rowRange}#${index + 1}` : undefined,
+            }));
+        });
     }
     withHeader(text, file, config) {
         if (!config.headerInjection)
             return text;
         return `文件: ${file.relativePath}\n类型: ${file.category}/${file.format}\n\n${text}`;
     }
-    splitBySemanticBoundary(text, category) {
-        if (category === 'document') {
-            return text.split(/\n(?=#{1,6}\s)|\n{2,}/u).map(part => part.trim()).filter(Boolean);
-        }
-        if (category === 'code') {
-            return text.split(/\n(?=(export\s+)?(async\s+)?(function|class|interface|type|const|let|var)\s)/u).map(part => part.trim()).filter(Boolean);
-        }
-        if (category === 'data') {
-            return text.split(/\n(?=[\w.[\]-]+:\s)|\n{2,}/u).map(part => part.trim()).filter(Boolean);
-        }
-        if (category === 'cad' || category === 'diagram') {
-            return text.split(/\n(?=(?:CAD|STEP|IGES|Mesh|Draw\.io|Excalidraw|SVG)\b)|\n{2,}/u).map(part => part.trim()).filter(Boolean);
-        }
-        return text.split(/\n{2,}/u).map(part => part.trim()).filter(Boolean);
-    }
-    createChunk(index, text, startChar, endChar, metadata) {
+    createChunk(index, candidate, file, metadata) {
+        const text = candidate.text.trim();
         return {
             index,
-            text: text.trim(),
-            startChar,
-            endChar,
+            text,
+            startChar: candidate.startChar,
+            endChar: candidate.endChar,
             tokenCount: this.estimateTokens(text),
-            sectionTitle: this.extractSectionTitle(text),
-            metadata,
+            sectionTitle: candidate.sectionTitle ?? this.extractSectionTitle(text),
+            metadata: {
+                ...metadata,
+                chunkType: 'child',
+                chunkKind: candidate.kind,
+                parentId: `${file.relativePath}#${candidate.parentId}`,
+                parentIndex: candidate.parentIndex,
+                childIndex: candidate.childIndex,
+                rowRange: candidate.rowRange,
+                sectionTitle: candidate.sectionTitle ?? this.extractSectionTitle(text),
+                startChar: candidate.startChar,
+                endChar: candidate.endChar,
+                splitStrategy: 'recursive_parent_child_v1',
+            },
         };
     }
+    kindForCategory(category) {
+        if (category === 'image' || category === 'cad' || category === 'diagram')
+            return 'metadata';
+        if (category === 'code')
+            return 'code';
+        if (category === 'data')
+            return 'data';
+        if (category === 'spreadsheet')
+            return 'table';
+        return 'text';
+    }
     extractSectionTitle(text) {
         const firstLine = text.trim().split(/\r?\n/u)[0]?.trim();
         if (!firstLine)

package/dist/core/change-tracker.d.ts

@@ -7,4 +7,5 @@ export declare class ChangeTracker {
     constructor(store: IndexStateStore);
     computeDiff(diskFiles: Map<string, DiskFileStat>, classifier: FileClassifier, kbPath: string): Promise<DiffResult>;
     hashFile(filePath: string): string;
+    private parseMetadata;
 }

package/dist/core/change-tracker.js

@@ -29,6 +29,16 @@ export class ChangeTracker {
                 newFiles.push(classified);
                 continue;
             }
+            const metadata = this.parseMetadata(indexed.metadataJson);
+            const needsReindex = indexed.status === 'error'
+                || indexed.chunkCount === 0
+                || (classified.format === 'pdf' && indexed.chunkCount <= 1)
+                || metadata.contentCoverage === 'metadata_filename'
+                || metadata.extractionMode === 'pdf_metadata_only';
+            if (needsReindex) {
+                modifiedFiles.push(classified);
+                continue;
+            }
             if (Math.round(diskStat.mtime) !== Math.round(indexed.mtime) || diskStat.size !== indexed.fileSize) {
                 const contentHash = this.hashFile(absolutePath);
                 if (contentHash !== indexed.contentHash) {
@@ -63,4 +73,15 @@ export class ChangeTracker {
     hashFile(filePath) {
         return crypto.createHash('sha256').update(fs.readFileSync(filePath)).digest('hex');
     }
+    parseMetadata(metadataJson) {
+        if (!metadataJson)
+            return {};
+        try {
+            const parsed = JSON.parse(metadataJson);
+            return parsed && typeof parsed === 'object' ? parsed : {};
+        }
+        catch {
+            return {};
+        }
+    }
 }

package/dist/core/index-state-store.d.ts

@@ -15,6 +15,24 @@ export interface StoredChunk {
 }
 export interface ChunkSearchResult extends StoredChunk {
     score: number;
+    scoreDetails?: {
+        keywordScore?: number;
+        bm25Score?: number;
+        exactPhraseBoost?: number;
+    };
+}
+export interface StoredParentChunk {
+    id: string;
+    relativePath: string;
+    parentId: string;
+    content: string;
+    category: FileCategory;
+    format: string;
+    collectionName: string;
+    sectionTitle?: string;
+    chunkCount: number;
+    metadataJson?: string;
+    createdAt: number;
 }
 export interface FileHashRecord {
     contentHash: string;
@@ -29,6 +47,7 @@ export interface MinHashRecord {
     filePath: string;
     signature: number[];
     shingleCount: number;
+    buckets: string[];
     createdAt: number;
 }
 export interface FileRelationship {
@@ -43,6 +62,7 @@ export interface FileRelationship {
 }
 export declare class IndexStateStore {
     private readonly db;
+    private ftsEnabled;
     constructor(dbPath: string);
     loadActiveRecords(): Map<string, IndexStateRecord>;
     upsertRecord(record: IndexStateRecord): void;
@@ -58,12 +78,19 @@ export declare class IndexStateStore {
         relativePath?: string;
         limit?: number;
     }): StoredChunk[];
+    getContextChunks(relativePath: string, chunkIndex: number, window?: number): StoredChunk[];
+    listParentChunks(relativePath: string): StoredParentChunk[];
+    getParentChunk(relativePath: string, parentId: string): StoredParentChunk | undefined;
+    getChunksByParent(relativePath: string, parentId: string, limit?: number): StoredChunk[];
     searchChunks(query: string, limit?: number): ChunkSearchResult[];
+    private searchChunksFts;
+    private searchChunksLike;
     findExactDuplicate(contentHash: string, excludePath?: string): FileHashRecord | undefined;
     findNormalizedDuplicate(normalizedHash: string, excludePath?: string): FileHashRecord | undefined;
     upsertFileHash(record: Omit<FileHashRecord, 'createdAt' | 'updatedAt'>): void;
     upsertMinHash(record: Omit<MinHashRecord, 'createdAt'>): void;
     listMinHashes(excludePath?: string): MinHashRecord[];
+    listMinHashesByBuckets(buckets: string[], excludePath?: string): MinHashRecord[];
     addRelationship(relationship: Omit<FileRelationship, 'id' | 'createdAt'>): void;
     listRelationships(filePath?: string): FileRelationship[];
     setTags(relativePath: string, tags: string[]): void;
@@ -81,6 +108,7 @@ export declare class IndexStateStore {
     }>;
     deleteRecord(relativePath: string): void;
     setMetadata(key: string, value: string): void;
+    getMetadata(key: string): string | undefined;
     getStats(): {
         fileCount: number;
         chunkCount: number;
@@ -93,11 +121,19 @@ export declare class IndexStateStore {
     }>;
     close(): void;
     private initTables;
+    private initFts;
+    private rebuildFtsIfNeeded;
     private rowToMinHash;
     private rowToFileHash;
     private rowToRelationship;
+    private rowToParentChunk;
+    private splitParentGroups;
+    private metadataString;
     private rowToChunk;
     private expandSearchTerms;
-    private scoreChunk;
+    private toFtsQuery;
+    private bm25ToPositiveScore;
+    private scoreChunkDetailed;
+    private countOccurrences;
     private rowToRecord;
 }