@customize-agent/knowledge 4.0.15 → 4.0.17

package/dist/chunking/text-chunker.d.ts

@@ -44,6 +44,7 @@ export declare class TextChunker {
     private splitByWindow;
     private splitBySentenceBoundary;
     private isMarkdownTable;
+    private extractMarkdownTableBlocks;
     private extractMarkdownTableRowRange;
     private splitMarkdownTable;
     private enforceCandidateLimit;

package/dist/chunking/text-chunker.js

@@ -79,6 +79,7 @@ export class TextChunker {
                     parentId,
                     parentIndex,
                     childIndex,
+                    parentText: section.text, // <=== 记录原始完整的 Section 文本
                 });
             });
         });
@@ -86,21 +87,29 @@ export class TextChunker {
     }
     createTableCandidates(text, config) {
         if (this.isMarkdownTable(text)) {
-            return this.splitMarkdownTable(text, config.maxChunkSize).map((part, index) => {
-                const startChar = Math.max(0, text.indexOf(part.slice(0, 40)));
-                return {
-                    text: part,
-                    startChar,
-                    endChar: startChar + part.length,
-                    sectionTitle: this.extractSectionTitle(part) ?? '表格数据',
-                    titlePath: this.extractSectionTitle(part) ?? '表格数据',
-                    kind: 'table',
-                    parentId: `table-${index}`,
-                    parentIndex: index,
-                    childIndex: 0,
-                    rowRange: this.extractMarkdownTableRowRange(part),
-                };
-            });
+            const tableBlocks = this.extractMarkdownTableBlocks(text);
+            if (tableBlocks.length > 0) {
+                return tableBlocks.flatMap((block, parentIndex) => {
+                    const sectionTitle = this.extractSectionTitle(block.text) ?? '表格数据';
+                    return this.splitMarkdownTable(block.text, config.maxChunkSize).map((part, childIndex) => {
+                        const localStart = Math.max(0, block.text.indexOf(part.slice(0, 40)));
+                        const startChar = block.startChar + localStart;
+                        return {
+                            text: part,
+                            startChar,
+                            endChar: startChar + part.length,
+                            sectionTitle,
+                            titlePath: sectionTitle,
+                            kind: 'table',
+                            parentId: `table-${parentIndex}`,
+                            parentIndex,
+                            childIndex,
+                            rowRange: this.extractMarkdownTableRowRange(part),
+                            parentText: block.text,
+                        };
+                    });
+                });
+            }
         }
         return this.createTextCandidates(text, 'spreadsheet', config);
     }
@@ -114,9 +123,10 @@ export class TextChunker {
             sectionTitle: this.extractSectionTitle(part),
             titlePath: this.extractSectionTitle(part),
             kind: 'data',
-            parentId: `data-${index}`,
-            parentIndex: index,
-            childIndex: 0,
+            parentId: `data-0`,
+            parentIndex: 0,
+            childIndex: index,
+            parentText: text,
         }));
     }
     createCodeCandidates(text, file, config) {
@@ -135,9 +145,10 @@ export class TextChunker {
                 sectionTitle: this.extractSectionTitle(part),
                 titlePath: this.extractSectionTitle(part),
                 kind: 'code',
-                parentId: `code-${language}-${index}`,
-                parentIndex: index,
-                childIndex: 0,
+                parentId: `code-${language}-0`,
+                parentIndex: 0,
+                childIndex: index,
+                parentText: text,
             };
         });
     }
@@ -308,6 +319,40 @@ export class TextChunker {
         const lines = text.trim().split(/\r?\n/u);
         return lines.length >= 3 && lines.some(line => /^\s*\|?\s*:?-{3,}:?\s*\|/u.test(line));
     }
+    extractMarkdownTableBlocks(text) {
+        const lines = text.split(/\r?\n/u);
+        const lineStarts = [];
+        let cursor = 0;
+        for (const line of lines) {
+            lineStarts.push(cursor);
+            cursor += line.length + 1;
+        }
+        const blocks = [];
+        let index = 0;
+        while (index < lines.length) {
+            const separatorIndex = lines.findIndex((line, lineIndex) => lineIndex >= index && /^\s*\|?\s*:?-{3,}:?\s*\|/u.test(line));
+            if (separatorIndex <= index)
+                break;
+            const headerIndex = separatorIndex - 1;
+            let startLine = headerIndex;
+            const titleIndex = headerIndex - 1;
+            if (titleIndex >= 0 && lines[titleIndex]?.trim() === '' && titleIndex - 1 >= 0) {
+                const title = lines[titleIndex - 1]?.trim() ?? '';
+                if (title && title.length <= 120 && !/^\s*\|/u.test(title))
+                    startLine = titleIndex - 1;
+            }
+            let endLine = separatorIndex + 1;
+            while (endLine < lines.length && /^\s*\|/u.test(lines[endLine] ?? ''))
+                endLine += 1;
+            const startChar = lineStarts[startLine] ?? 0;
+            const endChar = endLine < lineStarts.length ? (lineStarts[endLine] ?? text.length) : text.length;
+            const blockText = text.slice(startChar, endChar).trim();
+            if (blockText)
+                blocks.push({ text: blockText, startChar });
+            index = endLine;
+        }
+        return blocks;
+    }
     extractMarkdownTableRowRange(text) {
         const lines = text.trim().split(/\r?\n/u).filter(line => /^\s*\|/u.test(line));
         const rowCount = Math.max(0, lines.length - 2);
@@ -395,6 +440,7 @@ export class TextChunker {
                 startChar: candidate.startChar,
                 endChar: candidate.endChar,
                 splitStrategy: 'recursive_parent_child_v2',
+                parentText: candidate.parentText, // <=== 原始父块内容
             },
         };
     }

package/dist/core/index-state-store.js

@@ -159,9 +159,14 @@ export class IndexStateStore {
             }
             const parentContents = [];
             for (const [parentId, group] of parentGroups.entries()) {
-                const parentContent = group.map(chunk => chunk.text).join('\n\n---\n\n');
+                const firstChunkMeta = group[0]?.metadata;
+                const parentContent = this.metadataString(firstChunkMeta?.parentText) || group.map(chunk => chunk.text).join('\n\n---\n\n');
                 parentContents.push(parentContent);
-                insertParent.run(parentId, relativePath, parentId, parentContent, file.category, file.format, file.collectionName, group.find(chunk => chunk.sectionTitle)?.sectionTitle ?? null, group.length, JSON.stringify({ parentId, splitStrategy: this.metadataString(group[0]?.metadata.splitStrategy), chunkKind: this.metadataString(group[0]?.metadata.chunkKind), titlePath: this.metadataString(group[0]?.metadata.titlePath) }), now);
+                // 我们不想把一整个大文本冗余在每个切片的 metadata 里，所以存完 parent 之后清理一下
+                for (const chunk of group) {
+                    delete chunk.metadata.parentText;
+                }
+                insertParent.run(parentId, relativePath, parentId, parentContent, file.category, file.format, file.collectionName, group.find(chunk => chunk.sectionTitle)?.sectionTitle ?? null, group.length, JSON.stringify({ parentId, splitStrategy: this.metadataString(firstChunkMeta?.splitStrategy), chunkKind: this.metadataString(firstChunkMeta?.chunkKind), titlePath: this.metadataString(firstChunkMeta?.titlePath) }), now);
             }
             insertDocument.run(`${relativePath}#document`, relativePath, parentContents.join('\n\n=== SECTION ===\n\n'), file.category, file.format, file.collectionName, parentGroups.size, groupedChunks.length, JSON.stringify({ parentType: 'document', splitStrategy: 'document_section_child_v1' }), now);
             for (const chunk of groupedChunks) {

package/dist/core/knowledge-base-manager.d.ts

@@ -158,11 +158,11 @@ export declare class KnowledgeBaseManager {
     private llmExpandQueries;
     private retrievalWeights;
     private heuristicRerank;
-    private llmRerank;
     private hydrateVectorResultsFromSqlite;
     private toFederatedItem;
     private mergeHybridRankedLists;
     private mergeContexts;
+    private mergeExpandedContexts;
     private contextKey;
     private parseChunkIndex;
     private parseMetadataString;

package/dist/core/knowledge-base-manager.js

@@ -7,6 +7,7 @@ import { ALL_CATEGORIES, DEFAULT_CATEGORY_DIRS, GLOBAL_KNOWLEDGE_DIR, USER_DATA_
 import { DedupEngine } from '../dedup/dedup-engine.js';
 import { RelationshipDetector } from '../dedup/relationship-detector.js';
 import { createEmbeddingProviderFromEnvironment } from '../embedding/embedding-provider.js';
+import { LocalReranker } from '../embedding/local-reranker.js';
 import { ContentExtractor } from '../extraction/content-extractor.js';
 import { FederationSearch } from '../search/federation-search.js';
 import { CollectionManager } from '../vector/collection-manager.js';
@@ -350,20 +351,47 @@ export class KnowledgeBaseManager {
         }
         const keywordItems = rankedLists.filter(list => list.source === 'keyword').flatMap(list => list.items);
         const vectorItems = rankedLists.filter(list => list.source === 'vector').flatMap(list => list.items);
-        const merged = this.mergeContexts(this.mergeHybridRankedLists(rankedLists, limit * 4, weights).map(item => this.expandContext(item)), limit * 2);
-        const useLLMRerank = !!this.llmProvider;
-        const preReranked = useLLMRerank ? merged : this.heuristicRerank(query, merged);
-        const reranked = useLLMRerank ? (await this.llmRerank(query, preReranked)) : preReranked;
+        // 1. 先进行初筛合并，合并相同的子块并计算混合初始分（不获取大片段，保留子块自身用于精确打分）
+        const mergedChildChunks = this.mergeContexts(this.mergeHybridRankedLists(rankedLists, limit * 4, weights), limit * 4);
+        // 2. 对这些子块进行交叉编码器重排（Cross-Encoder Rerank）
+        let reranked = mergedChildChunks;
+        let rerankerName = 'local-heuristic-fallback';
+        if (mergedChildChunks.length > 0) {
+            const candidates = mergedChildChunks.slice(0, Math.min(30, limit * 4));
+            // 这里使用的是子块自身内容，通常在 500 tokens 左右，不仅相关性判断最准，而且不会超出 Reranker 的 max_length
+            const textsToRerank = candidates.map(item => `${item.titlePath ?? item.sectionTitle ?? ''}\n${item.content}`);
+            try {
+                const scores = await LocalReranker.rerank(query, textsToRerank);
+                const usableScores = scores.length === candidates.length && scores.some(score => Number.isFinite(score) && score > 0);
+                if (usableScores) {
+                    reranked = candidates.map((item, i) => ({
+                        ...item,
+                        score: scores[i] ?? item.score,
+                        scoreDetails: { ...item.scoreDetails, crossEncoderScore: scores[i] ?? 0 }
+                    })).sort((a, b) => b.score - a.score);
+                    rerankerName = 'bge-reranker-base';
+                }
+                else {
+                    reranked = this.heuristicRerank(query, mergedChildChunks);
+                    rerankerName = 'local-heuristic-fallback-empty-rerank';
+                }
+            }
+            catch {
+                reranked = this.heuristicRerank(query, mergedChildChunks);
+            }
+        }
+        // 3. 拿到精确打分后的 Top 结果，此时再进行 expandContext 向上追溯到完整的父块大片段
+        const finalExpandedResults = this.mergeExpandedContexts(reranked.map(item => this.expandContext(item)), limit);
         return {
-            results: reranked.slice(0, limit),
+            results: finalExpandedResults,
             scopesSearched: this.scope === 'global' ? ['global'] : ['project'],
             queryTimeMs: Date.now() - start,
             debug: {
                 originalQuery: query,
                 rewrittenQueries,
                 weights,
-                recallCounts: { keyword: keywordItems.length, vector: vectorItems.length, merged: merged.length },
-                reranker: useLLMRerank ? 'llm-semantic-reranker-v1' : 'local-statistical-reranker-v1',
+                recallCounts: { keyword: keywordItems.length, vector: vectorItems.length, merged: mergedChildChunks.length },
+                reranker: rerankerName,
             },
         };
     }
@@ -697,59 +725,6 @@ export class KnowledgeBaseManager {
             };
         }).sort((a, b) => b.score - a.score);
     }
-    async llmRerank(query, items) {
-        if (!this.llmProvider || items.length === 0)
-            return this.heuristicRerank(query, items);
-        const candidates = items.slice(0, 20);
-        const resultsText = candidates.map((item, index) => {
-            const contentPreview = item.content.slice(0, 300).replace(/[\n\r]+/g, ' ');
-            return `[DOC_${index}] 路径: ${item.filePath} | 标题路径: ${item.titlePath ?? item.sectionTitle ?? ''} | 类型: ${item.chunkKind ?? 'text'}\n  内容: ${contentPreview}`;
-        }).join('\n\n');
-        const prompt = `你是一个文档相关性评估器。根据用户查询，为以下文档片段打分（1-10）。
-1=完全不相关，10=高度相关。
-输出格式：每行 "DOC_ID:分数"，如 "DOC_0:8"
-
-查询：${query}
-
-${resultsText}
-
-相关性评分：`;
-        try {
-            const response = await this.llmProvider.chat([
-                { role: 'system', content: '你是一个精确的文档相关性评估器。只输出 DOC_ID:分数的列表。' },
-                { role: 'user', content: prompt },
-            ], { temperature: 0.1, maxTokens: 600 });
-            const scoreMap = new Map();
-            for (const line of response.content.split('\n')) {
-                const match = line.match(/DOC[_\s]*(\d+)[^\d]*(\d+)/i);
-                if (match)
-                    scoreMap.set(Number(match[1]), Math.min(10, Math.max(1, Number(match[2]))));
-            }
-            if (scoreMap.size === 0)
-                return this.heuristicRerank(query, items);
-            return items.map((item, index) => {
-                const llmScore = scoreMap.get(index);
-                if (llmScore == null || llmScore === undefined)
-                    return item;
-                // LLM 分数 (1-10) 映射为权重因子：10→2.0x, 5→1.0x, 1→0.2x
-                const llmFactor = 0.2 + (llmScore / 10) * 1.8;
-                const newScore = item.score * llmFactor;
-                return {
-                    ...item,
-                    score: newScore,
-                    scoreDetails: {
-                        ...item.scoreDetails,
-                        rerankBoost: newScore - (item.scoreDetails?.hybridScore ?? item.score),
-                        llmRelevanceScore: llmScore,
-                        hybridScore: newScore,
-                    },
-                };
-            }).sort((a, b) => b.score - a.score);
-        }
-        catch {
-            return this.heuristicRerank(query, items);
-        }
-    }
     hydrateVectorResultsFromSqlite(items) {
         return items.map(item => {
             if (!item.rowid)
@@ -846,6 +821,20 @@ ${resultsText}
         }
         return [...byKey.values()].sort((a, b) => b.score - a.score).slice(0, limit);
     }
+    mergeExpandedContexts(items, limit) {
+        const byKey = new Map();
+        for (const item of items) {
+            const key = `${item.scope}:${item.filePath}:${item.parentId ?? item.chunkIndex ?? item.id}`;
+            const existing = byKey.get(key);
+            if (!existing || item.score > existing.score) {
+                byKey.set(key, item);
+                continue;
+            }
+            existing.source = existing.source === item.source ? existing.source : 'hybrid';
+            existing.scoreDetails = { ...existing.scoreDetails, ...item.scoreDetails, hybridScore: existing.score };
+        }
+        return [...byKey.values()].sort((a, b) => b.score - a.score).slice(0, limit);
+    }
     contextKey(item) {
         if (item.rowid)
             return `${item.scope}:rowid:${item.rowid}`;

package/dist/embedding/local-reranker.d.ts

@@ -0,0 +1,13 @@
+export declare class LocalReranker {
+    private static instance;
+    private static loadingPromise;
+    private static disabledUntil;
+    private static modelName;
+    static getInstance(): Promise<any>;
+    /**
+     * 对多条文本和查询进行相关性重排
+     */
+    static rerank(query: string, texts: string[]): Promise<number[]>;
+    private static takeHeadTail;
+    private static extractScore;
+}

package/dist/embedding/local-reranker.js

@@ -0,0 +1,73 @@
+import { pipeline } from '@huggingface/transformers';
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+export class LocalReranker {
+    static instance = null;
+    static loadingPromise = null;
+    static disabledUntil = 0;
+    static modelName = process.env.KB_RERANKER_MODEL || 'Xenova/bge-reranker-base';
+    static async getInstance() {
+        if (this.instance)
+            return this.instance;
+        if (Date.now() < this.disabledUntil)
+            throw new Error('Local reranker is temporarily disabled after load failure');
+        if (process.env.KB_ENABLE_LOCAL_RERANKER === 'false')
+            throw new Error('Local reranker is disabled');
+        // 使用 loadingPromise 防止高并发下的重复加载
+        if (this.loadingPromise)
+            return this.loadingPromise;
+        this.loadingPromise = (async () => {
+            const cacheDir = process.env.TRANSFORMERS_CACHE || path.join(os.homedir(), '.customize-agent', 'models');
+            if (!fs.existsSync(cacheDir)) {
+                fs.mkdirSync(cacheDir, { recursive: true });
+            }
+            process.env.TRANSFORMERS_CACHE = cacheDir;
+            const pipe = await pipeline('text-classification', this.modelName, {
+                dtype: 'q8',
+            });
+            this.instance = pipe;
+            return pipe;
+        })().catch(error => {
+            this.loadingPromise = null;
+            this.disabledUntil = Date.now() + 60_000;
+            throw error;
+        });
+        return this.loadingPromise;
+    }
+    /**
+     * 对多条文本和查询进行相关性重排
+     */
+    static async rerank(query, texts) {
+        if (!texts.length)
+            return [];
+        const ranker = await this.getInstance();
+        const scores = [];
+        const safeQuery = this.takeHeadTail(query, 240);
+        for (const text of texts) {
+            try {
+                const safeText = this.takeHeadTail(text, 1400);
+                const out = await ranker({ text: safeQuery, text_pair: safeText });
+                scores.push(this.extractScore(out));
+            }
+            catch {
+                scores.push(0);
+            }
+        }
+        return scores;
+    }
+    static takeHeadTail(text, maxLength) {
+        if (text.length <= maxLength)
+            return text;
+        const headLength = Math.ceil(maxLength * 0.65);
+        const tailLength = maxLength - headLength;
+        return `${text.slice(0, headLength)}\n...\n${text.slice(-tailLength)}`;
+    }
+    static extractScore(output) {
+        const first = Array.isArray(output) ? output[0] : output;
+        if (Array.isArray(first))
+            return this.extractScore(first);
+        const score = Number(first?.score ?? 0);
+        return Number.isFinite(score) ? score : 0;
+    }
+}

package/dist/extraction/content-extractor.d.ts

@@ -57,6 +57,7 @@ export declare class ContentExtractor {
     /** pdfjs-dist + canvas 渲染（降级方案） */
     private tryRenderWithPdfJs;
     private scoreOcrText;
+    private cleanOcrText;
     /** 加载图片像素数据（依赖 sharp） */
     private loadImagePixels;
     private extractPdfText;

package/dist/extraction/content-extractor.js

@@ -776,8 +776,13 @@ export class ContentExtractor {
                 }
                 if (matrix.length > 0) {
                     const header = matrix[0] ?? [];
-                    const rows = matrix.slice(1);
-                    sheetTexts.push([`工作表：${name}`, this.toMarkdownTable(header, rows)].join('\n\n'));
+                    // 表格分页：为了防止超大 Excel 导致单块过大，将其每 500 行分为一个独立的 Markdown Table。
+                    const chunkSize = 500;
+                    for (let i = 1; i < matrix.length; i += chunkSize) {
+                        const rows = matrix.slice(i, i + chunkSize);
+                        const sheetSuffix = matrix.length > chunkSize ? ` (第 ${Math.floor(i / chunkSize) + 1} 部分)` : '';
+                        sheetTexts.push([`工作表：${name}${sheetSuffix}`, this.toMarkdownTable(header, rows)].join('\n\n'));
+                    }
                 }
             }
             if (sheetTexts.length > 0) {
@@ -804,17 +809,47 @@ export class ContentExtractor {
         const docXml = await zip.files['word/document.xml']?.async('string');
         if (!docXml)
             return '';
-        const paragraphs = Array.from(docXml.matchAll(/<w:p[\s\S]*?<\/w:p>/gu), match => match[0]);
+        // 我们需要按出现顺序提取 paragraph 和 table
+        // 在 xml 中 w:body 的一级子节点通常是 w:p 和 w:tbl
+        const elements = Array.from(docXml.matchAll(/<(w:p|w:tbl)[\s>][\s\S]*?<\/\1>/gu), match => ({ tag: match[1], xml: match[0] }));
         const lines = [];
-        for (const paragraph of paragraphs) {
-            const texts = Array.from(paragraph.matchAll(/<w:t[^>]*>([\s\S]*?)<\/w:t>/gu), match => this.stripXml(match[1] ?? '')).join('');
-            if (!texts.trim())
-                continue;
-            const style = /<w:pStyle\s+w:val="([^"]+)"/u.exec(paragraph)?.[1] ?? '';
-            const bold = /<w:b\b/u.test(paragraph);
-            const size = Number(/<w:sz\s+w:val="(\d+)"/u.exec(paragraph)?.[1] ?? 0);
-            const level = this.docxHeadingLevel(style, bold, size, texts);
-            lines.push(`${level > 0 ? `${'#'.repeat(level)} ` : ''}${texts.trim()}`);
+        for (const { tag, xml } of elements) {
+            if (tag === 'w:p') {
+                const texts = Array.from(xml.matchAll(/<w:t[^>]*>([\s\S]*?)<\/w:t>/gu), match => this.stripXml(match[1] ?? '')).join('');
+                if (!texts.trim())
+                    continue;
+                const style = /<w:pStyle\s+w:val="([^"]+)"/u.exec(xml)?.[1] ?? '';
+                const bold = /<w:b\b/u.test(xml);
+                const size = Number(/<w:sz\s+w:val="(\d+)"/u.exec(xml)?.[1] ?? 0);
+                const level = this.docxHeadingLevel(style, bold, size, texts);
+                lines.push(`${level > 0 ? `${'#'.repeat(level)} ` : ''}${texts.trim()}`);
+            }
+            else if (tag === 'w:tbl') {
+                const trList = Array.from(xml.matchAll(/<w:tr[\s>][\s\S]*?<\/w:tr>/gu), match => match[0]);
+                const rows = [];
+                for (const tr of trList) {
+                    const tcList = Array.from(tr.matchAll(/<w:tc[\s>][\s\S]*?<\/w:tc>/gu), match => match[0]);
+                    const cols = [];
+                    for (const tc of tcList) {
+                        const tcText = Array.from(tc.matchAll(/<w:t[^>]*>([\s\S]*?)<\/w:t>/gu), match => this.stripXml(match[1] ?? '')).join('');
+                        cols.push(tcText.trim().replace(/\r?\n/gu, ' '));
+                    }
+                    rows.push(cols);
+                }
+                if (rows.length > 0) {
+                    const maxCols = Math.max(...rows.map(r => r.length));
+                    // 补齐列数
+                    rows.forEach(r => { while (r.length < maxCols)
+                        r.push(''); });
+                    const header = rows[0] || Array(maxCols).fill('');
+                    const mdTable = [
+                        `| ${header.join(' | ')} |`,
+                        `| ${header.map(() => '---').join(' | ')} |`,
+                        ...rows.slice(1).map(row => `| ${row.join(' | ')} |`)
+                    ].join('\n');
+                    lines.push(mdTable);
+                }
+            }
         }
         return lines.join('\n\n');
     }
@@ -1065,7 +1100,9 @@ export class ContentExtractor {
                     width: 0, height: 0, channels: 0,
                     filePath: imgPath,
                 });
-                const ocrText = ocrResult.text.trim();
+                const ocrText = this.cleanOcrText(ocrResult.text);
+                if (ocrResult.warnings?.length)
+                    warnings.push(...ocrResult.warnings.map(item => `OCR 警告: ${item}`));
                 if (ocrText) {
                     ocrPages.push(i + 1);
                     ocrStrategies.push({ page: i + 1, strategy: renderer, score: this.scoreOcrText(ocrText) });
@@ -1079,6 +1116,8 @@ export class ContentExtractor {
                 failedPages.push({ page: i + 1, reason: error instanceof Error ? error.message : String(error) });
             }
         }
+        if (ocrProvider)
+            await ocrProvider.dispose();
         // 清理临时文件
         try {
             fs.rmSync(tmpDir, { recursive: true, force: true });
@@ -1164,6 +1203,15 @@ export class ContentExtractor {
         const replacementCount = (text.match(/[�□]/gu) ?? []).length;
         return cjkCount * 8 + normalizedLength - latinCount * 0.8 - replacementCount * 10;
     }
+    cleanOcrText(value) {
+        return String(value ?? '')
+            .replace(/[ \t]+/gu, ' ')
+            .replace(/([\p{Script=Han}])\s+([\p{Script=Han}])/gu, '$1$2')
+            .replace(/([\p{Script=Han}])\s+([，。；：！？、）】》])/gu, '$1$2')
+            .replace(/([（【《])\s+([\p{Script=Han}])/gu, '$1$2')
+            .replace(/\n{3,}/gu, '\n\n')
+            .trim();
+    }
     /** 加载图片像素数据（依赖 sharp） */
     async loadImagePixels(filePath) {
         const sharpMod = await resolveAndImport('sharp');

package/dist/extraction/ocr-providers.d.ts

@@ -18,6 +18,7 @@ export interface OcrResult {
     text: string;
     confidence: number;
     regions: OcrRegion[];
+    warnings?: string[];
 }
 export interface OcrProvider {
     readonly id: string;
@@ -29,11 +30,15 @@ export interface OcrProvider {
         channels?: number;
         filePath?: string;
     }): Promise<OcrResult>;
+    getWarnings?(): string[];
     dispose(): Promise<void>;
 }
 export declare class TesseractJsProvider implements OcrProvider {
     readonly id = "tesseract.js";
     private _available;
+    private worker;
+    private workerPromise;
+    private warnings;
     get available(): boolean;
     recognize(input: {
         data: Uint8Array;
@@ -42,6 +47,9 @@ export declare class TesseractJsProvider implements OcrProvider {
         channels?: number;
         filePath?: string;
     }): Promise<OcrResult>;
+    getWarnings(): string[];
+    private getWorker;
+    private createReusableWorker;
     dispose(): Promise<void>;
 }
 export declare function createOcrProvider(): Promise<OcrProvider>;

package/dist/extraction/ocr-providers.js

@@ -23,6 +23,9 @@ function tessdataDir() {
 export class TesseractJsProvider {
     id = 'tesseract.js';
     _available = null;
+    worker = null;
+    workerPromise = null;
+    warnings = [];
     get available() {
         if (this._available !== null)
             return this._available;
@@ -39,8 +42,6 @@ export class TesseractJsProvider {
         return this._available;
     }
     async recognize(input) {
-        const tessMod = await resolveAndImport('tesseract.js');
-        const { createWorker } = tessMod;
         let pngPath;
         let tmpDir = null;
         // 如果传了 filePath，直接使用；否则 raw pixels → PNG
@@ -61,39 +62,66 @@ export class TesseractJsProvider {
                 .png().toFile(pngPath);
         }
         try {
-            // tesseract.js worker（数组形式 languages）
-            const worker = await createWorker(['chi_sim', 'eng'], 1, {
-                langPath: tessdataDir(),
-                gzip: false,
-            });
-            try {
-                const result = await worker.recognize(pngPath);
-                const text = (result.data.text ?? '').trim();
-                const lines = (result.data.lines ?? []);
-                const regions = lines
-                    .filter((l) => l.text?.trim())
-                    .map((l) => ({
-                    text: l.text.trim(),
-                    confidence: l.confidence ?? 0,
-                    box: {
-                        x: l.bbox?.x0 ?? 0,
-                        y: l.bbox?.y0 ?? 0,
-                        width: (l.bbox?.x1 ?? 0) - (l.bbox?.x0 ?? 0),
-                        height: (l.bbox?.y1 ?? 0) - (l.bbox?.y0 ?? 0),
-                    },
-                }));
-                return { text, confidence: result.data.confidence ?? 0, regions };
-            }
-            finally {
-                await worker.terminate();
-            }
+            const worker = await this.getWorker();
+            const result = await worker.recognize(pngPath);
+            const text = (result.data.text ?? '').trim();
+            const lines = (result.data.lines ?? []);
+            const regions = lines
+                .filter((l) => l.text?.trim())
+                .map((l) => ({
+                text: l.text.trim(),
+                confidence: l.confidence ?? 0,
+                box: {
+                    x: l.bbox?.x0 ?? 0,
+                    y: l.bbox?.y0 ?? 0,
+                    width: (l.bbox?.x1 ?? 0) - (l.bbox?.x0 ?? 0),
+                    height: (l.bbox?.y1 ?? 0) - (l.bbox?.y0 ?? 0),
+                },
+            }));
+            return { text, confidence: result.data.confidence ?? 0, regions, warnings: this.getWarnings() };
         }
         finally {
             if (tmpDir)
                 fs.rmSync(tmpDir, { recursive: true, force: true });
         }
     }
-    async dispose() { }
+    getWarnings() {
+        return [...new Set(this.warnings)].slice(-20);
+    }
+    async getWorker() {
+        if (this.worker)
+            return this.worker;
+        if (!this.workerPromise)
+            this.workerPromise = this.createReusableWorker();
+        this.worker = await this.workerPromise;
+        return this.worker;
+    }
+    async createReusableWorker() {
+        const tessMod = await resolveAndImport('tesseract.js');
+        const { createWorker, OEM, setLogging } = tessMod;
+        if (typeof setLogging === 'function')
+            setLogging(false);
+        const worker = await createWorker('chi_sim', OEM?.LSTM_ONLY ?? 1, {
+            langPath: tessdataDir(),
+            gzip: false,
+            logger: () => undefined,
+            errorHandler: (error) => {
+                const message = error instanceof Error ? error.message : String(error);
+                if (message.trim())
+                    this.warnings.push(message.trim());
+            },
+        });
+        if (typeof worker.setParameters === 'function') {
+            await worker.setParameters({ preserve_interword_spaces: '0', user_defined_dpi: '300' });
+        }
+        return worker;
+    }
+    async dispose() {
+        if (this.worker)
+            await this.worker.terminate();
+        this.worker = null;
+        this.workerPromise = null;
+    }
 }
 // ─── 工厂 ───────────────────────────────────────────────────────
 export async function createOcrProvider() {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@customize-agent/knowledge",
-  "version": "4.0.15",
+  "version": "4.0.17",
   "description": "Local knowledge base infrastructure for customize-agent",
   "type": "module",
   "main": "./dist/index.js",