@customize-agent/knowledge 4.0.40 → 4.0.41

package/dist/core/knowledge-base-manager.d.ts

@@ -75,6 +75,7 @@ export declare class KnowledgeBaseManager {
         collections?: string[];
         weights?: RetrievalWeights;
         generationMode?: boolean;
+        disableReranker?: boolean;
     }): Promise<FederatedResult>;
     semanticSearch(query: string, options?: {
         limit?: number;
@@ -83,6 +84,10 @@ export declare class KnowledgeBaseManager {
     }): Promise<FederatedResult>;
     listRelationships(filePath?: string): FileRelationship[];
     listFiles(): IndexStateRecord[];
+    listChunks(options?: {
+        relativePath?: string;
+        limit?: number;
+    }): import("./index-state-store.js").StoredChunk[];
     getFileDetail(relativePath: string, options?: {
         maxChunkContentChars?: number;
     }): {
@@ -164,6 +169,10 @@ export declare class KnowledgeBaseManager {
     private llmExpandQueries;
     private retrievalWeights;
     private heuristicRerank;
+    private normalizeSearchText;
+    private isLowQualityCadText;
+    private queryTerms;
+    private queryFactLabels;
     private hydrateVectorResultsFromSqlite;
     private toFederatedItem;
     private mergeHybridRankedLists;

package/dist/core/knowledge-base-manager.js

@@ -403,7 +403,11 @@ export class KnowledgeBaseManager {
         // 2. 对这些子块进行交叉编码器重排（Cross-Encoder Rerank）
         let reranked = mergedChildChunks;
         let rerankerName = 'local-heuristic-fallback';
-        if (mergedChildChunks.length > 0) {
+        if (options.disableReranker) {
+            reranked = this.heuristicRerank(query, mergedChildChunks);
+            rerankerName = 'local-heuristic-disabled-reranker';
+        }
+        else if (mergedChildChunks.length > 0) {
             const rerankLimit = requestedLimit ? Math.min(30, mergeLimit) : mergedChildChunks.length;
             const candidates = mergedChildChunks.slice(0, rerankLimit);
             // 这里使用的是子块自身内容，通常在 500 tokens 左右，不仅相关性判断最准，而且不会超出 Reranker 的 max_length
@@ -470,6 +474,9 @@ export class KnowledgeBaseManager {
     listFiles() {
         return this.store.listRecords();
     }
+    listChunks(options = {}) {
+        return this.store.listChunks(options);
+    }
     getFileDetail(relativePath, options = {}) {
         const normalized = this.normalizeRelativePath(relativePath);
         const file = this.store.listRecords().find(record => record.relativePath === normalized);
@@ -746,28 +753,63 @@ export class KnowledgeBaseManager {
         };
     }
     heuristicRerank(query, items) {
-        const terms = query.toLowerCase().split(/[\s,，。；;：:、]+/u).filter(Boolean);
+        const terms = this.queryTerms(query);
         const phrase = query.toLowerCase().trim();
+        const normalizedPhrase = this.normalizeSearchText(query);
+        const factLabels = this.queryFactLabels(query);
+        const wantsTable = /表|行|列|金额|数量|报价|评分|清单|明细|统计|数据/u.test(query);
+        const wantsDrawing = /图纸|图层|轴网|标注|块|实体|cad|dwg|dxf|step|iges|模型/u.test(query);
+        const wantsData = /json|xml|yaml|字段|配置|数据|路径|price|id|name/u.test(query);
         return items.map(item => {
-            const content = `${item.filePath}\n${item.titlePath ?? ''}\n${item.sectionTitle ?? ''}\n${item.chunkKind ?? ''}\n${item.content}`.toLowerCase();
+            const rawContent = `${item.filePath}\n${item.titlePath ?? ''}\n${item.sectionTitle ?? ''}\n${item.chunkKind ?? ''}\n${item.content}`;
+            const content = rawContent.toLowerCase();
+            const normalizedContent = this.normalizeSearchText(rawContent);
             let rerankBoost = 0;
             if (phrase && content.includes(phrase))
-                rerankBoost += 120;
+                rerankBoost += 160;
+            if (normalizedPhrase.length >= 4 && normalizedContent.includes(normalizedPhrase))
+                rerankBoost += 220;
             const titleText = `${item.titlePath ?? ''}\n${item.sectionTitle ?? ''}`.toLowerCase();
+            const normalizedTitle = this.normalizeSearchText(titleText);
+            let matchedTermCount = 0;
             for (const term of terms) {
                 if (!term)
                     continue;
-                if (content.includes(term))
-                    rerankBoost += 8;
-                if (titleText.includes(term))
-                    rerankBoost += 18;
+                const normalizedTerm = this.normalizeSearchText(term);
+                const matchedContent = content.includes(term) || normalizedContent.includes(normalizedTerm);
+                const matchedTitle = titleText.includes(term) || normalizedTitle.includes(normalizedTerm);
+                if (matchedContent) {
+                    matchedTermCount += 1;
+                    rerankBoost += 12;
+                }
+                if (matchedTitle)
+                    rerankBoost += 22;
+            }
+            if (matchedTermCount >= 2)
+                rerankBoost += matchedTermCount * 18;
+            for (const label of factLabels) {
+                const normalizedLabel = this.normalizeSearchText(label);
+                if (normalizedContent.includes(normalizedLabel))
+                    rerankBoost += 80;
+                if (normalizedTitle.includes(normalizedLabel))
+                    rerankBoost += 120;
             }
-            if (item.chunkKind === 'table' && /表|行|列|金额|数量|报价|评分|清单|明细|统计|数据/u.test(query))
-                rerankBoost += 40;
-            if (item.chunkKind === 'metadata' && /图纸|图层|轴网|标注|块|实体|cad|dxf|step|iges|模型/u.test(query))
-                rerankBoost += 60;
-            if (item.chunkKind === 'data' && /json|xml|yaml|字段|配置|数据|路径|price|id|name/u.test(query))
-                rerankBoost += 30;
+            if (item.chunkKind === 'table')
+                rerankBoost += wantsTable ? 25 : -80;
+            if (item.chunkKind === 'metadata')
+                rerankBoost += wantsDrawing ? 60 : -100;
+            if (item.chunkKind === 'data')
+                rerankBoost += wantsData ? 30 : -12;
+            if (/\.(?:dwg|dxf)(?:$|[?#])/iu.test(item.filePath)) {
+                rerankBoost += wantsDrawing ? 120 : -60;
+                if (this.isLowQualityCadText(item.content))
+                    rerankBoost -= wantsDrawing ? 60 : 160;
+            }
+            if (/工作表：|COL\d+|专业工程暂估价计价表|分部分项工程量清单计价表|材料（工程设备）暂估单价一览表/u.test(item.content)) {
+                rerankBoost -= wantsTable ? 35 : 140;
+            }
+            if (/第\s*\d+\s*页\s*共\s*\d+\s*页|PDF\s*第\s*\d+\s*页/iu.test(item.content) && factLabels.length > 0)
+                rerankBoost -= 12;
             const score = item.score + rerankBoost;
             return {
                 ...item,
@@ -780,6 +822,47 @@ export class KnowledgeBaseManager {
             };
         }).sort((a, b) => b.score - a.score);
     }
+    normalizeSearchText(value) {
+        return value.toLowerCase().replace(/\s+/gu, '');
+    }
+    isLowQualityCadText(value) {
+        const compact = value.replace(/\s+/gu, '');
+        if (compact.length === 0)
+            return true;
+        const readable = compact.match(/[\u4e00-\u9fa5A-Za-z0-9（）()【】《》、，。；;：:,.\-/㎡%]/gu)?.length || 0;
+        return readable / compact.length < 0.55;
+    }
+    queryTerms(query) {
+        const base = query.toLowerCase().split(/[\s,，。；;：:、]+/u).filter(Boolean);
+        const labels = this.queryFactLabels(query).map(label => label.toLowerCase());
+        return [...new Set([...base, ...labels].filter(term => term.length > 0))];
+    }
+    queryFactLabels(query) {
+        const labels = [];
+        if (/建设地点|工程地点|项目地点|地点|在哪里|位于/u.test(query))
+            labels.push('建设地点', '工程地点', '项目地点', '项目位于', '位于');
+        if (/出资比例|资金比例|资金来源|出资/u.test(query))
+            labels.push('项目出资比例', '出资比例', '资金来源', '资金落实情况');
+        if (/项目名称|招标项目名称|工程名称/u.test(query))
+            labels.push('招标项目名称', '工程名称', '项目名称');
+        if (/项目编号|招标编号/u.test(query))
+            labels.push('招标项目编号', '项目编号');
+        if (/工期|日历天|计划开工|计划竣工/u.test(query))
+            labels.push('计划工期', '总工期', '工期', '计划开工日期', '计划竣工日期');
+        if (/质量标准|质量要求|合格/u.test(query))
+            labels.push('质量标准', '质量要求');
+        if (/招标范围|工程范围|承包范围|施工范围/u.test(query))
+            labels.push('招标范围', '工程承包范围', '施工范围');
+        if (/建设单位|招标人|项目业主/u.test(query))
+            labels.push('招标人', '项目业主', '建设单位');
+        if (/临水|临电|临时水电|水电接引|接驳点|挂表计量|施工水电/u.test(query))
+            labels.push('临水临电', '临时水电接引', '施工水电接引费', '接驳点挂表计量', '挂表计量');
+        if (/场地限制|材料堆场|办公区|生活区|加工区/u.test(query))
+            labels.push('场地限制', '不具备材料堆场', '搭设加工区', '搭设办公区', '搭设生活区');
+        if (/拆除|修补|破损处|改造维修/u.test(query))
+            labels.push('改造维修项目', '拆除内容比较多', '破损处进行修补');
+        return [...new Set(labels)];
+    }
     hydrateVectorResultsFromSqlite(items) {
         return items.map(item => {
             if (!item.rowid)

package/dist/core/multi-project-manager.d.ts

@@ -27,6 +27,7 @@ export declare class MultiProjectManager {
         filters?: SearchFilters;
         weights?: RetrievalWeights;
         generationMode?: boolean;
+        disableReranker?: boolean;
     }): Promise<FederatedResult>;
     semanticSearch(projectRoot: string, query: string, options?: {
         limit?: number;

package/dist/core/multi-project-manager.js

@@ -63,16 +63,16 @@ export class MultiProjectManager {
         const scope = options.scope ?? 'project';
         const project = await this.getProject(projectRoot);
         if (scope === 'project')
-            return project.hybridSearch(query, { limit, filters: options.filters, weights: options.weights, generationMode: options.generationMode });
+            return project.hybridSearch(query, { limit, filters: options.filters, weights: options.weights, generationMode: options.generationMode, disableReranker: options.disableReranker });
         const projectResults = scope === 'all'
-            ? await project.hybridSearch(query, { limit, filters: options.filters, weights: options.weights, generationMode: options.generationMode })
+            ? await project.hybridSearch(query, { limit, filters: options.filters, weights: options.weights, generationMode: options.generationMode, disableReranker: options.disableReranker })
             : { results: [], scopesSearched: [], queryTimeMs: 0 };
         if (scope === 'global') {
             const global = await this.getGlobalKB();
-            return global.hybridSearch(query, { limit, filters: options.filters, weights: options.weights, generationMode: options.generationMode });
+            return global.hybridSearch(query, { limit, filters: options.filters, weights: options.weights, generationMode: options.generationMode, disableReranker: options.disableReranker });
         }
         const global = await this.getGlobalKB();
-        const globalResults = await global.hybridSearch(query, { limit, filters: options.filters, weights: options.weights, generationMode: options.generationMode });
+        const globalResults = await global.hybridSearch(query, { limit, filters: options.filters, weights: options.weights, generationMode: options.generationMode, disableReranker: options.disableReranker });
         const mergeLimit = limit ?? (projectResults.results.length + globalResults.results.length);
         const merged = new FederationSearch().merge([...projectResults.results, ...globalResults.results], mergeLimit, 'all');
         return {

package/dist/extraction/content-extractor.d.ts

@@ -13,6 +13,7 @@ export declare class ContentExtractor {
     private swapUtf16Bytes;
     private extractReadableFragments;
     private cleanCadReadableText;
+    private isLikelyGarbledCadText;
     private isReadableCadValue;
     private cleanExtractedText;
     private textScore;

package/dist/extraction/content-extractor.js

@@ -7,6 +7,7 @@ import { resolveAndImport, resolvePackage } from './module-resolver.js';
 import { createOcrProvider } from './ocr-providers.js';
 const CAD_INTERNAL_TOKEN_RE = /\b(?:TDbPipe|TDbPipeValve|TDbPipeFitting|TDbWellh|AcDb\w+|Dwg\w+|ObjectId|Handle|ByLayer|Continuous|Model|Layout\d*)\b/giu;
 const CAD_INTERNAL_LINE_RE = /^(?:TDb\w+|AcDb\w+|[A-F0-9]{8,}|\d+|Model|Layout\d*|ByLayer|Continuous)$/iu;
+const CAD_DOMAIN_SIGNAL_RE = /工程|项目|施工|建筑|结构|装饰|电气|给排水|消防|暖通|平面|立面|剖面|节点|详图|材料|尺寸|标高|轴线|图层|门窗|墙|地面|顶面|照明|配电|弱电|空调|卫生间|楼梯|屋面|基础|柱|梁|板|图号|设计|说明/u;
 const OCR_NATIVE_NOISE_PATTERNS = [/^Image too small to scale!!/u, /^Line cannot be recognized!!$/u];
 /** 文件内容提取器，支持文档、表格、图片、CAD 等多种文件格式的内容抽取 */
 export class ContentExtractor {
@@ -146,9 +147,34 @@ export class ContentExtractor {
             .replace(/\s+/gu, ' ')
             .trim();
     }
+    isLikelyGarbledCadText(value) {
+        const compact = value.replace(/\s+/gu, '');
+        if (!compact)
+            return true;
+        const chars = [...compact];
+        const readable = chars.filter(char => /[\p{Script=Han}\p{Script=Latin}\d（）()【】《》、，。；;：:,.\-/㎡%]/u.test(char)).length;
+        const cjk = chars.filter(char => /[\p{Script=Han}]/u.test(char)).length;
+        const latin = chars.filter(char => /[\p{Script=Latin}]/u.test(char)).length;
+        const digits = chars.filter(char => /\d/u.test(char)).length;
+        const symbols = chars.length - readable;
+        const readableRatio = readable / chars.length;
+        const symbolRatio = symbols / chars.length;
+        const hasDomainSignal = CAD_DOMAIN_SIGNAL_RE.test(compact);
+        const hasCommonTextShape = /[，。；：、,.\-/()（）]|\d+(?:\.\d+)?\s*(?:mm|cm|m|㎡|%|°)?/iu.test(compact);
+        const latinVowelCount = chars.filter(char => /[aAeEiIoOuU]/u.test(char)).length;
+        if (readableRatio < 0.6)
+            return true;
+        if (symbolRatio > 0.35 && !hasDomainSignal)
+            return true;
+        if (latin >= 12 && latinVowelCount === 0 && !hasDomainSignal)
+            return true;
+        if (cjk >= 8 && digits === 0 && !hasDomainSignal && !hasCommonTextShape)
+            return true;
+        return false;
+    }
     isReadableCadValue(value) {
         const cleaned = this.cleanCadReadableText(value);
-        return cleaned.length >= 2 && !CAD_INTERNAL_LINE_RE.test(cleaned) && /[\p{Script=Han}\p{Letter}\d]/u.test(cleaned);
+        return cleaned.length >= 2 && !CAD_INTERNAL_LINE_RE.test(cleaned) && /[\p{Script=Han}\p{Letter}\d]/u.test(cleaned) && !this.isLikelyGarbledCadText(cleaned);
     }
     cleanExtractedText(value, file) {
         const normalized = [...value]
@@ -162,7 +188,7 @@ export class ContentExtractor {
         return normalized
             .split(/\r?\n/u)
             .map(line => this.cleanCadReadableText(line))
-            .filter(line => line && !CAD_INTERNAL_LINE_RE.test(line))
+            .filter(line => line && !CAD_INTERNAL_LINE_RE.test(line) && !this.isLikelyGarbledCadText(line))
             .join('\n')
             .replace(/\n{3,}/gu, '\n\n');
     }
@@ -172,7 +198,7 @@ export class ContentExtractor {
         return cjk * 4 + alnum + Math.min(value.length, 200) / 20;
     }
     async extractCad(file) {
-        const metadata = { extractionMode: 'builtin_cad_structural', vectorizable: true };
+        const metadata = { extractionMode: 'builtin_cad_structural', vectorizable: true, preferredExtractionMode: 'dwg_to_dxf_semantic' };
         const warnings = [];
         const ext = path.extname(file.absolutePath).toLowerCase();
         if (ext === '.dxf')
@@ -180,7 +206,7 @@ export class ContentExtractor {
         if (ext === '.dwg') {
             const converted = await this.tryConvertDwgToDxf(file.absolutePath);
             if (converted?.dxfText) {
-                const parsed = await this.extractDxf(file, converted.dxfText, { ...metadata, extractionMode: converted.tool, convertedFrom: 'dwg' });
+                const parsed = await this.extractDxf(file, converted.dxfText, { ...metadata, extractionMode: converted.tool, convertedFrom: 'dwg', professionalConversionUsed: true });
                 parsed.warnings.push(...converted.warnings);
                 return parsed;
             }
@@ -264,14 +290,20 @@ export class ContentExtractor {
             if (result.text.trim())
                 return result;
         }
-        const readable = this.extractBinaryReadableFragments(file.absolutePath).filter(value => this.isReadableCadValue(value)).slice(0, 5000);
+        const binaryFragments = this.extractBinaryReadableFragments(file.absolutePath);
+        const readable = binaryFragments.filter(value => this.isReadableCadValue(value)).slice(0, 5000);
+        const filteredCount = Math.max(0, binaryFragments.length - readable.length);
         metadata.extractionMode = 'builtin_cad_readable_fragments';
-        metadata.contentCoverage = readable.length > 0 ? 'cad_readable_text_fragments' : 'metadata';
+        metadata.professionalConversionUsed = false;
+        metadata.contentCoverage = readable.length > 0 ? 'cad_readable_text_fragments_filtered' : 'metadata';
+        metadata.contentConfidence = readable.length > 0 ? 'low_fallback_filtered' : 'metadata_only';
+        metadata.stringCandidateCount = binaryFragments.length;
         metadata.stringCount = readable.length;
+        metadata.filteredGarbledStringCount = filteredCount;
         if (readable.length === 0)
             warnings.push(`${file.format} 内置 CAD 解析器未提取到可用文本，仅记录文件元数据，未生成可检索正文切片`);
         else
-            warnings.push(`${file.format} 未检测到专业 DWG 转换器，已使用内置可读标注/标题块抽取；如需完整图纸结构，请安装 ODA File Converter 或 LibreDWG 并配置外部解析器`);
+            warnings.push(`${file.format} 内置 DWG→DXF 转换未成功，已使用低置信度可读标注/标题块兜底抽取并过滤疑似乱码 ${filteredCount} 条；该结果仅作为兜底证据，不应等同于完整图层、块、标注和尺寸语义解析`);
         return {
             text: readable.length > 0 ? [this.metadataOnlyText(file), `CAD 图纸可读标注/标题块/属性:\n${readable.join('\n')}`].join('\n') : this.metadataOnlyText(file),
             metadata,
@@ -373,15 +405,29 @@ export class ContentExtractor {
             const mod = await resolveAndImport('dwgdxf');
             if (!mod.convertDwgToDxf)
                 return { tool: 'dwgdxf_wasm', warnings: ['内置 dwgdxf WASM 转换器未导出 convertDwgToDxf'] };
-            const wasmBase = pathToFileURL(path.join(path.dirname(resolvePackage('dwgdxf')), 'wasm')).href;
-            const dxfBytes = await mod.convertDwgToDxf(fs.readFileSync(filePath), { wasmBase });
-            const dxfText = Buffer.from(dxfBytes).toString('utf8');
-            return dxfText.trim()
-                ? { dxfText, tool: 'dwgdxf_wasm', warnings: [] }
-                : { tool: 'dwgdxf_wasm', warnings: ['内置 dwgdxf WASM 转换器未输出 DXF 文本'] };
+            const dwgBytes = fs.readFileSync(filePath);
+            const attempts = [
+                { label: 'package-default' },
+                { label: 'package-dist-wasm', options: { wasmBase: pathToFileURL(path.join(path.dirname(resolvePackage('dwgdxf')), 'wasm')).href } },
+                { label: 'package-root-dist-wasm', options: { wasmBase: pathToFileURL(path.join(path.dirname(path.dirname(resolvePackage('dwgdxf'))), 'wasm')).href } },
+            ];
+            const failures = [];
+            for (const attempt of attempts) {
+                try {
+                    const dxfBytes = await mod.convertDwgToDxf(dwgBytes, attempt.options);
+                    const dxfText = Buffer.from(dxfBytes).toString('utf8');
+                    if (dxfText.trim())
+                        return { dxfText, tool: `dwgdxf_wasm:${attempt.label}`, warnings: [] };
+                    failures.push(`${attempt.label}: 未输出 DXF 文本`);
+                }
+                catch (error) {
+                    failures.push(`${attempt.label}: ${error instanceof Error ? error.message : String(error)}`);
+                }
+            }
+            return { tool: 'dwgdxf_wasm', warnings: [`内置 dwgdxf WASM 转换失败: ${failures.join('；')}`] };
         }
         catch (error) {
-            return { tool: 'dwgdxf_wasm', warnings: [`内置 dwgdxf WASM 转换失败: ${error instanceof Error ? error.message : String(error)}`] };
+            return { tool: 'dwgdxf_wasm', warnings: [`内置 dwgdxf WASM 加载失败: ${error instanceof Error ? error.message : String(error)}`] };
         }
     }
     async tryConvertDwgToDxf(filePath) {

package/dist/index.d.ts

@@ -17,5 +17,5 @@ export { HNSWVectorStore } from './vector/hnsw-vector-store.js';
 export { CollectionManager, globalCollectionName, projectCollectionName } from './vector/collection-manager.js';
 export type { CollectionClient, VectorCollectionInfo, VectorDocument, VectorSearchQuery, VectorSearchResult, VectorStoreInterface, VectorWriteOptions } from './vector/types.js';
 export { VectorIndexer, type VectorIndexResult } from './vector/vector-indexer.js';
-export { FederationSearch, type FederatedQuery, type FederatedResult, type FederatedSearchItem, type SearchFilters, type SearchScope } from './search/federation-search.js';
+export { FederationSearch, type FederatedQuery, type FederatedResult, type FederatedSearchItem, type RetrievalWeights, type SearchFilters, type SearchScope } from './search/federation-search.js';
 export type { LLMChatMessage, LLMChatOptions, LLMChatResponse, LLMSearchProvider } from './llm/llm-search-provider.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@customize-agent/knowledge",
-  "version": "4.0.40",
+  "version": "4.0.41",
   "description": "Local knowledge base infrastructure for customize-agent",
   "type": "module",
   "main": "./dist/index.js",