@customize-agent/knowledge 3.0.3 → 3.0.4

package/dist/classification/classifier.js

@@ -53,7 +53,7 @@ export class FileClassifier {
             ['.txt', 'document', 'plaintext'], ['.textclipping', 'document', 'text_clipping'], ['.rst', 'document', 'plaintext'], ['.asciidoc', 'document', 'plaintext'], ['.tex', 'document', 'plaintext'],
             ['.epub', 'document', 'ebook'], ['.mobi', 'document', 'ebook'],
             ['.xlsx', 'spreadsheet', 'excel'], ['.xls', 'spreadsheet', 'excel'], ['.xlsm', 'spreadsheet', 'excel'],
-            ['.csv', 'spreadsheet', 'csv'], ['.tsv', 'spreadsheet', 'csv'], ['.tab', 'spreadsheet', 'csv'], ['.ods', 'spreadsheet', 'opendoc'],
+            ['.csv', 'spreadsheet', 'csv'], ['.tsv', 'spreadsheet', 'tsv'], ['.tab', 'spreadsheet', 'tsv'], ['.ods', 'spreadsheet', 'opendoc'],
             ['.png', 'image', 'raster'], ['.jpg', 'image', 'raster'], ['.jpeg', 'image', 'raster'], ['.gif', 'image', 'raster'], ['.bmp', 'image', 'raster'], ['.webp', 'image', 'raster'], ['.tiff', 'image', 'raster'], ['.tif', 'image', 'raster'],
             ['.svg', 'image', 'vector'], ['.eps', 'image', 'vector'], ['.raw', 'image', 'raw'], ['.cr2', 'image', 'raw'], ['.nef', 'image', 'raw'], ['.dng', 'image', 'raw'],
             ['.dwg', 'cad', 'autocad'], ['.dxf', 'cad', 'autocad'], ['.dwt', 'cad', 'autocad'],

package/dist/core/change-tracker.js

@@ -30,11 +30,14 @@ export class ChangeTracker {
                 continue;
             }
             const metadata = this.parseMetadata(indexed.metadataJson);
+            const extraction = metadata.extraction && typeof metadata.extraction === 'object' ? metadata.extraction : {};
+            const contentCoverage = metadata.contentCoverage ?? extraction.contentCoverage;
+            const extractionMode = metadata.extractionMode ?? extraction.extractionMode;
             const needsReindex = indexed.status === 'error'
                 || indexed.chunkCount === 0
                 || (classified.format === 'pdf' && indexed.chunkCount <= 1)
-                || metadata.contentCoverage === 'metadata_filename'
-                || metadata.extractionMode === 'pdf_metadata_only';
+                || contentCoverage === 'metadata_filename'
+                || extractionMode === 'pdf_metadata_only';
             if (needsReindex) {
                 modifiedFiles.push(classified);
                 continue;

package/dist/core/knowledge-base-manager.js

@@ -6,7 +6,7 @@ import { FileClassifier } from '../classification/classifier.js';
 import { ALL_CATEGORIES, DEFAULT_CATEGORY_DIRS, GLOBAL_KNOWLEDGE_DIR, USER_DATA_DIR } from '../constants.js';
 import { DedupEngine } from '../dedup/dedup-engine.js';
 import { RelationshipDetector } from '../dedup/relationship-detector.js';
-import { HashEmbeddingProvider } from '../embedding/embedding-provider.js';
+import { createEmbeddingProviderFromEnvironment } from '../embedding/embedding-provider.js';
 import { ContentExtractor } from '../extraction/content-extractor.js';
 import { FederationSearch } from '../search/federation-search.js';
 import { CollectionManager } from '../vector/collection-manager.js';
@@ -41,7 +41,7 @@ export class KnowledgeBaseManager {
         this.scope = options.scope;
         this.projectRoot = options.projectRoot;
         this.projectId = options.projectId;
-        this.embeddingProvider = options.embeddingProvider ?? new HashEmbeddingProvider();
+        this.embeddingProvider = options.embeddingProvider ?? createEmbeddingProviderFromEnvironment();
         this.vectorStores = options.vectorStores ?? new Map();
         this.extractor = new ContentExtractor(options.externalExtractors);
         this.llmProvider = options.llmProvider;
@@ -200,6 +200,7 @@ export class KnowledgeBaseManager {
                     status: 'active',
                     metadataJson: JSON.stringify({
                         mimeType: file.mimeType,
+                        ...extraction.metadata,
                         extraction: extraction.metadata,
                         warnings: extraction.warnings,
                         extractionTimeMs: extraction.extractionTimeMs,
@@ -705,7 +706,7 @@ ${resultsText}
     }
     hasUsableContent(text, metadata) {
         const coverage = String(metadata.contentCoverage ?? '');
-        if (coverage === 'metadata')
+        if (['metadata', 'metadata_filename', 'pdf_metadata_only', 'office_zip_empty_text', 'office_zip_failed'].includes(coverage))
             return false;
         return text.trim().length > 0;
     }

package/dist/embedding/embedding-provider.d.ts

@@ -14,3 +14,20 @@ export declare class HashEmbeddingProvider implements EmbeddingProvider {
     private tokenize;
     private normalize;
 }
+export interface OpenAICompatibleEmbeddingOptions {
+    baseUrl: string;
+    apiKey?: string;
+    model: string;
+    dimensions?: number;
+}
+export declare class OpenAICompatibleEmbeddingProvider implements EmbeddingProvider {
+    readonly model: string;
+    readonly dimensions: number;
+    private readonly baseUrl;
+    private readonly apiKey?;
+    constructor(options: OpenAICompatibleEmbeddingOptions);
+    embedDocuments(texts: string[]): Promise<number[][]>;
+    embedQuery(text: string): Promise<number[]>;
+    private embed;
+}
+export declare function createEmbeddingProviderFromEnvironment(): EmbeddingProvider;

package/dist/embedding/embedding-provider.js

@@ -1,4 +1,7 @@
 import * as crypto from 'node:crypto';
+import * as fs from 'node:fs';
+import * as os from 'node:os';
+import * as path from 'node:path';
 export class HashEmbeddingProvider {
     dimensions;
     model = 'hash-embedding-local';
@@ -44,3 +47,66 @@ export class HashEmbeddingProvider {
         return vector.map(value => value / norm);
     }
 }
+export class OpenAICompatibleEmbeddingProvider {
+    model;
+    dimensions;
+    baseUrl;
+    apiKey;
+    constructor(options) {
+        this.baseUrl = options.baseUrl.replace(/\/+$/u, '');
+        this.apiKey = options.apiKey;
+        this.model = options.model;
+        this.dimensions = options.dimensions ?? 1024;
+    }
+    async embedDocuments(texts) {
+        return this.embed(texts);
+    }
+    async embedQuery(text) {
+        return (await this.embed([text]))[0] ?? [];
+    }
+    async embed(input) {
+        const response = await fetch(`${this.baseUrl}/embeddings`, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                ...(this.apiKey ? { Authorization: `Bearer ${this.apiKey}` } : {}),
+            },
+            body: JSON.stringify({ model: this.model, input }),
+        });
+        if (!response.ok)
+            throw new Error(`Embedding request failed: HTTP ${response.status} ${await response.text()}`);
+        const payload = await response.json();
+        return (payload.data ?? []).map(item => item.embedding ?? []);
+    }
+}
+function readStoredEmbeddingConfig() {
+    try {
+        const configPath = path.join(os.homedir(), '.customize-agent', 'config.json');
+        if (!fs.existsSync(configPath))
+            return undefined;
+        const raw = JSON.parse(fs.readFileSync(configPath, 'utf-8'));
+        return raw.embedding;
+    }
+    catch {
+        return undefined;
+    }
+}
+export function createEmbeddingProviderFromEnvironment() {
+    const stored = readStoredEmbeddingConfig();
+    const provider = process.env.CUSTOMIZE_EMBEDDING_PROVIDER ?? process.env.KB_EMBEDDING_PROVIDER ?? stored?.provider;
+    if (provider === 'openai-compatible') {
+        const baseUrl = process.env.CUSTOMIZE_EMBEDDING_BASE_URL ?? process.env.KB_EMBEDDING_BASE_URL ?? stored?.baseUrl;
+        const model = process.env.CUSTOMIZE_EMBEDDING_MODEL ?? process.env.KB_EMBEDDING_MODEL ?? stored?.model;
+        if (baseUrl && model) {
+            const rawDimensions = process.env.CUSTOMIZE_EMBEDDING_DIMENSIONS ?? process.env.KB_EMBEDDING_DIMENSIONS;
+            const dimensions = Number(rawDimensions ?? stored?.dimensions ?? 1024);
+            return new OpenAICompatibleEmbeddingProvider({
+                baseUrl,
+                model,
+                apiKey: process.env.CUSTOMIZE_EMBEDDING_API_KEY ?? process.env.KB_EMBEDDING_API_KEY ?? stored?.apiKey,
+                dimensions: Number.isFinite(dimensions) ? dimensions : 1024,
+            });
+        }
+    }
+    return new HashEmbeddingProvider();
+}

package/dist/extraction/content-extractor.js

@@ -19,88 +19,91 @@ export class ContentExtractor {
             format: file.format,
         };
         const external = this.tryExternalExtractor(file);
-        if (external) {
+        if (external?.text.trim()) {
             text = external.text;
             Object.assign(metadata, external.metadata);
             warnings.push(...external.warnings);
         }
-        else if (file.category === 'cad') {
-            const result = await this.extractCad(file);
-            text = result.text;
-            Object.assign(metadata, result.metadata);
-            warnings.push(...result.warnings);
-        }
-        else if (file.category === 'data') {
-            const result = this.extractData(file);
-            text = result.text;
-            Object.assign(metadata, result.metadata);
-            warnings.push(...result.warnings);
-        }
-        else if (file.category === 'diagram') {
-            const result = this.extractDiagram(file);
-            text = result.text;
-            Object.assign(metadata, result.metadata);
-            warnings.push(...result.warnings);
-        }
-        else if (file.category === 'document' && file.format === 'pdf') {
-            const result = await this.extractPdf(file);
-            text = result.text;
-            Object.assign(metadata, result.metadata);
-            warnings.push(...result.warnings);
-        }
-        else if (file.category === 'document' && ['office', 'presentation'].includes(file.format)) {
-            const result = await this.extractOfficeDocument(file);
-            text = result.text;
-            Object.assign(metadata, result.metadata);
-            warnings.push(...result.warnings);
-        }
-        else if (file.category === 'spreadsheet' && ['csv', 'tsv'].includes(file.format)) {
-            const result = this.extractDelimitedText(file);
-            text = result.text;
-            Object.assign(metadata, result.metadata);
-            warnings.push(...result.warnings);
-        }
-        else if (file.category === 'spreadsheet') {
-            const result = await this.extractSpreadsheet(file);
-            text = result.text;
-            Object.assign(metadata, result.metadata);
-            warnings.push(...result.warnings);
-        }
-        else if (file.category === 'archive') {
-            const result = await this.extractArchive(file);
-            text = result.text;
-            Object.assign(metadata, result.metadata);
-            warnings.push(...result.warnings);
-        }
-        else if (file.category === 'image' && file.format !== 'vector') {
-            const result = await this.extractRasterImage(file);
-            text = result.text;
-            Object.assign(metadata, result.metadata);
-            warnings.push(...result.warnings);
-        }
-        else if (file.category === 'image' && file.format === 'vector') {
-            const result = this.extractSvg(file);
-            text = result.text;
-            Object.assign(metadata, result.metadata);
-            warnings.push(...result.warnings);
-        }
-        else if (file.format === 'text_clipping') {
-            const result = this.extractTextClipping(file);
-            text = result.text;
-            Object.assign(metadata, result.metadata);
-            warnings.push(...result.warnings);
-        }
-        else if (this.isTextReadable(file)) {
-            text = fs.readFileSync(file.absolutePath, 'utf8');
-            metadata.extractionMode = 'plain_text';
-            metadata.vectorizable = true;
-        }
         else {
-            text = this.metadataOnlyText(file);
-            metadata.extractionMode = 'metadata_only';
-            metadata.vectorizable = true;
-            metadata.contentCoverage = 'metadata';
-            warnings.push(`暂不支持 ${file.category}/${file.format} 内容提取，未解析出正文，未入库`);
+            warnings.push(...(external?.warnings ?? []));
+            if (file.category === 'cad') {
+                const result = await this.extractCad(file);
+                text = result.text;
+                Object.assign(metadata, result.metadata);
+                warnings.push(...result.warnings);
+            }
+            else if (file.category === 'data') {
+                const result = this.extractData(file);
+                text = result.text;
+                Object.assign(metadata, result.metadata);
+                warnings.push(...result.warnings);
+            }
+            else if (file.category === 'diagram') {
+                const result = this.extractDiagram(file);
+                text = result.text;
+                Object.assign(metadata, result.metadata);
+                warnings.push(...result.warnings);
+            }
+            else if (file.category === 'document' && file.format === 'pdf') {
+                const result = await this.extractPdf(file);
+                text = result.text;
+                Object.assign(metadata, result.metadata);
+                warnings.push(...result.warnings);
+            }
+            else if (file.category === 'document' && ['office', 'presentation'].includes(file.format)) {
+                const result = await this.extractOfficeDocument(file);
+                text = result.text;
+                Object.assign(metadata, result.metadata);
+                warnings.push(...result.warnings);
+            }
+            else if (file.category === 'spreadsheet' && ['csv', 'tsv'].includes(file.format)) {
+                const result = this.extractDelimitedText(file);
+                text = result.text;
+                Object.assign(metadata, result.metadata);
+                warnings.push(...result.warnings);
+            }
+            else if (file.category === 'spreadsheet') {
+                const result = await this.extractSpreadsheet(file);
+                text = result.text;
+                Object.assign(metadata, result.metadata);
+                warnings.push(...result.warnings);
+            }
+            else if (file.category === 'archive') {
+                const result = await this.extractArchive(file);
+                text = result.text;
+                Object.assign(metadata, result.metadata);
+                warnings.push(...result.warnings);
+            }
+            else if (file.category === 'image' && file.format !== 'vector') {
+                const result = await this.extractRasterImage(file);
+                text = result.text;
+                Object.assign(metadata, result.metadata);
+                warnings.push(...result.warnings);
+            }
+            else if (file.category === 'image' && file.format === 'vector') {
+                const result = this.extractSvg(file);
+                text = result.text;
+                Object.assign(metadata, result.metadata);
+                warnings.push(...result.warnings);
+            }
+            else if (file.format === 'text_clipping') {
+                const result = this.extractTextClipping(file);
+                text = result.text;
+                Object.assign(metadata, result.metadata);
+                warnings.push(...result.warnings);
+            }
+            else if (this.isTextReadable(file)) {
+                text = fs.readFileSync(file.absolutePath, 'utf8');
+                metadata.extractionMode = 'plain_text';
+                metadata.vectorizable = true;
+            }
+            else {
+                text = this.metadataOnlyText(file);
+                metadata.extractionMode = 'metadata_only';
+                metadata.vectorizable = true;
+                metadata.contentCoverage = 'metadata';
+                warnings.push(`暂不支持 ${file.category}/${file.format} 内容提取，未解析出正文，未入库`);
+            }
         }
         return {
             text: text.trim(),
@@ -138,7 +141,7 @@ export class ContentExtractor {
                 warnings.push(`外部解析器 ${extractor.name} 失败: ${error instanceof Error ? error.message : String(error)}`);
             }
         }
-        return undefined;
+        return warnings.length > 0 ? { text: '', metadata: {}, warnings } : undefined;
     }
     extractTextClipping(file) {
         const buffer = fs.readFileSync(file.absolutePath);
@@ -282,7 +285,7 @@ export class ContentExtractor {
         metadata.contentCoverage = readable.length > 0 ? 'cad_readable_text_fragments' : 'metadata';
         metadata.stringCount = readable.length;
         if (readable.length === 0)
-            warnings.push(`${file.format} 内置 CAD 解析器未提取到可用文本，已入库文件元数据用于检索`);
+            warnings.push(`${file.format} 内置 CAD 解析器未提取到可用文本，仅记录文件元数据，未生成可检索正文切片`);
         else
             warnings.push(`${file.format} 未检测到专业 DWG 转换器，已使用内置可读标注/标题块抽取；如需完整图纸结构，请安装 ODA File Converter 或 LibreDWG 并配置外部解析器`);
         return {
@@ -336,18 +339,34 @@ export class ContentExtractor {
         try {
             const customCmd = process.env.CUSTOMIZE_DWG_TO_DXF_CMD;
             if (customCmd) {
-                const command = customCmd.replace(/\{input\}/gu, filePath).replace(/\{output\}/gu, outputPath);
-                const result = spawnSync(command, { shell: true, encoding: 'utf8', timeout: 120_000 });
+                if (/\s/u.test(customCmd))
+                    return { tool: 'external_dwg_to_dxf', warnings: ['CUSTOMIZE_DWG_TO_DXF_CMD 只支持可执行文件路径；参数请使用 CUSTOMIZE_DWG_TO_DXF_ARGS JSON 数组配置'] };
+                let argTemplate = ['{input}', '{output}'];
+                try {
+                    if (process.env.CUSTOMIZE_DWG_TO_DXF_ARGS)
+                        argTemplate = JSON.parse(process.env.CUSTOMIZE_DWG_TO_DXF_ARGS);
+                }
+                catch {
+                    return { tool: 'external_dwg_to_dxf', warnings: ['CUSTOMIZE_DWG_TO_DXF_ARGS 必须是字符串数组 JSON'] };
+                }
+                if (!Array.isArray(argTemplate) || !argTemplate.every(arg => typeof arg === 'string'))
+                    return { tool: 'external_dwg_to_dxf', warnings: ['CUSTOMIZE_DWG_TO_DXF_ARGS 必须是字符串数组 JSON'] };
+                const args = argTemplate.map(arg => arg.replace(/\{input\}/gu, filePath).replace(/\{output\}/gu, outputPath));
+                const result = spawnSync(customCmd, args, { shell: false, encoding: 'utf8', timeout: 120_000 });
                 if (result.status === 0 && fs.existsSync(outputPath))
                     return { dxfText: fs.readFileSync(outputPath, 'utf8'), tool: 'external_dwg_to_dxf', warnings: [] };
                 return { tool: 'external_dwg_to_dxf', warnings: [`CUSTOMIZE_DWG_TO_DXF_CMD 转换失败: ${result.stderr || result.stdout || result.error?.message || 'unknown error'}`] };
             }
+            const failures = [];
             for (const bin of ['dwgread', 'dwg2dxf']) {
                 const result = spawnSync(bin, bin === 'dwgread' ? ['-O', 'DXF', '-o', outputPath, filePath] : [filePath, outputPath], { encoding: 'utf8', timeout: 120_000 });
                 if (result.status === 0 && fs.existsSync(outputPath))
                     return { dxfText: fs.readFileSync(outputPath, 'utf8'), tool: bin, warnings: [] };
+                if (result.error && 'code' in result.error && result.error.code === 'ENOENT')
+                    continue;
+                failures.push(`${bin} 转换失败: ${result.stderr || result.stdout || result.error?.message || `exit ${result.status ?? 'unknown'}`}`);
             }
-            return { tool: 'builtin_fallback', warnings: ['未检测到可用 DWG→DXF 转换器（dwgread/dwg2dxf/CUSTOMIZE_DWG_TO_DXF_CMD），使用内置图纸可读文本抽取'] };
+            return { tool: 'builtin_fallback', warnings: [...failures, failures.length ? 'DWG→DXF 转换失败，使用内置图纸可读文本抽取' : '未检测到可用 DWG→DXF 转换器（dwgread/dwg2dxf/CUSTOMIZE_DWG_TO_DXF_CMD），使用内置图纸可读文本抽取'] };
         }
         finally {
             fs.rmSync(tmpDir, { recursive: true, force: true });
@@ -559,6 +578,7 @@ export class ContentExtractor {
         };
     }
     async extractLegacyWordDocument(file) {
+        const warnings = [];
         try {
             const mod = await resolveAndImport('word-extractor');
             const WordExtractor = mod.default ?? mod;
@@ -571,11 +591,14 @@ export class ContentExtractor {
                     warnings: [],
                 };
             }
+            warnings.push('word-extractor 未提取到正文，已降级为二进制可读文本抽取');
         }
-        catch {
-            // fallback below
+        catch (error) {
+            warnings.push(`word-extractor 解析失败，已降级为二进制可读文本抽取: ${error instanceof Error ? error.message : String(error)}`);
         }
-        return this.extractLegacyOfficeBinary(file);
+        const fallback = this.extractLegacyOfficeBinary(file);
+        fallback.warnings.unshift(...warnings);
+        return fallback;
     }
     extractLegacyOfficeBinary(file) {
         const strings = this.extractBinaryStrings(file.absolutePath).slice(0, 1_000);

package/dist/index.d.ts

@@ -2,7 +2,7 @@ export { TextChunker, type ChunkConfig, type TextChunk } from './chunking/text-c
 export { FileClassifier } from './classification/classifier.js';
 export { DedupEngine, type MinHashSignature, type SimilarityMatch } from './dedup/dedup-engine.js';
 export { RelationshipDetector } from './dedup/relationship-detector.js';
-export { HashEmbeddingProvider, type EmbeddingProvider } from './embedding/embedding-provider.js';
+export { HashEmbeddingProvider, OpenAICompatibleEmbeddingProvider, createEmbeddingProviderFromEnvironment, type EmbeddingProvider, type OpenAICompatibleEmbeddingOptions } from './embedding/embedding-provider.js';
 export { ContentExtractor, type ExtractionResult } from './extraction/content-extractor.js';
 export { CommandExternalExtractor, ExternalExtractorRegistry, type CommandExternalExtractorOptions, type ExternalExtractionResult, type ExternalExtractor, type ExternalExtractorCapability } from './extraction/external-extractor.js';
 export { ChangeTracker } from './core/change-tracker.js';

package/dist/index.js

@@ -4,7 +4,7 @@ export { TextChunker } from './chunking/text-chunker.js';
 export { FileClassifier } from './classification/classifier.js';
 export { DedupEngine } from './dedup/dedup-engine.js';
 export { RelationshipDetector } from './dedup/relationship-detector.js';
-export { HashEmbeddingProvider } from './embedding/embedding-provider.js';
+export { HashEmbeddingProvider, OpenAICompatibleEmbeddingProvider, createEmbeddingProviderFromEnvironment } from './embedding/embedding-provider.js';
 export { ContentExtractor } from './extraction/content-extractor.js';
 export { CommandExternalExtractor, ExternalExtractorRegistry } from './extraction/external-extractor.js';
 export { ChangeTracker } from './core/change-tracker.js';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@customize-agent/knowledge",
-  "version": "3.0.3",
+  "version": "3.0.4",
   "description": "Local knowledge base infrastructure for customize-agent",
   "type": "module",
   "main": "./dist/index.js",
@@ -13,6 +13,9 @@
     }
   },
   "license": "MIT",
+  "engines": {
+    "node": ">=22.13.0"
+  },
   "author": "Pan-jijian",
   "repository": {
     "type": "git",