@customize-agent/knowledge 4.0.1 → 4.0.3

package/dist/dedup/dedup-engine.d.ts

@@ -1,12 +1,15 @@
+/** MinHash 签名结果 */
 export interface MinHashSignature {
     signature: number[];
     shingleCount: number;
     buckets: string[];
 }
+/** 相似度匹配结果 */
 export interface SimilarityMatch {
     filePath: string;
     similarity: number;
 }
+/** 去重引擎，支持归一化哈希、MinHash 相似度计算和 LSH 分桶 */
 export declare class DedupEngine {
     private readonly hashCount;
     private readonly bandSize;

package/dist/dedup/dedup-engine.js

@@ -1,4 +1,5 @@
 import * as crypto from 'node:crypto';
+/** 去重引擎，支持归一化哈希、MinHash 相似度计算和 LSH 分桶 */
 export class DedupEngine {
     hashCount = 192;
     bandSize = 6;

package/dist/dedup/relationship-detector.d.ts

@@ -1,6 +1,13 @@
 import type { ClassifiedFile, IndexStateRecord } from '../types.js';
 import type { FileRelationship } from '../core/index-state-store.js';
+/** 文件关系检测器，自动识别版本链、翻译关系和互补关系 */
 export declare class RelationshipDetector {
+    /**
+     * 检测文件与已有索引记录之间的关系
+     * @param file 当前处理的文件
+     * @param indexedRecords 已有索引记录列表
+     * @returns 检测到的关系列表
+     */
     detect(file: ClassifiedFile, indexedRecords: IndexStateRecord[]): Array<Omit<FileRelationship, 'id' | 'createdAt'>>;
     private detectVersionChain;
     private detectTranslation;

package/dist/dedup/relationship-detector.js

@@ -1,5 +1,12 @@
 import * as path from 'node:path';
+/** 文件关系检测器，自动识别版本链、翻译关系和互补关系 */
 export class RelationshipDetector {
+    /**
+     * 检测文件与已有索引记录之间的关系
+     * @param file 当前处理的文件
+     * @param indexedRecords 已有索引记录列表
+     * @returns 检测到的关系列表
+     */
     detect(file, indexedRecords) {
         const relationships = [];
         for (const record of indexedRecords) {

package/dist/embedding/embedding-provider.d.ts

@@ -1,9 +1,11 @@
+/** Embedding Provider 接口，负责将文本转换为向量 */
 export interface EmbeddingProvider {
     readonly model: string;
     readonly dimensions: number;
     embedDocuments(texts: string[]): Promise<number[][]>;
     embedQuery(text: string): Promise<number[]>;
 }
+/** 基于哈希的本地 Embedding Provider（无需模型，使用哈希算法生成特征向量） */
 export declare class HashEmbeddingProvider implements EmbeddingProvider {
     readonly dimensions: number;
     readonly model = "hash-embedding-local";
@@ -20,6 +22,7 @@ export interface OpenAICompatibleEmbeddingOptions {
     model: string;
     dimensions?: number;
 }
+/** OpenAI 兼容的 Embedding Provider，支持任何兼容 OpenAI API 的嵌入服务 */
 export declare class OpenAICompatibleEmbeddingProvider implements EmbeddingProvider {
     readonly model: string;
     readonly dimensions: number;
@@ -30,4 +33,33 @@ export declare class OpenAICompatibleEmbeddingProvider implements EmbeddingProvi
     embedQuery(text: string): Promise<number[]>;
     private embed;
 }
+export interface LocalTransformersEmbeddingOptions {
+    model?: string;
+    dimensions?: number;
+    modelPath?: string;
+    batchSize?: number;
+}
+/** 本地 Transformers.js 模型 Embedding Provider，使用 BGE 小模型生成向量 */
+export declare class LocalTransformersEmbeddingProvider implements EmbeddingProvider {
+    readonly model: string;
+    readonly dimensions: number;
+    private readonly modelPath?;
+    private readonly batchSize;
+    private static pipelines;
+    constructor(options?: LocalTransformersEmbeddingOptions);
+    embedDocuments(texts: string[]): Promise<number[][]>;
+    embedQuery(text: string): Promise<number[]>;
+    private embed;
+    private getPipeline;
+    private createPipeline;
+    private parseVectors;
+    private isTensorLike;
+    private splitFlatVectors;
+    private resizeVector;
+    private normalize;
+}
+/**
+ * 根据环境变量或配置文件自动创建 Embedding Provider
+ * 优先级：环境变量 > 配置文件 > 本地 Transformers.js 默认
+ */
 export declare function createEmbeddingProviderFromEnvironment(): EmbeddingProvider;

package/dist/embedding/embedding-provider.js

@@ -2,6 +2,8 @@ import * as crypto from 'node:crypto';
 import * as fs from 'node:fs';
 import * as os from 'node:os';
 import * as path from 'node:path';
+import { fileURLToPath } from 'node:url';
+/** 基于哈希的本地 Embedding Provider（无需模型，使用哈希算法生成特征向量） */
 export class HashEmbeddingProvider {
     dimensions;
     model = 'hash-embedding-local';
@@ -47,6 +49,7 @@ export class HashEmbeddingProvider {
         return vector.map(value => value / norm);
     }
 }
+/** OpenAI 兼容的 Embedding Provider，支持任何兼容 OpenAI API 的嵌入服务 */
 export class OpenAICompatibleEmbeddingProvider {
     model;
     dimensions;
@@ -79,6 +82,128 @@ export class OpenAICompatibleEmbeddingProvider {
         return (payload.data ?? []).map(item => item.embedding ?? []);
     }
 }
+function resolveLocalEmbeddingBatchSize(configured) {
+    const raw = configured ?? Number(process.env.CUSTOMIZE_EMBEDDING_BATCH_SIZE ?? process.env.KB_EMBEDDING_BATCH_SIZE);
+    const fallback = process.platform === 'win32' ? 8 : 16;
+    if (!Number.isFinite(raw) || raw <= 0)
+        return fallback;
+    return Math.max(1, Math.min(128, Math.floor(raw)));
+}
+function resolveBundledBgeModelPath() {
+    const currentDir = path.dirname(fileURLToPath(import.meta.url));
+    const candidates = [
+        process.env.CUSTOMIZE_BGE_MODEL_PATH,
+        process.env.KB_BGE_MODEL_PATH,
+        path.resolve(process.cwd(), 'models', 'bge-small-zh-v1.5'),
+        path.resolve(process.cwd(), 'packages', 'knowledge', 'models', 'bge-small-zh-v1.5'),
+        path.resolve(currentDir, '..', '..', 'models', 'bge-small-zh-v1.5'),
+    ].filter(Boolean);
+    return candidates.find(candidate => fs.existsSync(path.join(candidate, 'config.json')) && fs.existsSync(path.join(candidate, 'tokenizer.json')));
+}
+/** 本地 Transformers.js 模型 Embedding Provider，使用 BGE 小模型生成向量 */
+export class LocalTransformersEmbeddingProvider {
+    model;
+    dimensions;
+    modelPath;
+    batchSize;
+    static pipelines = new Map();
+    constructor(options = {}) {
+        this.model = options.model?.trim() || 'BAAI/bge-small-zh-v1.5';
+        this.dimensions = options.dimensions ?? 512;
+        this.modelPath = options.modelPath || resolveBundledBgeModelPath();
+        this.batchSize = resolveLocalEmbeddingBatchSize(options.batchSize);
+    }
+    async embedDocuments(texts) {
+        return this.embed(texts);
+    }
+    async embedQuery(text) {
+        return (await this.embed([text]))[0] ?? [];
+    }
+    async embed(input) {
+        const extractor = await this.getPipeline();
+        const vectors = [];
+        for (let offset = 0; offset < input.length; offset += this.batchSize) {
+            const batch = input.slice(offset, offset + this.batchSize);
+            const output = await extractor(batch, { pooling: 'mean', normalize: true });
+            vectors.push(...this.parseVectors(output, batch.length).map(vector => this.resizeVector(vector)));
+        }
+        return vectors;
+    }
+    getPipeline() {
+        const existing = LocalTransformersEmbeddingProvider.pipelines.get(this.model);
+        if (existing)
+            return existing;
+        const created = this.createPipeline();
+        LocalTransformersEmbeddingProvider.pipelines.set(this.model, created);
+        return created;
+    }
+    async createPipeline() {
+        const modelPath = this.modelPath;
+        if (!modelPath) {
+            throw new Error('本地 bge-small-zh-v1.5 模型资源缺失：请将模型文件放到 packages/knowledge/models/bge-small-zh-v1.5，或通过 CUSTOMIZE_BGE_MODEL_PATH 指定本地模型目录。');
+        }
+        const dynamicImport = new Function('specifier', 'return import(specifier)');
+        const mod = await dynamicImport('@huggingface/transformers').catch(error => {
+            const message = error instanceof Error ? error.message : String(error);
+            throw new Error(`本地语义模型运行依赖 @huggingface/transformers 未安装或无法解析：${message}`);
+        });
+        if (!mod.pipeline)
+            throw new Error('Transformers.js pipeline is unavailable');
+        if (mod.env) {
+            mod.env.allowRemoteModels = false;
+            mod.env.allowLocalModels = true;
+            mod.env.localModelPath = path.dirname(modelPath);
+        }
+        return mod.pipeline('feature-extraction', modelPath, { dtype: 'q8' });
+    }
+    parseVectors(output, count) {
+        if (this.isTensorLike(output)) {
+            const dims = output.dims;
+            const data = Array.from(output.data, Number);
+            if (dims.length === 2)
+                return this.splitFlatVectors(data, dims[0] ?? count, dims[1] ?? this.dimensions);
+            if (dims.length === 3) {
+                const batch = dims[0] ?? count;
+                const tokens = dims[1] ?? 1;
+                const width = dims[2] ?? this.dimensions;
+                return Array.from({ length: batch }, (_, batchIndex) => {
+                    const vector = Array.from({ length: width }, () => 0);
+                    for (let tokenIndex = 0; tokenIndex < tokens; tokenIndex++) {
+                        const offset = batchIndex * tokens * width + tokenIndex * width;
+                        for (let i = 0; i < width; i++)
+                            vector[i] = (vector[i] ?? 0) + (data[offset + i] ?? 0);
+                    }
+                    return this.normalize(vector.map(value => value / Math.max(1, tokens)));
+                });
+            }
+        }
+        if (Array.isArray(output)) {
+            const first = output[0];
+            if (Array.isArray(first) && typeof first[0] === 'number')
+                return output;
+            if (typeof first === 'number')
+                return [output];
+        }
+        throw new Error('Unsupported Transformers embedding output');
+    }
+    isTensorLike(value) {
+        return typeof value === 'object' && value !== null && 'data' in value && 'dims' in value && Array.isArray(value.dims);
+    }
+    splitFlatVectors(data, count, width) {
+        return Array.from({ length: count }, (_, index) => this.normalize(data.slice(index * width, (index + 1) * width)));
+    }
+    resizeVector(vector) {
+        if (vector.length === this.dimensions)
+            return vector;
+        if (vector.length > this.dimensions)
+            return this.normalize(vector.slice(0, this.dimensions));
+        return this.normalize([...vector, ...Array.from({ length: this.dimensions - vector.length }, () => 0)]);
+    }
+    normalize(vector) {
+        const norm = Math.sqrt(vector.reduce((sum, value) => sum + value * value, 0));
+        return norm > 0 ? vector.map(value => value / norm) : vector;
+    }
+}
 function readStoredEmbeddingConfig() {
     try {
         const configPath = path.join(os.homedir(), '.customize-agent', 'config.json');
@@ -91,9 +216,13 @@ function readStoredEmbeddingConfig() {
         return undefined;
     }
 }
+/**
+ * 根据环境变量或配置文件自动创建 Embedding Provider
+ * 优先级：环境变量 > 配置文件 > 本地 Transformers.js 默认
+ */
 export function createEmbeddingProviderFromEnvironment() {
     const stored = readStoredEmbeddingConfig();
-    const provider = process.env.CUSTOMIZE_EMBEDDING_PROVIDER ?? process.env.KB_EMBEDDING_PROVIDER ?? stored?.provider;
+    const provider = process.env.CUSTOMIZE_EMBEDDING_PROVIDER ?? process.env.KB_EMBEDDING_PROVIDER ?? stored?.provider ?? 'transformers-local';
     if (provider === 'openai-compatible') {
         const baseUrl = process.env.CUSTOMIZE_EMBEDDING_BASE_URL ?? process.env.KB_EMBEDDING_BASE_URL ?? stored?.baseUrl;
         const model = process.env.CUSTOMIZE_EMBEDDING_MODEL ?? process.env.KB_EMBEDDING_MODEL ?? stored?.model;
@@ -108,5 +237,10 @@ export function createEmbeddingProviderFromEnvironment() {
             });
         }
     }
-    return new HashEmbeddingProvider();
+    const rawDimensions = process.env.CUSTOMIZE_EMBEDDING_DIMENSIONS ?? process.env.KB_EMBEDDING_DIMENSIONS;
+    const dimensions = Number(rawDimensions ?? (stored?.provider === 'transformers-local' ? stored.dimensions : undefined) ?? 512);
+    return new LocalTransformersEmbeddingProvider({
+        model: process.env.CUSTOMIZE_EMBEDDING_MODEL ?? process.env.KB_EMBEDDING_MODEL ?? (stored?.provider === 'transformers-local' ? stored.model : undefined) ?? 'BAAI/bge-small-zh-v1.5',
+        dimensions: Number.isFinite(dimensions) ? dimensions : 512,
+    });
 }

package/dist/extraction/content-extractor.d.ts

@@ -1,11 +1,13 @@
 import { ExternalExtractorRegistry } from './external-extractor.js';
 import type { ClassifiedFile } from '../types.js';
+/** 文件内容提取结果 */
 export interface ExtractionResult {
     text: string;
     metadata: Record<string, unknown>;
     warnings: string[];
     extractionTimeMs: number;
 }
+/** 文件内容提取器，支持文档、表格、图片、CAD 等多种文件格式的内容抽取 */
 export declare class ContentExtractor {
     private readonly externalExtractors;
     constructor(externalExtractors?: ExternalExtractorRegistry);
@@ -17,6 +19,10 @@ export declare class ContentExtractor {
     private textScore;
     private extractCad;
     private extractDxf;
+    private buildCadSemanticNodes;
+    private extractDxfTextAnnotations;
+    private findNearestCadAnnotation;
+    private inferCadEntityType;
     private tryConvertDwgWithBundledWasm;
     private tryConvertDwgToDxf;
     private extractCadMesh;
@@ -25,26 +31,50 @@ export declare class ContentExtractor {
     private extractBinaryStrings;
     private extractData;
     private extractDiagram;
+    private extractDrawioGraph;
+    private extractExcalidrawGraph;
+    private isPointInside;
+    private parseXmlAttributes;
     private parseDelimitedLine;
+    private toMarkdownTable;
     private extractDelimitedText;
     private extractOfficeDocument;
     private extractRtf;
     private extractLegacyWordDocument;
     private extractLegacyOfficeBinary;
+    private findMergedCellValue;
     private extractSpreadsheet;
+    private extractDocxStyleTreeMarkdown;
+    private docxHeadingLevel;
     private extractOfficeZip;
-    private extractArchive;
     private extractRasterImage;
+    private tryPaddleOcrLayout;
+    private parseOcrJson;
+    private formatOcrRegions;
+    private classifyOcrRegion;
+    private formatBoundingBox;
     private validateRasterImage;
     private extractPdf;
-    private pdfOcrPageLimit;
     private extractScannedPdfOcr;
     private extractPdfText;
+    private toPdfTextItem;
+    private layoutPdfTextItems;
+    private groupPdfItemsIntoRows;
+    private detectPdfColumnSplit;
+    private rowsToPdfMarkdownWithTables;
+    private splitLikelyTableRow;
+    private pdfRowToMarkdown;
+    private toMarkdownDocument;
+    private normalizeMarkdownHeadings;
     private extractSvg;
+    private extractSvgSemanticNodes;
     private isTextReadable;
     private looksTextFile;
     private matchAll;
     private stripXml;
     private flattenJson;
+    private atomicJsonObjects;
+    private flattenYamlByIndent;
+    private flattenXmlPaths;
     private metadataOnlyText;
 }