@customize-agent/knowledge 1.0.1

package/dist/dedup/dedup-engine.d.ts

@@ -0,0 +1,20 @@
+export interface MinHashSignature {
+    signature: number[];
+    shingleCount: number;
+}
+export interface SimilarityMatch {
+    filePath: string;
+    similarity: number;
+}
+export declare class DedupEngine {
+    private readonly hashCount;
+    normalizeText(text: string): string;
+    normalizedHash(text: string): string | undefined;
+    relationshipForFormats(sourceFormat: string, targetFormat: string): 'format_variant' | 'translation';
+    computeMinHash(text: string, shingleSize?: number): MinHashSignature | undefined;
+    estimateSimilarity(a: number[], b: number[]): number;
+    relationshipForSimilarity(similarity: number): 'near_duplicate' | 'revision' | undefined;
+    private hashToUint32;
+    private looksTranslationFormat;
+    private isStopWord;
+}

package/dist/dedup/dedup-engine.js

@@ -0,0 +1,78 @@
+import * as crypto from 'node:crypto';
+export class DedupEngine {
+    hashCount = 128;
+    normalizeText(text) {
+        return text
+            .toLowerCase()
+            .normalize('NFKC')
+            .replace(/\b\d{4}\b/gu, 'yyyy')
+            .replace(/\b\d{1,2}[-/]\d{1,2}[-/]\d{2,4}\b/gu, 'date')
+            .replace(/[\p{P}\p{S}]+/gu, ' ')
+            .split(/\s+/u)
+            .filter(token => token.length > 0 && !this.isStopWord(token))
+            .join(' ')
+            .trim();
+    }
+    normalizedHash(text) {
+        const normalized = this.normalizeText(text);
+        if (normalized.length === 0)
+            return undefined;
+        return crypto.createHash('sha256').update(normalized).digest('hex');
+    }
+    relationshipForFormats(sourceFormat, targetFormat) {
+        if (this.looksTranslationFormat(sourceFormat) || this.looksTranslationFormat(targetFormat))
+            return 'translation';
+        return 'format_variant';
+    }
+    computeMinHash(text, shingleSize = 5) {
+        const tokens = this.normalizeText(text).split(/\s+/u).filter(Boolean);
+        if (tokens.length < shingleSize)
+            return undefined;
+        const shingles = new Set();
+        for (let i = 0; i <= tokens.length - shingleSize; i++) {
+            shingles.add(tokens.slice(i, i + shingleSize).join(' '));
+        }
+        if (shingles.size === 0)
+            return undefined;
+        const signature = Array.from({ length: this.hashCount }, () => Number.MAX_SAFE_INTEGER);
+        for (const shingle of shingles) {
+            for (let seed = 0; seed < this.hashCount; seed++) {
+                const value = this.hashToUint32(`${seed}:${shingle}`);
+                if (value < signature[seed])
+                    signature[seed] = value;
+            }
+        }
+        return { signature, shingleCount: shingles.size };
+    }
+    estimateSimilarity(a, b) {
+        const length = Math.min(a.length, b.length);
+        if (length === 0)
+            return 0;
+        let equal = 0;
+        for (let i = 0; i < length; i++) {
+            if (a[i] === b[i])
+                equal += 1;
+        }
+        return equal / length;
+    }
+    relationshipForSimilarity(similarity) {
+        if (similarity >= 0.95)
+            return 'near_duplicate';
+        if (similarity >= 0.8)
+            return 'revision';
+        return undefined;
+    }
+    hashToUint32(input) {
+        return crypto.createHash('sha256').update(input).digest().readUInt32BE(0);
+    }
+    looksTranslationFormat(format) {
+        return ['translation', 'bilingual'].includes(format);
+    }
+    isStopWord(token) {
+        return STOP_WORDS.has(token);
+    }
+}
+const STOP_WORDS = new Set([
+    'the', 'a', 'an', 'and', 'or', 'of', 'to', 'in', 'for', 'on', 'with', 'by',
+    '是', '的', '了', '和', '与', '及', '或', '在', '为', '对', '中', '本文',
+]);

package/dist/dedup/relationship-detector.d.ts

@@ -0,0 +1,10 @@
+import type { ClassifiedFile, IndexStateRecord } from '../types.js';
+import type { FileRelationship } from '../core/index-state-store.js';
+export declare class RelationshipDetector {
+    detect(file: ClassifiedFile, indexedRecords: IndexStateRecord[]): Array<Omit<FileRelationship, 'id' | 'createdAt'>>;
+    private detectVersionChain;
+    private detectTranslation;
+    private detectSameDirectoryComplement;
+    private parseVersion;
+    private parseLanguageSuffix;
+}

package/dist/dedup/relationship-detector.js

@@ -0,0 +1,84 @@
+import * as path from 'node:path';
+export class RelationshipDetector {
+    detect(file, indexedRecords) {
+        const relationships = [];
+        for (const record of indexedRecords) {
+            if (record.relativePath === file.relativePath)
+                continue;
+            const version = this.detectVersionChain(file.relativePath, record.relativePath);
+            if (version)
+                relationships.push(version);
+            const translation = this.detectTranslation(file.relativePath, record.relativePath);
+            if (translation)
+                relationships.push(translation);
+            const complementary = this.detectSameDirectoryComplement(file, record);
+            if (complementary)
+                relationships.push(complementary);
+        }
+        return relationships;
+    }
+    detectVersionChain(source, target) {
+        const sourceVersion = this.parseVersion(source);
+        const targetVersion = this.parseVersion(target);
+        if (!sourceVersion || !targetVersion)
+            return undefined;
+        if (sourceVersion.base !== targetVersion.base)
+            return undefined;
+        return {
+            sourceFile: source,
+            targetFile: target,
+            relationshipType: 'version_chain',
+            confidence: 0.9,
+            detail: `版本链: v${targetVersion.version} → v${sourceVersion.version}`,
+            userConfirmed: 0,
+        };
+    }
+    detectTranslation(source, target) {
+        const sourceLang = this.parseLanguageSuffix(source);
+        const targetLang = this.parseLanguageSuffix(target);
+        if (!sourceLang || !targetLang)
+            return undefined;
+        if (sourceLang.base !== targetLang.base || sourceLang.lang === targetLang.lang)
+            return undefined;
+        return {
+            sourceFile: source,
+            targetFile: target,
+            relationshipType: 'translation',
+            confidence: 0.85,
+            detail: `语言版本: ${targetLang.lang} ↔ ${sourceLang.lang}`,
+            userConfirmed: 0,
+        };
+    }
+    detectSameDirectoryComplement(file, record) {
+        if (path.dirname(file.relativePath) !== path.dirname(record.relativePath))
+            return undefined;
+        if (file.category !== record.category)
+            return undefined;
+        const max = Math.max(file.fileSize, record.fileSize);
+        const min = Math.min(file.fileSize, record.fileSize);
+        if (max === 0 || min / max < 0.6)
+            return undefined;
+        return {
+            sourceFile: file.relativePath,
+            targetFile: record.relativePath,
+            relationshipType: 'complementary',
+            confidence: 0.45,
+            detail: '同目录、同类型且文件大小相近',
+            userConfirmed: 0,
+        };
+    }
+    parseVersion(filePath) {
+        const parsed = path.parse(filePath);
+        const match = parsed.name.match(/^(.*?)(?:[_-]?v)(\d+)$/iu);
+        if (!match?.[1] || !match[2])
+            return undefined;
+        return { base: path.join(parsed.dir, match[1]).toLowerCase(), version: Number(match[2]) };
+    }
+    parseLanguageSuffix(filePath) {
+        const parsed = path.parse(filePath);
+        const match = parsed.name.match(/^(.*?)[_-](cn|zh|en)$/iu);
+        if (!match?.[1] || !match[2])
+            return undefined;
+        return { base: path.join(parsed.dir, match[1]).toLowerCase(), lang: match[2].toLowerCase() };
+    }
+}

package/dist/embedding/embedding-provider.d.ts

@@ -0,0 +1,15 @@
+export interface EmbeddingProvider {
+    readonly model: string;
+    readonly dimensions: number;
+    embedDocuments(texts: string[]): Promise<number[][]>;
+    embedQuery(text: string): Promise<number[]>;
+}
+export declare class HashEmbeddingProvider implements EmbeddingProvider {
+    readonly dimensions: number;
+    readonly model = "hash-embedding-local";
+    constructor(dimensions?: number);
+    embedDocuments(texts: string[]): Promise<number[][]>;
+    embedQuery(text: string): Promise<number[]>;
+    private embed;
+    private normalize;
+}

package/dist/embedding/embedding-provider.js

@@ -0,0 +1,31 @@
+import * as crypto from 'node:crypto';
+export class HashEmbeddingProvider {
+    dimensions;
+    model = 'hash-embedding-local';
+    constructor(dimensions = 384) {
+        this.dimensions = dimensions;
+    }
+    async embedDocuments(texts) {
+        return texts.map(text => this.embed(text));
+    }
+    async embedQuery(text) {
+        return this.embed(text);
+    }
+    embed(text) {
+        const vector = Array.from({ length: this.dimensions }, () => 0);
+        const tokens = text.toLowerCase().normalize('NFKC').split(/\s+/u).filter(Boolean);
+        for (const token of tokens) {
+            const hash = crypto.createHash('sha256').update(token).digest();
+            const index = hash.readUInt32BE(0) % this.dimensions;
+            const sign = hash.readUInt8(4) % 2 === 0 ? 1 : -1;
+            vector[index] = (vector[index] ?? 0) + sign;
+        }
+        return this.normalize(vector);
+    }
+    normalize(vector) {
+        const norm = Math.sqrt(vector.reduce((sum, value) => sum + value * value, 0));
+        if (norm === 0)
+            return vector;
+        return vector.map(value => value / norm);
+    }
+}

package/dist/extraction/content-extractor.d.ts

@@ -0,0 +1,36 @@
+import { ExternalExtractorRegistry } from './external-extractor.js';
+import type { ClassifiedFile } from '../types.js';
+export interface ExtractionResult {
+    text: string;
+    metadata: Record<string, unknown>;
+    warnings: string[];
+    extractionTimeMs: number;
+}
+export declare class ContentExtractor {
+    private readonly externalExtractors;
+    constructor(externalExtractors?: ExternalExtractorRegistry);
+    extract(file: ClassifiedFile): Promise<ExtractionResult>;
+    private tryExternalExtractor;
+    private extractCad;
+    private extractCadMesh;
+    private extractBinaryStrings;
+    private extractData;
+    private extractDiagram;
+    private parseDelimitedLine;
+    private extractDelimitedText;
+    private extractOfficeDocument;
+    private extractSpreadsheet;
+    private extractOfficeZip;
+    private extractArchive;
+    private extractRasterImage;
+    private validateRasterImage;
+    private extractPdf;
+    private extractPdfText;
+    private extractSvg;
+    private isTextReadable;
+    private looksTextFile;
+    private matchAll;
+    private stripXml;
+    private flattenJson;
+    private metadataOnlyText;
+}