@customize-agent/knowledge 1.0.1

package/dist/chunking/text-chunker.d.ts

@@ -0,0 +1,24 @@
+import type { ClassifiedFile } from '../types.js';
+export interface TextChunk {
+    index: number;
+    text: string;
+    startChar: number;
+    endChar: number;
+    tokenCount: number;
+    sectionTitle?: string;
+    metadata: Record<string, unknown>;
+}
+export interface ChunkConfig {
+    maxChunkSize: number;
+    overlap: number;
+    headerInjection: boolean;
+}
+export declare class TextChunker {
+    chunk(text: string, file: ClassifiedFile, metadata?: Record<string, unknown>): TextChunk[];
+    private withHeader;
+    private splitBySemanticBoundary;
+    private createChunk;
+    private extractSectionTitle;
+    private takeOverlap;
+    private estimateTokens;
+}

package/dist/chunking/text-chunker.js

@@ -0,0 +1,90 @@
+const DEFAULT_CONFIGS = {
+    document: { maxChunkSize: 800, overlap: 100, headerInjection: true },
+    spreadsheet: { maxChunkSize: 1000, overlap: 200, headerInjection: true },
+    image: { maxChunkSize: 512, overlap: 0, headerInjection: true },
+    cad: { maxChunkSize: 600, overlap: 100, headerInjection: true },
+    code: { maxChunkSize: 1000, overlap: 200, headerInjection: true },
+    data: { maxChunkSize: 600, overlap: 100, headerInjection: true },
+    web: { maxChunkSize: 800, overlap: 100, headerInjection: true },
+    diagram: { maxChunkSize: 512, overlap: 0, headerInjection: true },
+    archive: { maxChunkSize: 500, overlap: 50, headerInjection: false },
+    other: { maxChunkSize: 500, overlap: 50, headerInjection: false },
+};
+export class TextChunker {
+    chunk(text, file, metadata = {}) {
+        if (text.trim().length === 0)
+            return [];
+        const config = DEFAULT_CONFIGS[file.category];
+        const normalized = this.withHeader(text, file, config);
+        const paragraphs = this.splitBySemanticBoundary(normalized, file.category);
+        const chunks = [];
+        let buffer = '';
+        let chunkStart = 0;
+        let cursor = 0;
+        for (const paragraph of paragraphs) {
+            const candidate = buffer.length === 0 ? paragraph : `${buffer}\n\n${paragraph}`;
+            if (this.estimateTokens(candidate) > config.maxChunkSize && buffer.length > 0) {
+                chunks.push(this.createChunk(chunks.length, buffer, chunkStart, cursor, metadata));
+                const overlapText = this.takeOverlap(buffer, config.overlap);
+                buffer = overlapText.length > 0 ? `${overlapText}\n\n${paragraph}` : paragraph;
+                chunkStart = Math.max(0, cursor - overlapText.length);
+            }
+            else {
+                buffer = candidate;
+            }
+            cursor += paragraph.length + 2;
+        }
+        if (buffer.trim().length > 0) {
+            chunks.push(this.createChunk(chunks.length, buffer, chunkStart, normalized.length, metadata));
+        }
+        return chunks;
+    }
+    withHeader(text, file, config) {
+        if (!config.headerInjection)
+            return text;
+        return `文件: ${file.relativePath}\n类型: ${file.category}/${file.format}\n\n${text}`;
+    }
+    splitBySemanticBoundary(text, category) {
+        if (category === 'document') {
+            return text.split(/\n(?=#{1,6}\s)|\n{2,}/u).map(part => part.trim()).filter(Boolean);
+        }
+        if (category === 'code') {
+            return text.split(/\n(?=(export\s+)?(async\s+)?(function|class|interface|type|const|let|var)\s)/u).map(part => part.trim()).filter(Boolean);
+        }
+        if (category === 'data') {
+            return text.split(/\n(?=[\w.[\]-]+:\s)|\n{2,}/u).map(part => part.trim()).filter(Boolean);
+        }
+        if (category === 'cad' || category === 'diagram') {
+            return text.split(/\n(?=(?:CAD|STEP|IGES|Mesh|Draw\.io|Excalidraw|SVG)\b)|\n{2,}/u).map(part => part.trim()).filter(Boolean);
+        }
+        return text.split(/\n{2,}/u).map(part => part.trim()).filter(Boolean);
+    }
+    createChunk(index, text, startChar, endChar, metadata) {
+        return {
+            index,
+            text: text.trim(),
+            startChar,
+            endChar,
+            tokenCount: this.estimateTokens(text),
+            sectionTitle: this.extractSectionTitle(text),
+            metadata,
+        };
+    }
+    extractSectionTitle(text) {
+        const firstLine = text.trim().split(/\r?\n/u)[0]?.trim();
+        if (!firstLine)
+            return undefined;
+        if (firstLine.startsWith('#'))
+            return firstLine.replace(/^#+\s*/u, '');
+        return firstLine.length <= 80 ? firstLine : undefined;
+    }
+    takeOverlap(text, overlapTokens) {
+        if (overlapTokens <= 0)
+            return '';
+        const chars = overlapTokens * 4;
+        return text.slice(Math.max(0, text.length - chars));
+    }
+    estimateTokens(text) {
+        return Math.max(1, Math.ceil(text.length / 4));
+    }
+}

package/dist/classification/classifier.d.ts

@@ -0,0 +1,12 @@
+import type { Stats } from 'node:fs';
+import type { ClassifiedFile, FileCategory } from '../types.js';
+export declare class FileClassifier {
+    private readonly extensionMap;
+    constructor();
+    classify(absolutePath: string, relativePath: string, stat: Stats): ClassifiedFile;
+    classifyVirtual(fileName: string): Pick<ClassifiedFile, 'category' | 'format' | 'mimeType'>;
+    groupByCategory(files: ClassifiedFile[]): Map<FileCategory, ClassifiedFile[]>;
+    shouldSkip(file: ClassifiedFile): string | null;
+    private buildExtensionMap;
+    private inferMimeType;
+}

package/dist/classification/classifier.js

@@ -0,0 +1,95 @@
+import * as path from 'node:path';
+export class FileClassifier {
+    extensionMap;
+    constructor() {
+        this.extensionMap = this.buildExtensionMap();
+    }
+    classify(absolutePath, relativePath, stat) {
+        const ext = path.extname(absolutePath).toLowerCase();
+        const [category, format] = this.extensionMap.get(ext) ?? ['other', 'unknown'];
+        return {
+            absolutePath,
+            relativePath,
+            category,
+            format,
+            fileSize: stat.size,
+            mtime: stat.mtimeMs,
+            mimeType: this.inferMimeType(ext, category),
+        };
+    }
+    classifyVirtual(fileName) {
+        const ext = path.extname(fileName).toLowerCase();
+        const [category, format] = this.extensionMap.get(ext) ?? ['other', 'unknown'];
+        return { category, format, mimeType: this.inferMimeType(ext, category) };
+    }
+    groupByCategory(files) {
+        const groups = new Map();
+        for (const file of files) {
+            const list = groups.get(file.category) ?? [];
+            list.push(file);
+            groups.set(file.category, list);
+        }
+        return groups;
+    }
+    shouldSkip(file) {
+        if (file.fileSize > 50 * 1024 * 1024)
+            return '文件超过 50MB 限制';
+        if (file.fileSize === 0)
+            return '空文件';
+        const ext = path.extname(file.absolutePath).toLowerCase();
+        const skipExts = ['.exe', '.dll', '.so', '.dylib', '.bin', '.dat', '.class', '.pyc', '.o'];
+        if (skipExts.includes(ext))
+            return '二进制可执行文件，跳过';
+        const basename = path.basename(file.absolutePath);
+        if (basename.startsWith('.') || basename.startsWith('._'))
+            return '隐藏/系统文件，跳过';
+        return null;
+    }
+    buildExtensionMap() {
+        const entries = [
+            ['.pdf', 'document', 'pdf'], ['.docx', 'document', 'office'], ['.doc', 'document', 'office'], ['.rtf', 'document', 'office'], ['.odt', 'document', 'office'],
+            ['.pptx', 'document', 'presentation'], ['.ppt', 'document', 'presentation'], ['.odp', 'document', 'presentation'],
+            ['.md', 'document', 'markdown'], ['.markdown', 'document', 'markdown'], ['.mdx', 'document', 'markdown'],
+            ['.txt', 'document', 'plaintext'], ['.rst', 'document', 'plaintext'], ['.asciidoc', 'document', 'plaintext'], ['.tex', 'document', 'plaintext'],
+            ['.epub', 'document', 'ebook'], ['.mobi', 'document', 'ebook'],
+            ['.xlsx', 'spreadsheet', 'excel'], ['.xls', 'spreadsheet', 'excel'], ['.xlsm', 'spreadsheet', 'excel'],
+            ['.csv', 'spreadsheet', 'csv'], ['.tsv', 'spreadsheet', 'csv'], ['.tab', 'spreadsheet', 'csv'], ['.ods', 'spreadsheet', 'opendoc'],
+            ['.png', 'image', 'raster'], ['.jpg', 'image', 'raster'], ['.jpeg', 'image', 'raster'], ['.gif', 'image', 'raster'], ['.bmp', 'image', 'raster'], ['.webp', 'image', 'raster'], ['.tiff', 'image', 'raster'], ['.tif', 'image', 'raster'],
+            ['.svg', 'image', 'vector'], ['.eps', 'image', 'vector'], ['.raw', 'image', 'raw'], ['.cr2', 'image', 'raw'], ['.nef', 'image', 'raw'], ['.dng', 'image', 'raw'],
+            ['.dwg', 'cad', 'autocad'], ['.dxf', 'cad', 'autocad'], ['.dwt', 'cad', 'autocad'],
+            ['.step', 'cad', 'step'], ['.stp', 'cad', 'step'], ['.p21', 'cad', 'step'], ['.iges', 'cad', 'iges'], ['.igs', 'cad', 'iges'],
+            ['.stl', 'cad', 'mesh'], ['.obj', 'cad', 'mesh'], ['.3mf', 'cad', 'mesh'], ['.fbx', 'cad', 'mesh'], ['.glb', 'cad', 'mesh'], ['.gltf', 'cad', 'mesh'],
+            ['.sldprt', 'cad', 'solidworks'], ['.sldasm', 'cad', 'solidworks'], ['.slddrw', 'cad', 'solidworks'],
+            ['.ts', 'code', 'typescript'], ['.tsx', 'code', 'typescript'], ['.mts', 'code', 'typescript'], ['.cts', 'code', 'typescript'],
+            ['.js', 'code', 'javascript'], ['.jsx', 'code', 'javascript'], ['.mjs', 'code', 'javascript'], ['.cjs', 'code', 'javascript'],
+            ['.py', 'code', 'python'], ['.pyi', 'code', 'python'], ['.pyx', 'code', 'python'], ['.ipynb', 'code', 'python'],
+            ['.java', 'code', 'java_kotlin'], ['.kt', 'code', 'java_kotlin'], ['.scala', 'code', 'java_kotlin'],
+            ['.c', 'code', 'c_family'], ['.cpp', 'code', 'c_family'], ['.cc', 'code', 'c_family'], ['.cxx', 'code', 'c_family'], ['.h', 'code', 'c_family'], ['.hpp', 'code', 'c_family'],
+            ['.go', 'code', 'go'], ['.rs', 'code', 'rust'], ['.rb', 'code', 'ruby'], ['.php', 'code', 'php'],
+            ['.sh', 'code', 'shell'], ['.bash', 'code', 'shell'], ['.zsh', 'code', 'shell'], ['.fish', 'code', 'shell'], ['.sql', 'code', 'sql'],
+            ['.toml', 'code', 'config'], ['.ini', 'code', 'config'], ['.cfg', 'code', 'config'], ['.conf', 'code', 'config'], ['.env', 'code', 'config'],
+            ['.json', 'data', 'json'], ['.jsonl', 'data', 'json'], ['.json5', 'data', 'json'], ['.geojson', 'data', 'json'],
+            ['.yaml', 'data', 'yaml'], ['.yml', 'data', 'yaml'], ['.xml', 'data', 'xml'], ['.xsd', 'data', 'xml'], ['.wsdl', 'data', 'xml'], ['.proto', 'data', 'protobuf'], ['.graphql', 'data', 'graphql'], ['.gql', 'data', 'graphql'],
+            ['.html', 'web', 'html'], ['.htm', 'web', 'html'], ['.xhtml', 'web', 'html'], ['.css', 'web', 'stylesheet'], ['.scss', 'web', 'stylesheet'], ['.sass', 'web', 'stylesheet'], ['.less', 'web', 'stylesheet'],
+            ['.hbs', 'web', 'template'], ['.ejs', 'web', 'template'], ['.pug', 'web', 'template'], ['.j2', 'web', 'template'], ['.jinja2', 'web', 'template'],
+            ['.drawio', 'diagram', 'drawio'], ['.dio', 'diagram', 'drawio'], ['.vsdx', 'diagram', 'visio'], ['.vdx', 'diagram', 'visio'], ['.puml', 'diagram', 'plantuml'], ['.plantuml', 'diagram', 'plantuml'], ['.mmd', 'diagram', 'mermaid'], ['.mermaid', 'diagram', 'mermaid'], ['.excalidraw', 'diagram', 'excalidraw'],
+            ['.zip', 'archive', 'zip'], ['.jar', 'archive', 'zip'], ['.war', 'archive', 'zip'], ['.apk', 'archive', 'zip'], ['.tar', 'archive', 'tar'], ['.gz', 'archive', 'other'], ['.tgz', 'archive', 'tar'], ['.bz2', 'archive', 'other'], ['.rar', 'archive', 'other'], ['.7z', 'archive', 'other'],
+        ];
+        return new Map(entries.map(([ext, category, format]) => [ext, [category, format]]));
+    }
+    inferMimeType(ext, category) {
+        const known = {
+            '.pdf': 'application/pdf',
+            '.json': 'application/json',
+            '.md': 'text/markdown',
+            '.txt': 'text/plain',
+            '.csv': 'text/csv',
+            '.html': 'text/html',
+            '.png': 'image/png',
+            '.jpg': 'image/jpeg',
+            '.jpeg': 'image/jpeg',
+            '.svg': 'image/svg+xml',
+        };
+        return known[ext] ?? `application/x-customize-agent-${category}`;
+    }
+}

package/dist/constants.d.ts

@@ -0,0 +1,8 @@
+import type { FileCategory } from './types.js';
+export declare const KNOWLEDGE_BASE_DIR = "knowledgeBase";
+export declare const PROJECT_CONFIG_PATH: readonly [".customize-agent", "kb", "project.json"];
+export declare const USER_DATA_DIR = ".customize-agent";
+export declare const GLOBAL_KNOWLEDGE_DIR = "global-knowledge";
+export declare const ALL_CATEGORIES: readonly FileCategory[];
+export declare const DEFAULT_CATEGORY_DIRS: Record<FileCategory, string>;
+export declare const COLLECTION_CATEGORY_NAMES: Record<FileCategory, string>;

package/dist/constants.js

@@ -0,0 +1,40 @@
+export const KNOWLEDGE_BASE_DIR = 'knowledgeBase';
+export const PROJECT_CONFIG_PATH = ['.customize-agent', 'kb', 'project.json'];
+export const USER_DATA_DIR = '.customize-agent';
+export const GLOBAL_KNOWLEDGE_DIR = 'global-knowledge';
+export const ALL_CATEGORIES = [
+    'document',
+    'spreadsheet',
+    'image',
+    'cad',
+    'code',
+    'data',
+    'web',
+    'diagram',
+    'archive',
+    'other',
+];
+export const DEFAULT_CATEGORY_DIRS = {
+    document: '文档资料',
+    spreadsheet: '表格数据',
+    image: '图片素材',
+    cad: '图纸文件',
+    code: '代码文件',
+    data: '数据文件',
+    web: '网页文件',
+    diagram: '图表流程',
+    archive: '压缩包',
+    other: '其他文件',
+};
+export const COLLECTION_CATEGORY_NAMES = {
+    document: 'documents',
+    spreadsheet: 'spreadsheets',
+    image: 'images',
+    cad: 'cad',
+    code: 'code',
+    data: 'data',
+    web: 'web',
+    diagram: 'diagrams',
+    archive: 'archives',
+    other: 'other',
+};

package/dist/core/change-tracker.d.ts

@@ -0,0 +1,10 @@
+import type { FileClassifier } from '../classification/classifier.js';
+import type { DiffResult } from '../types.js';
+import type { DiskFileStat } from './file-scanner.js';
+import type { IndexStateStore } from './index-state-store.js';
+export declare class ChangeTracker {
+    private readonly store;
+    constructor(store: IndexStateStore);
+    computeDiff(diskFiles: Map<string, DiskFileStat>, classifier: FileClassifier, kbPath: string): Promise<DiffResult>;
+    hashFile(filePath: string): string;
+}

package/dist/core/change-tracker.js

@@ -0,0 +1,66 @@
+import * as crypto from 'node:crypto';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+export class ChangeTracker {
+    store;
+    constructor(store) {
+        this.store = store;
+    }
+    async computeDiff(diskFiles, classifier, kbPath) {
+        const startTime = Date.now();
+        const indexedFiles = this.store.loadActiveRecords();
+        const newFiles = [];
+        const modifiedFiles = [];
+        const deletedFiles = [];
+        const skippedFiles = [];
+        let unchangedCount = 0;
+        let mtimeOnlyCount = 0;
+        for (const [relativePath, diskStat] of diskFiles) {
+            const absolutePath = path.join(kbPath, relativePath);
+            const stat = fs.statSync(absolutePath);
+            const classified = classifier.classify(absolutePath, relativePath, stat);
+            const skipReason = classifier.shouldSkip(classified);
+            if (skipReason) {
+                skippedFiles.push({ file: classified, reason: skipReason });
+                continue;
+            }
+            const indexed = indexedFiles.get(relativePath);
+            if (!indexed) {
+                newFiles.push(classified);
+                continue;
+            }
+            if (Math.round(diskStat.mtime) !== Math.round(indexed.mtime) || diskStat.size !== indexed.fileSize) {
+                const contentHash = this.hashFile(absolutePath);
+                if (contentHash !== indexed.contentHash) {
+                    modifiedFiles.push(classified);
+                }
+                else {
+                    mtimeOnlyCount += 1;
+                    this.store.updateVerified(relativePath, diskStat.mtime);
+                }
+            }
+            else {
+                unchangedCount += 1;
+                this.store.updateVerified(relativePath, diskStat.mtime);
+            }
+        }
+        for (const [relativePath, record] of indexedFiles) {
+            if (!diskFiles.has(relativePath))
+                deletedFiles.push(record);
+        }
+        const hasChanges = newFiles.length > 0 || modifiedFiles.length > 0 || deletedFiles.length > 0;
+        return {
+            newFiles,
+            modifiedFiles,
+            deletedFiles,
+            unchangedCount,
+            mtimeOnlyCount,
+            skippedFiles,
+            hasChanges,
+            diffTimeMs: Date.now() - startTime,
+        };
+    }
+    hashFile(filePath) {
+        return crypto.createHash('sha256').update(fs.readFileSync(filePath)).digest('hex');
+    }
+}

package/dist/core/file-scanner.d.ts

@@ -0,0 +1,8 @@
+export interface DiskFileStat {
+    size: number;
+    mtime: number;
+}
+export declare class KnowledgeFileScanner {
+    scan(kbPath: string, ignorePatterns?: string[]): Promise<Map<string, DiskFileStat>>;
+    loadKbIgnore(kbPath: string): string[];
+}

package/dist/core/file-scanner.js

@@ -0,0 +1,32 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import fg from 'fast-glob';
+export class KnowledgeFileScanner {
+    async scan(kbPath, ignorePatterns = []) {
+        if (!fs.existsSync(kbPath))
+            return new Map();
+        const entries = await fg('**/*', {
+            cwd: kbPath,
+            onlyFiles: true,
+            dot: true,
+            ignore: ['.kbignore', ...ignorePatterns],
+            unique: true,
+        });
+        const files = new Map();
+        for (const relativePath of entries) {
+            const absolutePath = path.join(kbPath, relativePath);
+            const stat = fs.statSync(absolutePath);
+            files.set(relativePath, { size: stat.size, mtime: stat.mtimeMs });
+        }
+        return files;
+    }
+    loadKbIgnore(kbPath) {
+        const ignorePath = path.join(kbPath, '.kbignore');
+        if (!fs.existsSync(ignorePath))
+            return [];
+        return fs.readFileSync(ignorePath, 'utf8')
+            .split(/\r?\n/u)
+            .map(line => line.trim())
+            .filter(line => line.length > 0 && !line.startsWith('#'));
+    }
+}

package/dist/core/index-state-store.d.ts

@@ -0,0 +1,103 @@
+import type { TextChunk } from '../chunking/text-chunker.js';
+import type { FileCategory, IndexStateRecord } from '../types.js';
+export interface StoredChunk {
+    id: string;
+    relativePath: string;
+    chunkIndex: number;
+    content: string;
+    category: FileCategory;
+    format: string;
+    collectionName: string;
+    tokenCount: number;
+    sectionTitle?: string;
+    metadataJson?: string;
+    createdAt: number;
+}
+export interface ChunkSearchResult extends StoredChunk {
+    score: number;
+}
+export interface FileHashRecord {
+    contentHash: string;
+    filePath: string;
+    fileSize: number;
+    category: FileCategory;
+    normalizedHash?: string;
+    createdAt: number;
+    updatedAt: number;
+}
+export interface MinHashRecord {
+    filePath: string;
+    signature: number[];
+    shingleCount: number;
+    createdAt: number;
+}
+export interface FileRelationship {
+    id?: number;
+    sourceFile: string;
+    targetFile: string;
+    relationshipType: 'exact_duplicate' | 'format_variant' | 'translation' | 'near_duplicate' | 'revision' | 'version_chain' | 'derived' | 'complementary';
+    confidence: number;
+    detail?: string;
+    userConfirmed: number;
+    createdAt: number;
+}
+export declare class IndexStateStore {
+    private readonly db;
+    constructor(dbPath: string);
+    loadActiveRecords(): Map<string, IndexStateRecord>;
+    upsertRecord(record: IndexStateRecord): void;
+    updateVerified(relativePath: string, mtime: number): void;
+    listRecords(): IndexStateRecord[];
+    replaceChunks(relativePath: string, chunks: TextChunk[], file: {
+        category: FileCategory;
+        format: string;
+        collectionName: string;
+    }): void;
+    listChunks(options?: {
+        collectionName?: string;
+        relativePath?: string;
+        limit?: number;
+    }): StoredChunk[];
+    searchChunks(query: string, limit?: number): ChunkSearchResult[];
+    findExactDuplicate(contentHash: string, excludePath?: string): FileHashRecord | undefined;
+    findNormalizedDuplicate(normalizedHash: string, excludePath?: string): FileHashRecord | undefined;
+    upsertFileHash(record: Omit<FileHashRecord, 'createdAt' | 'updatedAt'>): void;
+    upsertMinHash(record: Omit<MinHashRecord, 'createdAt'>): void;
+    listMinHashes(excludePath?: string): MinHashRecord[];
+    addRelationship(relationship: Omit<FileRelationship, 'id' | 'createdAt'>): void;
+    listRelationships(filePath?: string): FileRelationship[];
+    setTags(relativePath: string, tags: string[]): void;
+    listTags(relativePath?: string): Array<{
+        filePath: string;
+        tag: string;
+        createdAt: number;
+    }>;
+    addIgnoreRule(pattern: string): void;
+    listIgnoreRules(): Array<{
+        id: number;
+        pattern: string;
+        enabled: boolean;
+        createdAt: number;
+    }>;
+    deleteRecord(relativePath: string): void;
+    setMetadata(key: string, value: string): void;
+    getStats(): {
+        fileCount: number;
+        chunkCount: number;
+        totalSizeBytes: number;
+        lastIndexedAt: number;
+    };
+    listContentHashes(): Array<{
+        contentHash: string;
+        relativePath: string;
+    }>;
+    close(): void;
+    private initTables;
+    private rowToMinHash;
+    private rowToFileHash;
+    private rowToRelationship;
+    private rowToChunk;
+    private expandSearchTerms;
+    private scoreChunk;
+    private rowToRecord;
+}