@customize-agent/knowledge 1.0.1

package/dist/core/index-state-store.js

@@ -0,0 +1,439 @@
+import Database from 'better-sqlite3';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+export class IndexStateStore {
+    db;
+    constructor(dbPath) {
+        fs.mkdirSync(path.dirname(dbPath), { recursive: true });
+        this.db = new Database(dbPath);
+        this.db.pragma('journal_mode = WAL');
+        this.initTables();
+    }
+    loadActiveRecords() {
+        const rows = this.db.prepare(`
+      SELECT * FROM kb_index_state
+      WHERE status IN ('active', 'outdated', 'error')
+    `).all();
+        return new Map(rows.map(row => {
+            const record = this.rowToRecord(row);
+            return [record.relativePath, record];
+        }));
+    }
+    upsertRecord(record) {
+        this.db.prepare(`
+      INSERT INTO kb_index_state (
+        relative_path, category, format, content_hash, file_size, mtime,
+        chunk_count, collection_name, indexed_at, last_verified_at,
+        status, error_message, metadata_json
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      ON CONFLICT(relative_path) DO UPDATE SET
+        category = excluded.category,
+        format = excluded.format,
+        content_hash = excluded.content_hash,
+        file_size = excluded.file_size,
+        mtime = excluded.mtime,
+        chunk_count = excluded.chunk_count,
+        collection_name = excluded.collection_name,
+        last_verified_at = excluded.last_verified_at,
+        status = excluded.status,
+        error_message = excluded.error_message,
+        metadata_json = excluded.metadata_json
+    `).run(record.relativePath, record.category, record.format, record.contentHash, record.fileSize, Math.round(record.mtime), record.chunkCount, record.collectionName, record.indexedAt, record.lastVerifiedAt, record.status, record.errorMessage ?? null, record.metadataJson ?? null);
+    }
+    updateVerified(relativePath, mtime) {
+        this.db.prepare(`
+      UPDATE kb_index_state
+      SET mtime = ?, last_verified_at = ?, status = 'active'
+      WHERE relative_path = ?
+    `).run(Math.round(mtime), Date.now(), relativePath);
+    }
+    listRecords() {
+        const rows = this.db.prepare(`
+      SELECT * FROM kb_index_state
+      WHERE status = 'active'
+      ORDER BY category, relative_path
+    `).all();
+        return rows.map(row => this.rowToRecord(row));
+    }
+    replaceChunks(relativePath, chunks, file) {
+        const now = Date.now();
+        const transaction = this.db.transaction(() => {
+            this.db.prepare('DELETE FROM kb_chunks WHERE relative_path = ?').run(relativePath);
+            const insert = this.db.prepare(`
+        INSERT INTO kb_chunks (
+          id, relative_path, chunk_index, content, category, format,
+          collection_name, token_count, section_title, metadata_json, created_at
+        ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `);
+            for (const chunk of chunks) {
+                insert.run(`${relativePath}#${chunk.index}`, relativePath, chunk.index, chunk.text, file.category, file.format, file.collectionName, chunk.tokenCount, chunk.sectionTitle ?? null, JSON.stringify(chunk.metadata), now);
+            }
+        });
+        transaction();
+    }
+    listChunks(options = {}) {
+        const conditions = [];
+        const params = [];
+        if (options.collectionName) {
+            conditions.push('collection_name = ?');
+            params.push(options.collectionName);
+        }
+        if (options.relativePath) {
+            conditions.push('relative_path = ?');
+            params.push(options.relativePath);
+        }
+        const where = conditions.length > 0 ? `WHERE ${conditions.join(' AND ')}` : '';
+        const limit = options.limit ? 'LIMIT ?' : '';
+        if (options.limit)
+            params.push(options.limit);
+        const rows = this.db.prepare(`
+      SELECT * FROM kb_chunks
+      ${where}
+      ORDER BY relative_path, chunk_index
+      ${limit}
+    `).all(...params);
+        return rows.map(row => this.rowToChunk(row, 0));
+    }
+    searchChunks(query, limit = 10) {
+        const terms = this.expandSearchTerms(query);
+        if (terms.length === 0)
+            return [];
+        const rows = this.db.prepare(`
+      SELECT * FROM kb_chunks
+      WHERE ${terms.map(() => '(LOWER(content) LIKE ? OR LOWER(relative_path) LIKE ? OR LOWER(category) LIKE ? OR LOWER(format) LIKE ?)').join(' OR ')}
+      ORDER BY created_at DESC
+      LIMIT ?
+    `).all(...terms.flatMap(term => [`%${term}%`, `%${term}%`, `%${term}%`, `%${term}%`]), limit * 6);
+        return rows
+            .map(row => this.rowToChunk(row, this.scoreChunk(`${String(row.relative_path)}\n${String(row.category)}\n${String(row.format)}\n${String(row.content)}`, terms)))
+            .filter(row => row.score > 0)
+            .sort((a, b) => b.score - a.score)
+            .slice(0, limit);
+    }
+    findExactDuplicate(contentHash, excludePath) {
+        const row = this.db.prepare(`
+      SELECT * FROM kb_file_hashes
+      WHERE content_hash = ? ${excludePath ? 'AND file_path != ?' : ''}
+      LIMIT 1
+    `).get(...(excludePath ? [contentHash, excludePath] : [contentHash]));
+        return row ? this.rowToFileHash(row) : undefined;
+    }
+    findNormalizedDuplicate(normalizedHash, excludePath) {
+        const row = this.db.prepare(`
+      SELECT * FROM kb_file_hashes
+      WHERE normalized_hash = ? ${excludePath ? 'AND file_path != ?' : ''}
+      LIMIT 1
+    `).get(...(excludePath ? [normalizedHash, excludePath] : [normalizedHash]));
+        return row ? this.rowToFileHash(row) : undefined;
+    }
+    upsertFileHash(record) {
+        const now = Date.now();
+        this.db.prepare(`
+      INSERT INTO kb_file_hashes (content_hash, file_path, file_size, category, normalized_hash, created_at, updated_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?)
+      ON CONFLICT(file_path) DO UPDATE SET
+        content_hash = excluded.content_hash,
+        file_size = excluded.file_size,
+        category = excluded.category,
+        normalized_hash = excluded.normalized_hash,
+        updated_at = excluded.updated_at
+    `).run(record.contentHash, record.filePath, record.fileSize, record.category, record.normalizedHash ?? null, now, now);
+    }
+    upsertMinHash(record) {
+        this.db.prepare(`
+      INSERT INTO kb_minhash (file_path, signature, shingle_count, created_at)
+      VALUES (?, ?, ?, ?)
+      ON CONFLICT(file_path) DO UPDATE SET
+        signature = excluded.signature,
+        shingle_count = excluded.shingle_count,
+        created_at = excluded.created_at
+    `).run(record.filePath, Buffer.from(JSON.stringify(record.signature), 'utf8'), record.shingleCount, Date.now());
+    }
+    listMinHashes(excludePath) {
+        const rows = excludePath
+            ? this.db.prepare('SELECT * FROM kb_minhash WHERE file_path != ?').all(excludePath)
+            : this.db.prepare('SELECT * FROM kb_minhash').all();
+        return rows.map(row => this.rowToMinHash(row));
+    }
+    addRelationship(relationship) {
+        this.db.prepare(`
+      INSERT INTO kb_relationships (
+        source_file, target_file, relationship_type, confidence, detail, user_confirmed, created_at
+      ) VALUES (?, ?, ?, ?, ?, ?, ?)
+      ON CONFLICT(source_file, target_file, relationship_type) DO UPDATE SET
+        confidence = excluded.confidence,
+        detail = excluded.detail,
+        user_confirmed = excluded.user_confirmed
+    `).run(relationship.sourceFile, relationship.targetFile, relationship.relationshipType, relationship.confidence, relationship.detail ?? null, relationship.userConfirmed, Date.now());
+    }
+    listRelationships(filePath) {
+        const rows = filePath
+            ? this.db.prepare('SELECT * FROM kb_relationships WHERE source_file = ? OR target_file = ? ORDER BY created_at DESC').all(filePath, filePath)
+            : this.db.prepare('SELECT * FROM kb_relationships ORDER BY created_at DESC').all();
+        return rows.map(row => this.rowToRelationship(row));
+    }
+    setTags(relativePath, tags) {
+        const now = Date.now();
+        const transaction = this.db.transaction(() => {
+            this.db.prepare('DELETE FROM kb_tags WHERE file_path = ?').run(relativePath);
+            const insert = this.db.prepare('INSERT OR IGNORE INTO kb_tags (file_path, tag, created_at) VALUES (?, ?, ?)');
+            for (const tag of tags.map(tag => tag.trim()).filter(Boolean))
+                insert.run(relativePath, tag, now);
+        });
+        transaction();
+    }
+    listTags(relativePath) {
+        const rows = relativePath
+            ? this.db.prepare('SELECT * FROM kb_tags WHERE file_path = ? ORDER BY tag').all(relativePath)
+            : this.db.prepare('SELECT * FROM kb_tags ORDER BY file_path, tag').all();
+        return rows.map(row => ({
+            filePath: String(row.file_path),
+            tag: String(row.tag),
+            createdAt: Number(row.created_at),
+        }));
+    }
+    addIgnoreRule(pattern) {
+        this.db.prepare('INSERT OR IGNORE INTO kb_ignore_rules (pattern, enabled, created_at) VALUES (?, 1, ?)').run(pattern, Date.now());
+    }
+    listIgnoreRules() {
+        const rows = this.db.prepare('SELECT * FROM kb_ignore_rules ORDER BY created_at DESC').all();
+        return rows.map(row => ({
+            id: Number(row.id),
+            pattern: String(row.pattern),
+            enabled: Number(row.enabled) === 1,
+            createdAt: Number(row.created_at),
+        }));
+    }
+    deleteRecord(relativePath) {
+        this.db.prepare('DELETE FROM kb_chunks WHERE relative_path = ?').run(relativePath);
+        this.db.prepare('DELETE FROM kb_index_state WHERE relative_path = ?').run(relativePath);
+        this.db.prepare('DELETE FROM kb_file_hashes WHERE file_path = ?').run(relativePath);
+        this.db.prepare('DELETE FROM kb_minhash WHERE file_path = ?').run(relativePath);
+        this.db.prepare('DELETE FROM kb_tags WHERE file_path = ?').run(relativePath);
+        this.db.prepare('DELETE FROM kb_relationships WHERE source_file = ? OR target_file = ?').run(relativePath, relativePath);
+    }
+    setMetadata(key, value) {
+        this.db.prepare(`
+      INSERT INTO kb_metadata (key, value) VALUES (?, ?)
+      ON CONFLICT(key) DO UPDATE SET value = excluded.value
+    `).run(key, value);
+    }
+    getStats() {
+        const stats = this.db.prepare(`
+      SELECT
+        COUNT(*) as file_count,
+        COALESCE(SUM(chunk_count), 0) as chunk_count,
+        COALESCE(SUM(file_size), 0) as total_size_bytes,
+        COALESCE(MAX(indexed_at), 0) as last_indexed_at
+      FROM kb_index_state
+      WHERE status = 'active'
+    `).get();
+        return {
+            fileCount: Number(stats.file_count ?? 0),
+            chunkCount: Number(stats.chunk_count ?? 0),
+            totalSizeBytes: Number(stats.total_size_bytes ?? 0),
+            lastIndexedAt: Number(stats.last_indexed_at ?? 0),
+        };
+    }
+    listContentHashes() {
+        const rows = this.db.prepare(`
+      SELECT content_hash, relative_path FROM kb_index_state WHERE status = 'active'
+    `).all();
+        return rows.map(row => ({ contentHash: String(row.content_hash), relativePath: String(row.relative_path) }));
+    }
+    close() {
+        this.db.close();
+    }
+    initTables() {
+        this.db.exec(`
+      CREATE TABLE IF NOT EXISTS kb_index_state (
+        relative_path     TEXT PRIMARY KEY,
+        category          TEXT NOT NULL,
+        format            TEXT NOT NULL,
+        content_hash      TEXT NOT NULL,
+        file_size         INTEGER NOT NULL,
+        mtime             INTEGER NOT NULL,
+        chunk_count       INTEGER NOT NULL DEFAULT 0,
+        collection_name   TEXT NOT NULL,
+        indexed_at        INTEGER NOT NULL,
+        last_verified_at  INTEGER NOT NULL,
+        status            TEXT NOT NULL DEFAULT 'active',
+        error_message     TEXT,
+        metadata_json     TEXT
+      );
+      CREATE INDEX IF NOT EXISTS idx_kb_state_status ON kb_index_state(status);
+      CREATE INDEX IF NOT EXISTS idx_kb_state_category ON kb_index_state(category);
+      CREATE INDEX IF NOT EXISTS idx_kb_state_collection ON kb_index_state(collection_name);
+
+      CREATE TABLE IF NOT EXISTS kb_chunks (
+        id              TEXT PRIMARY KEY,
+        relative_path   TEXT NOT NULL,
+        chunk_index     INTEGER NOT NULL,
+        content         TEXT NOT NULL,
+        category        TEXT NOT NULL,
+        format          TEXT NOT NULL,
+        collection_name TEXT NOT NULL,
+        token_count     INTEGER NOT NULL,
+        section_title   TEXT,
+        metadata_json   TEXT,
+        created_at      INTEGER NOT NULL
+      );
+      CREATE INDEX IF NOT EXISTS idx_kb_chunks_path ON kb_chunks(relative_path);
+      CREATE INDEX IF NOT EXISTS idx_kb_chunks_category ON kb_chunks(category);
+      CREATE INDEX IF NOT EXISTS idx_kb_chunks_collection ON kb_chunks(collection_name);
+
+      CREATE TABLE IF NOT EXISTS kb_file_hashes (
+        content_hash     TEXT NOT NULL,
+        file_path        TEXT PRIMARY KEY,
+        file_size        INTEGER NOT NULL,
+        category         TEXT NOT NULL,
+        normalized_hash  TEXT,
+        created_at       INTEGER NOT NULL,
+        updated_at       INTEGER NOT NULL
+      );
+      CREATE INDEX IF NOT EXISTS idx_kb_hashes_content ON kb_file_hashes(content_hash);
+      CREATE INDEX IF NOT EXISTS idx_kb_hashes_norm ON kb_file_hashes(normalized_hash);
+
+      CREATE TABLE IF NOT EXISTS kb_minhash (
+        file_path        TEXT PRIMARY KEY,
+        signature        BLOB NOT NULL,
+        shingle_count    INTEGER NOT NULL,
+        created_at       INTEGER NOT NULL
+      );
+
+      CREATE TABLE IF NOT EXISTS kb_relationships (
+        id                INTEGER PRIMARY KEY AUTOINCREMENT,
+        source_file       TEXT NOT NULL,
+        target_file       TEXT NOT NULL,
+        relationship_type TEXT NOT NULL,
+        confidence        REAL NOT NULL DEFAULT 1.0,
+        detail            TEXT,
+        user_confirmed    INTEGER NOT NULL DEFAULT 0,
+        created_at        INTEGER NOT NULL,
+        UNIQUE(source_file, target_file, relationship_type)
+      );
+      CREATE INDEX IF NOT EXISTS idx_rel_source ON kb_relationships(source_file);
+      CREATE INDEX IF NOT EXISTS idx_rel_target ON kb_relationships(target_file);
+      CREATE INDEX IF NOT EXISTS idx_rel_type ON kb_relationships(relationship_type);
+
+      CREATE TABLE IF NOT EXISTS kb_tags (
+        file_path   TEXT NOT NULL,
+        tag         TEXT NOT NULL,
+        created_at  INTEGER NOT NULL,
+        PRIMARY KEY (file_path, tag)
+      );
+      CREATE INDEX IF NOT EXISTS idx_tags_tag ON kb_tags(tag);
+
+      CREATE TABLE IF NOT EXISTS kb_metadata (
+        key   TEXT PRIMARY KEY,
+        value TEXT NOT NULL
+      );
+
+      CREATE TABLE IF NOT EXISTS kb_ignore_rules (
+        id          INTEGER PRIMARY KEY AUTOINCREMENT,
+        pattern     TEXT NOT NULL UNIQUE,
+        enabled     INTEGER NOT NULL DEFAULT 1,
+        created_at  INTEGER NOT NULL
+      );
+    `);
+        this.setMetadata('schema_version', '1');
+    }
+    rowToMinHash(row) {
+        const raw = row.signature;
+        const json = Buffer.isBuffer(raw) ? raw.toString('utf8') : String(raw);
+        return {
+            filePath: String(row.file_path),
+            signature: JSON.parse(json),
+            shingleCount: Number(row.shingle_count),
+            createdAt: Number(row.created_at),
+        };
+    }
+    rowToFileHash(row) {
+        return {
+            contentHash: String(row.content_hash),
+            filePath: String(row.file_path),
+            fileSize: Number(row.file_size),
+            category: String(row.category),
+            normalizedHash: row.normalized_hash == null ? undefined : String(row.normalized_hash),
+            createdAt: Number(row.created_at),
+            updatedAt: Number(row.updated_at),
+        };
+    }
+    rowToRelationship(row) {
+        return {
+            id: Number(row.id),
+            sourceFile: String(row.source_file),
+            targetFile: String(row.target_file),
+            relationshipType: String(row.relationship_type),
+            confidence: Number(row.confidence),
+            detail: row.detail == null ? undefined : String(row.detail),
+            userConfirmed: Number(row.user_confirmed),
+            createdAt: Number(row.created_at),
+        };
+    }
+    rowToChunk(row, score) {
+        return {
+            id: String(row.id),
+            relativePath: String(row.relative_path),
+            chunkIndex: Number(row.chunk_index),
+            content: String(row.content),
+            category: String(row.category),
+            format: String(row.format),
+            collectionName: String(row.collection_name),
+            tokenCount: Number(row.token_count),
+            sectionTitle: row.section_title == null ? undefined : String(row.section_title),
+            metadataJson: row.metadata_json == null ? undefined : String(row.metadata_json),
+            createdAt: Number(row.created_at),
+            score,
+        };
+    }
+    expandSearchTerms(query) {
+        const normalized = query.toLowerCase().trim();
+        const terms = new Set(normalized.split(/[\s,，。；;：:、]+/u).filter(Boolean));
+        if (normalized)
+            terms.add(normalized);
+        const synonyms = {
+            招标: ['招标', '投标', '标书', '招标文件', '投标文件', 'bid', 'tender', 'bidding'],
+            投标: ['招标', '投标', '标书', '招标文件', '投标文件', 'bid', 'tender', 'bidding'],
+            标书: ['招标', '投标', '标书', '招标文件', '投标文件', 'bid', 'tender', 'bidding'],
+            合同: ['合同', '协议', 'contract', 'agreement'],
+            pdf: ['pdf', 'document'],
+            文档: ['文档', '文件', 'document', 'pdf', 'office'],
+        };
+        for (const [key, values] of Object.entries(synonyms)) {
+            if (normalized.includes(key))
+                for (const value of values)
+                    terms.add(value.toLowerCase());
+        }
+        return [...terms];
+    }
+    scoreChunk(content, terms) {
+        const lower = content.toLowerCase();
+        let score = 0;
+        for (const term of terms) {
+            let index = lower.indexOf(term);
+            while (index !== -1) {
+                score += 1;
+                index = lower.indexOf(term, index + term.length);
+            }
+        }
+        return score / Math.max(1, content.length / 1000);
+    }
+    rowToRecord(row) {
+        return {
+            relativePath: String(row.relative_path),
+            category: String(row.category),
+            format: String(row.format),
+            contentHash: String(row.content_hash),
+            fileSize: Number(row.file_size),
+            mtime: Number(row.mtime),
+            chunkCount: Number(row.chunk_count),
+            collectionName: String(row.collection_name),
+            indexedAt: Number(row.indexed_at),
+            lastVerifiedAt: Number(row.last_verified_at),
+            status: String(row.status),
+            errorMessage: row.error_message == null ? undefined : String(row.error_message),
+            metadataJson: row.metadata_json == null ? undefined : String(row.metadata_json),
+        };
+    }
+}

package/dist/core/knowledge-base-manager.d.ts

@@ -0,0 +1,76 @@
+import { type EmbeddingProvider } from '../embedding/embedding-provider.js';
+import type { ExternalExtractorRegistry } from '../extraction/external-extractor.js';
+import { type FederatedResult, type SearchFilters } from '../search/federation-search.js';
+import type { DiffResult, IndexStateRecord, KBScope, KnowledgeBaseStats, ProjectConfig } from '../types.js';
+import type { VectorStoreInterface } from '../vector/types.js';
+import { type VectorIndexResult } from '../vector/vector-indexer.js';
+import { IndexStateStore, type ChunkSearchResult, type FileRelationship } from './index-state-store.js';
+export interface KnowledgeBaseManagerOptions {
+    scope: Exclude<KBScope, 'session'>;
+    projectRoot?: string;
+    projectId?: string;
+    kbPath?: string;
+    storageRoot?: string;
+    embeddingProvider?: EmbeddingProvider;
+    vectorStores?: Map<string, VectorStoreInterface>;
+    externalExtractors?: ExternalExtractorRegistry;
+}
+export declare class KnowledgeBaseManager {
+    readonly scope: Exclude<KBScope, 'session'>;
+    readonly projectRoot?: string;
+    readonly projectId?: string;
+    readonly kbPath: string;
+    readonly store: IndexStateStore;
+    private readonly classifier;
+    private readonly scanner;
+    private readonly collections;
+    private readonly extractor;
+    private readonly chunker;
+    private readonly dedup;
+    private readonly relationshipDetector;
+    private readonly embeddingProvider;
+    private readonly vectorStores;
+    private readonly configManager;
+    private projectConfig?;
+    private lastSkippedFiles;
+    constructor(options: KnowledgeBaseManagerOptions);
+    initialize(): void;
+    incrementalIndex(): Promise<DiffResult>;
+    search(query: string, limit?: number): ChunkSearchResult[];
+    semanticSearch(query: string, options?: {
+        limit?: number;
+        filters?: SearchFilters;
+        collections?: string[];
+    }): Promise<FederatedResult>;
+    listRelationships(filePath?: string): FileRelationship[];
+    listFiles(): IndexStateRecord[];
+    addFile(sourcePath: string, targetRelativePath?: string): Promise<DiffResult>;
+    uploadFile(fileName: string, content: Buffer, targetRelativePath?: string): Promise<DiffResult>;
+    listFailedFiles(): DiffResult['skippedFiles'];
+    removeFile(relativePath: string): Promise<void>;
+    tagFile(relativePath: string, tags: string[]): void;
+    listTags(relativePath?: string): Array<{
+        filePath: string;
+        tag: string;
+        createdAt: number;
+    }>;
+    addIgnoreRule(pattern: string): void;
+    listIgnoreRules(): Array<{
+        id: number;
+        pattern: string;
+        enabled: boolean;
+        createdAt: number;
+    }>;
+    indexVectors(options?: {
+        collectionName?: string;
+        relativePath?: string;
+        limit?: number;
+    }): Promise<VectorIndexResult[]>;
+    getProjectConfig(): ProjectConfig | undefined;
+    getStats(): KnowledgeBaseStats;
+    close(): void;
+    private hasUsableContent;
+    private defaultUploadRelativePath;
+    private resolveKbRelativePath;
+    private normalizeRelativePath;
+}