@customize-agent/knowledge 4.0.1 → 4.0.2

package/dist/index.d.ts

@@ -2,7 +2,7 @@ export { TextChunker, type ChunkConfig, type TextChunk } from './chunking/text-c
 export { FileClassifier } from './classification/classifier.js';
 export { DedupEngine, type MinHashSignature, type SimilarityMatch } from './dedup/dedup-engine.js';
 export { RelationshipDetector } from './dedup/relationship-detector.js';
-export { HashEmbeddingProvider, OpenAICompatibleEmbeddingProvider, createEmbeddingProviderFromEnvironment, type EmbeddingProvider, type OpenAICompatibleEmbeddingOptions } from './embedding/embedding-provider.js';
+export { HashEmbeddingProvider, LocalTransformersEmbeddingProvider, OpenAICompatibleEmbeddingProvider, createEmbeddingProviderFromEnvironment, type EmbeddingProvider, type LocalTransformersEmbeddingOptions, type OpenAICompatibleEmbeddingOptions } from './embedding/embedding-provider.js';
 export { ContentExtractor, type ExtractionResult } from './extraction/content-extractor.js';
 export { CommandExternalExtractor, ExternalExtractorRegistry, type CommandExternalExtractorOptions, type ExternalExtractionResult, type ExternalExtractor, type ExternalExtractorCapability } from './extraction/external-extractor.js';
 export { ChangeTracker } from './core/change-tracker.js';
@@ -13,7 +13,7 @@ export { MultiProjectManager } from './core/multi-project-manager.js';
 export { computeProjectId } from './core/project-id.js';
 export { ensureProjectCustomizeFile, getProjectConfigPath, getProjectKbPath, ProjectConfigManager } from './core/project-config.js';
 export { ProjectRegistry } from './core/project-registry.js';
-export { SQLiteVecClient, SQLiteVecVectorStore, type SQLiteVecClientOptions } from './vector/sqlite-vec-store.js';
+export { HNSWVectorStore } from './vector/hnsw-vector-store.js';
 export { CollectionManager, globalCollectionName, projectCollectionName } from './vector/collection-manager.js';
 export type { CollectionClient, VectorCollectionInfo, VectorDocument, VectorSearchQuery, VectorSearchResult, VectorStoreInterface } from './vector/types.js';
 export { VectorIndexer, type VectorIndexResult } from './vector/vector-indexer.js';

package/dist/index.js

@@ -4,7 +4,7 @@ export { TextChunker } from './chunking/text-chunker.js';
 export { FileClassifier } from './classification/classifier.js';
 export { DedupEngine } from './dedup/dedup-engine.js';
 export { RelationshipDetector } from './dedup/relationship-detector.js';
-export { HashEmbeddingProvider, OpenAICompatibleEmbeddingProvider, createEmbeddingProviderFromEnvironment } from './embedding/embedding-provider.js';
+export { HashEmbeddingProvider, LocalTransformersEmbeddingProvider, OpenAICompatibleEmbeddingProvider, createEmbeddingProviderFromEnvironment } from './embedding/embedding-provider.js';
 export { ContentExtractor } from './extraction/content-extractor.js';
 export { CommandExternalExtractor, ExternalExtractorRegistry } from './extraction/external-extractor.js';
 export { ChangeTracker } from './core/change-tracker.js';
@@ -15,7 +15,7 @@ export { MultiProjectManager } from './core/multi-project-manager.js';
 export { computeProjectId } from './core/project-id.js';
 export { ensureProjectCustomizeFile, getProjectConfigPath, getProjectKbPath, ProjectConfigManager } from './core/project-config.js';
 export { ProjectRegistry } from './core/project-registry.js';
-export { SQLiteVecClient, SQLiteVecVectorStore } from './vector/sqlite-vec-store.js';
+export { HNSWVectorStore } from './vector/hnsw-vector-store.js';
 export { CollectionManager, globalCollectionName, projectCollectionName } from './vector/collection-manager.js';
 export { VectorIndexer } from './vector/vector-indexer.js';
 export { FederationSearch } from './search/federation-search.js';

package/dist/search/federation-search.d.ts

@@ -2,6 +2,7 @@ import type { VectorStoreInterface } from '../vector/types.js';
 export type SearchScope = 'project' | 'global' | 'all';
 export interface FederatedSearchItem {
     id: string;
+    rowid?: number;
     content: string;
     filePath: string;
     scope: 'project' | 'global';

package/dist/search/federation-search.js

@@ -73,6 +73,7 @@ export class FederationSearch {
     toFederatedItem(result, scope) {
         return {
             id: result.document.id,
+            rowid: typeof result.document.metadata.sqlite_rowid === 'number' ? result.document.metadata.sqlite_rowid : undefined,
             content: result.document.content,
             filePath: String(result.document.metadata.file_path ?? ''),
             scope,

package/dist/vector/hnsw-vector-store.d.ts

@@ -0,0 +1,20 @@
+import type { VectorDocument, VectorSearchQuery, VectorSearchResult, VectorStoreInterface } from './types.js';
+export declare class HNSWVectorStore implements VectorStoreInterface {
+    readonly collectionName: string;
+    private readonly indexPath;
+    private readonly dimensions;
+    private readonly maxElements;
+    private index?;
+    private deletedSinceRebuild;
+    private readonly documents;
+    constructor(collectionName: string, indexPath: string, dimensions?: number, maxElements?: number);
+    ensureCollection(): Promise<void>;
+    upsert(documents: VectorDocument[]): Promise<void>;
+    clearCollection(): Promise<void>;
+    deleteByFilePath(filePath: string): Promise<void>;
+    needsRebuild(): boolean;
+    search(query: VectorSearchQuery): Promise<VectorSearchResult[]>;
+    private persist;
+    private loadDocuments;
+    private metadataPath;
+}

package/dist/vector/hnsw-vector-store.js

@@ -0,0 +1,107 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import { createRequire } from 'node:module';
+const require = createRequire(import.meta.url);
+export class HNSWVectorStore {
+    collectionName;
+    indexPath;
+    dimensions;
+    maxElements;
+    index;
+    deletedSinceRebuild = 0;
+    documents = new Map();
+    constructor(collectionName, indexPath, dimensions = 512, maxElements = 500_000) {
+        this.collectionName = collectionName;
+        this.indexPath = indexPath;
+        this.dimensions = dimensions;
+        this.maxElements = maxElements;
+    }
+    async ensureCollection() {
+        if (this.index)
+            return;
+        fs.mkdirSync(path.dirname(this.indexPath), { recursive: true });
+        this.loadDocuments();
+        const mod = require('hnswlib-node');
+        this.index = new mod.HierarchicalNSW('cosine', this.dimensions);
+        if (fs.existsSync(this.indexPath))
+            this.index.readIndexSync(this.indexPath, true);
+        else
+            this.index.initIndex(this.maxElements, 16, 200, 100, true);
+    }
+    async upsert(documents) {
+        await this.ensureCollection();
+        for (const document of documents) {
+            const rowid = Number(document.metadata.sqlite_rowid);
+            if (!Number.isFinite(rowid) || rowid <= 0)
+                throw new Error(`HNSW 向量写入缺少有效 sqlite_rowid: ${document.id}`);
+            this.index.addPoint(document.embedding, rowid, true);
+            this.documents.set(rowid, document);
+        }
+        this.persist();
+    }
+    async clearCollection() {
+        if (fs.existsSync(this.indexPath))
+            fs.rmSync(this.indexPath, { force: true });
+        if (fs.existsSync(this.metadataPath()))
+            fs.rmSync(this.metadataPath(), { force: true });
+        this.documents.clear();
+        this.deletedSinceRebuild = 0;
+        this.index = undefined;
+        await this.ensureCollection();
+    }
+    async deleteByFilePath(filePath) {
+        await this.ensureCollection();
+        for (const [rowid, document] of this.documents.entries()) {
+            if (document.metadata.file_path === filePath) {
+                try {
+                    this.index.markDelete(rowid);
+                    this.deletedSinceRebuild += 1;
+                }
+                catch { /* ignore missing labels */ }
+                this.documents.delete(rowid);
+            }
+        }
+        this.persist();
+    }
+    needsRebuild() {
+        const total = this.documents.size + this.deletedSinceRebuild;
+        return total >= 1000 && this.deletedSinceRebuild / total > 0.25;
+    }
+    async search(query) {
+        await this.ensureCollection();
+        const result = this.index.searchKnn(query.queryEmbedding, query.topK);
+        return result.neighbors.flatMap((rowid, index) => {
+            const document = this.documents.get(rowid);
+            if (!document)
+                return [];
+            if (typeof query.where?.file_path === 'string' && document.metadata.file_path !== query.where.file_path)
+                return [];
+            const { embedding: _embedding, ...stored } = document;
+            const distance = result.distances[index] ?? 0;
+            return [{ collection: this.collectionName, document: stored, score: 1 / (1 + distance) }];
+        });
+    }
+    persist() {
+        this.index.writeIndexSync(this.indexPath);
+        fs.writeFileSync(this.metadataPath(), JSON.stringify({ deletedSinceRebuild: this.deletedSinceRebuild, documents: [...this.documents.entries()] }), 'utf8');
+    }
+    loadDocuments() {
+        const file = this.metadataPath();
+        if (!fs.existsSync(file))
+            return;
+        try {
+            const parsed = JSON.parse(fs.readFileSync(file, 'utf8'));
+            const entries = Array.isArray(parsed) ? parsed : parsed.documents ?? [];
+            this.deletedSinceRebuild = Array.isArray(parsed) ? 0 : Number(parsed.deletedSinceRebuild ?? 0);
+            this.documents.clear();
+            for (const [rowid, document] of entries)
+                this.documents.set(Number(rowid), document);
+        }
+        catch {
+            this.documents.clear();
+        }
+    }
+    metadataPath() {
+        return `${this.indexPath}.documents.json`;
+    }
+}

package/dist/vector/types.d.ts

@@ -24,6 +24,8 @@ export interface VectorStoreInterface {
     ensureCollection(metadata?: Record<string, unknown>): Promise<void>;
     upsert(documents: VectorDocument[]): Promise<void>;
     deleteByFilePath(filePath: string): Promise<void>;
+    clearCollection?(): Promise<void>;
+    needsRebuild?(): boolean;
     search(query: VectorSearchQuery): Promise<VectorSearchResult[]>;
 }
 export interface CollectionClient {

package/dist/vector/vector-indexer.d.ts

@@ -13,6 +13,8 @@ export declare class VectorIndexer {
     constructor(embeddingProvider: EmbeddingProvider, vectorStores: Map<string, VectorStoreInterface>);
     indexChunks(chunks: StoredChunk[]): Promise<VectorIndexResult[]>;
     deleteFile(collectionName: string, filePath: string): Promise<void>;
+    private embedDocuments;
+    private isValidEmbeddings;
     private groupByCollection;
     private toVectorDocument;
     private parseMetadata;

package/dist/vector/vector-indexer.js

@@ -12,11 +12,12 @@ export class VectorIndexer {
             const store = this.vectorStores.get(collectionName);
             if (!store)
                 continue;
+            const texts = collectionChunks.map(chunk => chunk.content);
+            const embeddings = await this.embedDocuments(texts);
             await store.ensureCollection({
                 embedding_model: this.embeddingProvider.model,
                 embedding_dimension: this.embeddingProvider.dimensions,
             });
-            const embeddings = await this.embeddingProvider.embedDocuments(collectionChunks.map(chunk => chunk.content));
             const documents = collectionChunks.map((chunk, index) => this.toVectorDocument(chunk, embeddings[index] ?? []));
             await store.upsert(documents);
             results.push({
@@ -34,6 +35,16 @@ export class VectorIndexer {
             return;
         await store.deleteByFilePath(filePath);
     }
+    async embedDocuments(texts) {
+        const embeddings = await this.embeddingProvider.embedDocuments(texts);
+        if (!this.isValidEmbeddings(embeddings, texts.length, this.embeddingProvider.dimensions)) {
+            throw new Error(`Embedding 结果异常：期望 ${texts.length} 条 ${this.embeddingProvider.dimensions} 维向量，实际返回 ${embeddings.length} 条`);
+        }
+        return embeddings;
+    }
+    isValidEmbeddings(embeddings, count, dimensions) {
+        return embeddings.length === count && embeddings.every(vector => vector.length === dimensions && vector.every(value => Number.isFinite(value)));
+    }
     groupByCollection(chunks) {
         const grouped = new Map();
         for (const chunk of chunks) {
@@ -50,6 +61,7 @@ export class VectorIndexer {
             content: chunk.content,
             embedding,
             metadata: {
+                sqlite_rowid: chunk.rowid,
                 file_path: chunk.relativePath,
                 chunk_index: chunk.chunkIndex,
                 category: chunk.category,

package/models/bge-small-zh-v1.5/config.json

@@ -0,0 +1,31 @@
+{
+  "_name_or_path": "BAAI/bge-small-zh-v1.5",
+  "architectures": [
+    "BertModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "classifier_dropout": null,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 512,
+  "id2label": {
+    "0": "LABEL_0"
+  },
+  "initializer_range": 0.02,
+  "intermediate_size": 2048,
+  "label2id": {
+    "LABEL_0": 0
+  },
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 8,
+  "num_hidden_layers": 4,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "transformers_version": "4.34.0.dev0",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 21128
+}

package/models/bge-small-zh-v1.5/special_tokens_map.json

@@ -0,0 +1,7 @@
+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}