@customize-agent/knowledge 4.0.1 → 4.0.3

package/models/bge-small-zh-v1.5/tokenizer_config.json

@@ -0,0 +1,15 @@
+{
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": false,
+  "mask_token": "[MASK]",
+  "model_max_length": 512,
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@customize-agent/knowledge",
-  "version": "4.0.1",
+  "version": "4.0.3",
   "description": "Local knowledge base infrastructure for customize-agent",
   "type": "module",
   "main": "./dist/index.js",
@@ -12,6 +12,14 @@
       "default": "./dist/index.js"
     }
   },
+  "scripts": {
+    "prebuild": "node -e \"require('node:fs').rmSync('dist',{recursive:true,force:true})\"",
+    "build": "tsc",
+    "typecheck": "tsc --noEmit",
+    "lint": "eslint src/",
+    "postinstall": "node scripts/install-hnsw.cjs",
+    "doctor:hnsw": "node scripts/install-hnsw.cjs"
+  },
   "license": "MIT",
   "engines": {
     "node": ">=22.13.0"
@@ -23,7 +31,9 @@
     "directory": "packages/knowledge"
   },
   "files": [
-    "dist"
+    "dist",
+    "models",
+    "scripts"
   ],
   "publishConfig": {
     "access": "public"
@@ -35,6 +45,7 @@
     "knowledge-base"
   ],
   "dependencies": {
+    "@huggingface/transformers": "^3.8.0",
     "@napi-rs/canvas": "^0.1.82",
     "better-sqlite3": "^12.10.0",
     "dwgdxf": "^2.0.1",
@@ -44,7 +55,8 @@
     "mammoth": "^1.12.0",
     "pdf-parse": "^2.4.5",
     "pdfjs-dist": "^5.4.394",
-    "sqlite-vec": "^0.1.9",
+    "hnswlib-node": "^3.0.0",
+    "node-gyp": "^12.1.0",
     "tesseract.js": "^7.0.0",
     "word-extractor": "^1.0.4",
     "xlsx": "^0.18.5"
@@ -52,10 +64,5 @@
   "devDependencies": {
     "@types/better-sqlite3": "^7.6.13",
     "@types/node": "^25.9.3"
-  },
-  "scripts": {
-    "build": "tsc",
-    "typecheck": "tsc --noEmit",
-    "lint": "eslint src/"
   }
-}
+}

package/scripts/install-hnsw.cjs

@@ -0,0 +1,47 @@
+#!/usr/bin/env node
+const { spawnSync } = require('node:child_process');
+const fs = require('node:fs');
+const os = require('node:os');
+const path = require('node:path');
+
+function packageDir(name) {
+  return path.dirname(require.resolve(`${name}/package.json`, { paths: [process.cwd(), __dirname] }));
+}
+
+function run(command, args, cwd) {
+  const result = spawnSync(command, args, { cwd, stdio: 'inherit', shell: false });
+  if (result.status !== 0) process.exit(result.status || 1);
+}
+
+function verify(hnswDir) {
+  const hnsw = require(path.join(hnswDir, 'lib/index.js'));
+  const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'hnsw-doctor-'));
+  try {
+    const indexPath = path.join(dir, 'index.bin');
+    const index = new hnsw.HierarchicalNSW('cosine', 2);
+    index.initIndex(10, 16, 200, 100, true);
+    index.addPoint([1, 0], 1);
+    index.addPoint([0, 1], 2);
+    const result = index.searchKnn([1, 0], 1);
+    index.writeIndexSync(indexPath);
+    const loaded = new hnsw.HierarchicalNSW('cosine', 2);
+    loaded.readIndexSync(indexPath, true);
+    const loadedResult = loaded.searchKnn([1, 0], 1);
+    if (result.neighbors[0] !== 1 || loadedResult.neighbors[0] !== 1) throw new Error('HNSW 检索验证失败');
+  } finally {
+    fs.rmSync(dir, { recursive: true, force: true });
+  }
+}
+
+try {
+  const hnswDir = packageDir('hnswlib-node');
+  const nodeGypBin = require.resolve('node-gyp/bin/node-gyp.js', { paths: [process.cwd(), __dirname] });
+  console.log(`[hnsw] 构建 hnswlib-node native binding: ${hnswDir}`);
+  run(process.execPath, [nodeGypBin, 'rebuild'], hnswDir);
+  verify(hnswDir);
+  console.log('[hnsw] hnswlib-node 安装和运行验证通过');
+} catch (error) {
+  console.error('[hnsw] hnswlib-node 安装或运行验证失败。请确认当前平台已安装 native 编译工具链。');
+  console.error(error && error.stack ? error.stack : error);
+  process.exit(1);
+}

package/dist/vector/sqlite-vec-store.d.ts

@@ -1,38 +0,0 @@
-import type { CollectionClient, VectorCollectionInfo, VectorDocument, VectorSearchQuery, VectorSearchResult, VectorStoreInterface } from './types.js';
-interface SearchRow {
-    id: string;
-    content: string;
-    metadata_json: string | null;
-    distance: number;
-}
-export interface SQLiteVecClientOptions {
-    dbPath: string;
-}
-export declare class SQLiteVecClient implements CollectionClient {
-    readonly dbPath: string;
-    private readonly db;
-    constructor(options: SQLiteVecClientOptions);
-    getOrCreateCollection(name: string, metadata?: Record<string, unknown>): Promise<VectorCollectionInfo>;
-    listCollections(): Promise<VectorCollectionInfo[]>;
-    deleteCollection(name: string): Promise<void>;
-    upsert(collectionName: string, documents: VectorDocument[]): void;
-    deleteWhere(collectionName: string, where: Record<string, string | number | boolean>): void;
-    deleteByFilePath(collectionName: string, filePath: string): void;
-    search(collectionName: string, query: VectorSearchQuery): SearchRow[];
-    private getCollection;
-    private requireCollection;
-    private tableName;
-    private parseMetadata;
-    private metadataString;
-}
-export declare class SQLiteVecVectorStore implements VectorStoreInterface {
-    private readonly client;
-    readonly collectionName: string;
-    constructor(client: SQLiteVecClient, collectionName: string);
-    ensureCollection(metadata?: Record<string, unknown>): Promise<void>;
-    upsert(documents: VectorDocument[]): Promise<void>;
-    deleteByFilePath(filePath: string): Promise<void>;
-    search(query: VectorSearchQuery): Promise<VectorSearchResult[]>;
-    private parseDocumentMetadata;
-}
-export {};

package/dist/vector/sqlite-vec-store.js

@@ -1,203 +0,0 @@
-import Database from 'better-sqlite3';
-import * as crypto from 'node:crypto';
-import * as fs from 'node:fs';
-import * as path from 'node:path';
-import * as sqliteVec from 'sqlite-vec';
-export class SQLiteVecClient {
-    dbPath;
-    db;
-    constructor(options) {
-        this.dbPath = options.dbPath;
-        fs.mkdirSync(path.dirname(options.dbPath), { recursive: true });
-        this.db = new Database(options.dbPath);
-        this.db.pragma('journal_mode = WAL');
-        sqliteVec.load(this.db);
-        this.db.exec(`
-      CREATE TABLE IF NOT EXISTS vector_collections (
-        name TEXT PRIMARY KEY,
-        table_name TEXT NOT NULL UNIQUE,
-        dimension INTEGER NOT NULL,
-        metadata_json TEXT,
-        created_at INTEGER NOT NULL
-      );
-      CREATE TABLE IF NOT EXISTS vector_documents (
-        rowid INTEGER PRIMARY KEY AUTOINCREMENT,
-        collection_name TEXT NOT NULL,
-        id TEXT NOT NULL,
-        content TEXT NOT NULL,
-        metadata_json TEXT,
-                file_path TEXT,
-                vector_rowid INTEGER,
-                created_at INTEGER NOT NULL,
-        updated_at INTEGER NOT NULL,
-        UNIQUE(collection_name, id)
-      );
-      CREATE INDEX IF NOT EXISTS idx_vector_documents_collection_file ON vector_documents(collection_name, file_path);
-    `);
-        try {
-            this.db.exec('ALTER TABLE vector_documents ADD COLUMN vector_rowid INTEGER');
-        }
-        catch { /* column already exists */ }
-    }
-    async getOrCreateCollection(name, metadata = {}) {
-        const existing = this.getCollection(name);
-        if (existing)
-            return { name: existing.name, metadata: this.parseMetadata(existing.metadata_json) };
-        const dimension = Number(metadata.embedding_dimension ?? 384);
-        const tableName = this.tableName(name);
-        this.db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS ${tableName} USING vec0(embedding float[${dimension}])`);
-        this.db.prepare(`
-      INSERT INTO vector_collections (name, table_name, dimension, metadata_json, created_at)
-      VALUES (?, ?, ?, ?, ?)
-    `).run(name, tableName, dimension, JSON.stringify(metadata), Date.now());
-        return { name, metadata };
-    }
-    async listCollections() {
-        return this.db.prepare('SELECT name, metadata_json FROM vector_collections ORDER BY name').all()
-            .map(row => {
-            const record = row;
-            return { name: record.name, metadata: this.parseMetadata(record.metadata_json) };
-        });
-    }
-    async deleteCollection(name) {
-        const collection = this.getCollection(name);
-        if (!collection)
-            return;
-        this.db.exec(`DROP TABLE IF EXISTS ${collection.table_name}`);
-        this.db.prepare('DELETE FROM vector_documents WHERE collection_name = ?').run(name);
-        this.db.prepare('DELETE FROM vector_collections WHERE name = ?').run(name);
-    }
-    upsert(collectionName, documents) {
-        if (documents.length === 0)
-            return;
-        const collection = this.requireCollection(collectionName);
-        const deleteVec = this.db.prepare(`DELETE FROM ${collection.table_name} WHERE rowid = ?`);
-        const deleteDoc = this.db.prepare('DELETE FROM vector_documents WHERE collection_name = ? AND id = ?');
-        const insertDoc = this.db.prepare(`
-      INSERT INTO vector_documents (collection_name, id, content, metadata_json, file_path, created_at, updated_at)
-      VALUES (?, ?, ?, ?, ?, ?, ?)
-    `);
-        const updateVectorRowid = this.db.prepare('UPDATE vector_documents SET vector_rowid = ? WHERE rowid = ?');
-        const insertVec = this.db.prepare(`INSERT INTO ${collection.table_name} (embedding) VALUES (vec_f32(?))`);
-        const existingRow = this.db.prepare('SELECT rowid, vector_rowid FROM vector_documents WHERE collection_name = ? AND id = ?');
-        const transaction = this.db.transaction((items) => {
-            for (const document of items) {
-                const existing = existingRow.get(collectionName, document.id);
-                if (existing?.vector_rowid)
-                    deleteVec.run(existing.vector_rowid);
-                deleteDoc.run(collectionName, document.id);
-                const now = Date.now();
-                const metadataJson = JSON.stringify(document.metadata);
-                const filePath = this.metadataString(document.metadata.file_path);
-                const documentResult = insertDoc.run(collectionName, document.id, document.content, metadataJson, filePath, now, now);
-                const vectorResult = insertVec.run(JSON.stringify(document.embedding));
-                updateVectorRowid.run(Number(vectorResult.lastInsertRowid), Number(documentResult.lastInsertRowid));
-            }
-        });
-        transaction(documents);
-    }
-    deleteWhere(collectionName, where) {
-        if (typeof where.file_path !== 'string')
-            return;
-        this.deleteByFilePath(collectionName, where.file_path);
-    }
-    deleteByFilePath(collectionName, filePath) {
-        const collection = this.getCollection(collectionName);
-        if (!collection)
-            return;
-        const rows = this.db.prepare('SELECT vector_rowid FROM vector_documents WHERE collection_name = ? AND file_path = ?').all(collectionName, filePath);
-        const deleteVec = this.db.prepare(`DELETE FROM ${collection.table_name} WHERE rowid = ?`);
-        const deleteDocs = this.db.prepare('DELETE FROM vector_documents WHERE collection_name = ? AND file_path = ?');
-        const transaction = this.db.transaction(() => {
-            for (const row of rows) {
-                if (row.vector_rowid)
-                    deleteVec.run(row.vector_rowid);
-            }
-            deleteDocs.run(collectionName, filePath);
-        });
-        transaction();
-    }
-    search(collectionName, query) {
-        const collection = this.getCollection(collectionName);
-        if (!collection)
-            return [];
-        const filePath = query.where?.file_path;
-        if (typeof filePath === 'string') {
-            return this.db.prepare(`
-        SELECT d.id, d.content, d.metadata_json, v.distance
-        FROM (
-          SELECT rowid, distance
-          FROM ${collection.table_name}
-          WHERE embedding MATCH vec_f32(?) AND k = ?
-        ) v
-        JOIN vector_documents d ON d.vector_rowid = v.rowid
-        WHERE d.file_path = ?
-        ORDER BY v.distance
-      `).all(JSON.stringify(query.queryEmbedding), query.topK, filePath);
-        }
-        return this.db.prepare(`
-      SELECT d.id, d.content, d.metadata_json, v.distance
-      FROM (
-        SELECT rowid, distance
-        FROM ${collection.table_name}
-        WHERE embedding MATCH vec_f32(?) AND k = ?
-      ) v
-      JOIN vector_documents d ON d.vector_rowid = v.rowid
-      ORDER BY v.distance
-    `).all(JSON.stringify(query.queryEmbedding), query.topK);
-    }
-    getCollection(name) {
-        return this.db.prepare('SELECT name, table_name, dimension, metadata_json FROM vector_collections WHERE name = ?').get(name);
-    }
-    requireCollection(name) {
-        const collection = this.getCollection(name);
-        if (!collection)
-            throw new Error(`SQLite vec collection not found: ${name}`);
-        return collection;
-    }
-    tableName(collectionName) {
-        const hash = crypto.createHash('sha256').update(collectionName).digest('hex').slice(0, 16);
-        return `vec_${hash}`;
-    }
-    parseMetadata(metadataJson) {
-        if (!metadataJson)
-            return undefined;
-        return JSON.parse(metadataJson);
-    }
-    metadataString(value) {
-        return typeof value === 'string' ? value : null;
-    }
-}
-export class SQLiteVecVectorStore {
-    client;
-    collectionName;
-    constructor(client, collectionName) {
-        this.client = client;
-        this.collectionName = collectionName;
-    }
-    async ensureCollection(metadata) {
-        await this.client.getOrCreateCollection(this.collectionName, metadata);
-    }
-    async upsert(documents) {
-        this.client.upsert(this.collectionName, documents);
-    }
-    async deleteByFilePath(filePath) {
-        this.client.deleteByFilePath(this.collectionName, filePath);
-    }
-    async search(query) {
-        return this.client.search(this.collectionName, query).map(row => ({
-            collection: this.collectionName,
-            score: 1 / (1 + Number(row.distance ?? 0)),
-            document: {
-                id: row.id,
-                content: row.content,
-                metadata: this.parseDocumentMetadata(row.metadata_json),
-            },
-        }));
-    }
-    parseDocumentMetadata(metadataJson) {
-        if (!metadataJson)
-            return {};
-        return JSON.parse(metadataJson);
-    }
-}