@customize-agent/knowledge 2.1.3 → 2.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/knowledge-base-manager.d.ts +1 -1
- package/dist/core/knowledge-base-manager.js +13 -10
- package/dist/extraction/content-extractor.js +5 -0
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/vector/sqlite-vec-store.d.ts +38 -0
- package/dist/vector/sqlite-vec-store.js +203 -0
- package/package.json +10 -8
- package/dist/vector/chroma-store.d.ts +0 -41
- package/dist/vector/chroma-store.js +0 -162
|
@@ -34,7 +34,7 @@ export declare class KnowledgeBaseManager {
|
|
|
34
34
|
readonly projectId?: string;
|
|
35
35
|
readonly kbPath: string;
|
|
36
36
|
readonly store: IndexStateStore;
|
|
37
|
-
private readonly
|
|
37
|
+
private readonly sqliteVecClient;
|
|
38
38
|
private readonly classifier;
|
|
39
39
|
private readonly scanner;
|
|
40
40
|
private readonly collections;
|
|
@@ -10,7 +10,7 @@ import { HashEmbeddingProvider } from '../embedding/embedding-provider.js';
|
|
|
10
10
|
import { ContentExtractor } from '../extraction/content-extractor.js';
|
|
11
11
|
import { FederationSearch } from '../search/federation-search.js';
|
|
12
12
|
import { CollectionManager } from '../vector/collection-manager.js';
|
|
13
|
-
import {
|
|
13
|
+
import { SQLiteVecClient, SQLiteVecVectorStore } from '../vector/sqlite-vec-store.js';
|
|
14
14
|
import { VectorIndexer } from '../vector/vector-indexer.js';
|
|
15
15
|
import { ChangeTracker } from './change-tracker.js';
|
|
16
16
|
import { KnowledgeFileScanner } from './file-scanner.js';
|
|
@@ -22,7 +22,7 @@ export class KnowledgeBaseManager {
|
|
|
22
22
|
projectId;
|
|
23
23
|
kbPath;
|
|
24
24
|
store;
|
|
25
|
-
|
|
25
|
+
sqliteVecClient;
|
|
26
26
|
classifier = new FileClassifier();
|
|
27
27
|
scanner = new KnowledgeFileScanner();
|
|
28
28
|
collections = new CollectionManager();
|
|
@@ -47,17 +47,20 @@ export class KnowledgeBaseManager {
|
|
|
47
47
|
this.llmProvider = options.llmProvider;
|
|
48
48
|
this.onProgress = options.onProgress;
|
|
49
49
|
const storageRoot = options.storageRoot ?? path.join(os.homedir(), USER_DATA_DIR);
|
|
50
|
+
let dbPath;
|
|
50
51
|
if (this.scope === 'global') {
|
|
51
52
|
this.kbPath = options.kbPath ?? path.join(storageRoot, GLOBAL_KNOWLEDGE_DIR);
|
|
52
|
-
|
|
53
|
+
dbPath = path.join(storageRoot, 'global-knowledge.db');
|
|
53
54
|
}
|
|
54
55
|
else {
|
|
55
56
|
if (!options.projectRoot || !options.projectId) {
|
|
56
57
|
throw new Error('project knowledge base requires projectRoot and projectId');
|
|
57
58
|
}
|
|
58
59
|
this.kbPath = options.kbPath ?? getProjectKbPath(options.projectRoot);
|
|
59
|
-
|
|
60
|
+
dbPath = path.join(storageRoot, 'projects', options.projectId, 'kb.db');
|
|
60
61
|
}
|
|
62
|
+
this.store = new IndexStateStore(dbPath);
|
|
63
|
+
this.sqliteVecClient = new SQLiteVecClient({ dbPath });
|
|
61
64
|
}
|
|
62
65
|
initialize() {
|
|
63
66
|
if (this.scope === 'project' && this.projectRoot) {
|
|
@@ -228,7 +231,7 @@ export class KnowledgeBaseManager {
|
|
|
228
231
|
message: vectorDeferred
|
|
229
232
|
? '解析、切片和 SQLite 入库已完成,向量入库后台执行'
|
|
230
233
|
: vectorStatus.status === 'error'
|
|
231
|
-
? '解析和切片已完成,
|
|
234
|
+
? '解析和切片已完成,sqlite-vec 向量待入库'
|
|
232
235
|
: '知识库索引完成',
|
|
233
236
|
chunkCount: stats.chunkCount,
|
|
234
237
|
vectorStatus,
|
|
@@ -380,7 +383,7 @@ export class KnowledgeBaseManager {
|
|
|
380
383
|
const targetPath = this.resolveKbRelativePath(normalized);
|
|
381
384
|
if (fs.existsSync(targetPath))
|
|
382
385
|
fs.unlinkSync(targetPath);
|
|
383
|
-
// 同步删除
|
|
386
|
+
// 同步删除 sqlite-vec 向量数据,避免孤儿向量污染搜索结果
|
|
384
387
|
const record = this.store.listRecords().find(r => r.relativePath === normalized);
|
|
385
388
|
if (record) {
|
|
386
389
|
await this.deleteVectorFile(record.collectionName, normalized);
|
|
@@ -409,7 +412,7 @@ export class KnowledgeBaseManager {
|
|
|
409
412
|
const chunks = this.store.listChunks(options);
|
|
410
413
|
for (const collectionName of new Set(chunks.map(chunk => chunk.collectionName)))
|
|
411
414
|
this.ensureVectorStore(collectionName);
|
|
412
|
-
this.reportProgress({ stage: 'vectorizing', percent: 85, message: `正在写入
|
|
415
|
+
this.reportProgress({ stage: 'vectorizing', percent: 85, message: `正在写入 sqlite-vec 向量库,共 ${chunks.length} 个切片`, chunkCount: chunks.length });
|
|
413
416
|
const indexer = new VectorIndexer(this.embeddingProvider, this.vectorStores);
|
|
414
417
|
try {
|
|
415
418
|
const results = await indexer.indexChunks(chunks);
|
|
@@ -426,7 +429,7 @@ export class KnowledgeBaseManager {
|
|
|
426
429
|
this.store.setMetadata('vector_index_status', 'error');
|
|
427
430
|
this.store.setMetadata('vector_index_error', message);
|
|
428
431
|
this.store.setMetadata('last_vector_index_at', String(Date.now()));
|
|
429
|
-
this.reportProgress({ stage: 'error', percent: 85, message: '
|
|
432
|
+
this.reportProgress({ stage: 'error', percent: 85, message: 'sqlite-vec 向量入库失败', chunkCount: chunks.length, vectorStatus: this.getVectorStatus() });
|
|
430
433
|
return [];
|
|
431
434
|
}
|
|
432
435
|
}
|
|
@@ -450,7 +453,7 @@ export class KnowledgeBaseManager {
|
|
|
450
453
|
error: this.store.getMetadata('vector_index_error') || undefined,
|
|
451
454
|
indexedChunks: Number(this.store.getMetadata('vector_indexed_chunks') ?? 0),
|
|
452
455
|
lastIndexedAt: Number(this.store.getMetadata('last_vector_index_at') ?? 0),
|
|
453
|
-
backend: `
|
|
456
|
+
backend: `SQLite + sqlite-vec (${this.sqliteVecClient.dbPath})`,
|
|
454
457
|
};
|
|
455
458
|
}
|
|
456
459
|
async rewriteQueries(query) {
|
|
@@ -673,7 +676,7 @@ ${resultsText}
|
|
|
673
676
|
ensureVectorStore(collectionName) {
|
|
674
677
|
if (this.vectorStores.has(collectionName))
|
|
675
678
|
return;
|
|
676
|
-
this.vectorStores.set(collectionName, new
|
|
679
|
+
this.vectorStores.set(collectionName, new SQLiteVecVectorStore(this.sqliteVecClient, collectionName));
|
|
677
680
|
}
|
|
678
681
|
async deleteVectorFile(collectionName, relativePath) {
|
|
679
682
|
this.ensureVectorStore(collectionName);
|
|
@@ -492,6 +492,11 @@ export class ContentExtractor {
|
|
|
492
492
|
}
|
|
493
493
|
async extractRasterImage(file) {
|
|
494
494
|
const metadata = { extractionMode: 'builtin_tesseract_ocr_isolated', vectorizable: true };
|
|
495
|
+
if (process.env.CUSTOMIZE_AGENT_DISABLE_OCR === '1') {
|
|
496
|
+
metadata.extractionMode = 'raster_image_metadata';
|
|
497
|
+
metadata.contentCoverage = 'metadata_filename';
|
|
498
|
+
return { text: this.metadataOnlyText(file), metadata, warnings: ['OCR disabled; indexed image metadata only'] };
|
|
499
|
+
}
|
|
495
500
|
const validationError = this.validateRasterImage(file.absolutePath);
|
|
496
501
|
if (validationError) {
|
|
497
502
|
metadata.contentCoverage = 'invalid_image';
|
package/dist/index.d.ts
CHANGED
|
@@ -13,7 +13,7 @@ export { MultiProjectManager } from './core/multi-project-manager.js';
|
|
|
13
13
|
export { computeProjectId } from './core/project-id.js';
|
|
14
14
|
export { ensureProjectCustomizeFile, getProjectConfigPath, getProjectKbPath, ProjectConfigManager } from './core/project-config.js';
|
|
15
15
|
export { ProjectRegistry } from './core/project-registry.js';
|
|
16
|
-
export {
|
|
16
|
+
export { SQLiteVecClient, SQLiteVecVectorStore, type SQLiteVecClientOptions } from './vector/sqlite-vec-store.js';
|
|
17
17
|
export { CollectionManager, globalCollectionName, projectCollectionName } from './vector/collection-manager.js';
|
|
18
18
|
export type { CollectionClient, VectorCollectionInfo, VectorDocument, VectorSearchQuery, VectorSearchResult, VectorStoreInterface } from './vector/types.js';
|
|
19
19
|
export { VectorIndexer, type VectorIndexResult } from './vector/vector-indexer.js';
|
package/dist/index.js
CHANGED
|
@@ -15,7 +15,7 @@ export { MultiProjectManager } from './core/multi-project-manager.js';
|
|
|
15
15
|
export { computeProjectId } from './core/project-id.js';
|
|
16
16
|
export { ensureProjectCustomizeFile, getProjectConfigPath, getProjectKbPath, ProjectConfigManager } from './core/project-config.js';
|
|
17
17
|
export { ProjectRegistry } from './core/project-registry.js';
|
|
18
|
-
export {
|
|
18
|
+
export { SQLiteVecClient, SQLiteVecVectorStore } from './vector/sqlite-vec-store.js';
|
|
19
19
|
export { CollectionManager, globalCollectionName, projectCollectionName } from './vector/collection-manager.js';
|
|
20
20
|
export { VectorIndexer } from './vector/vector-indexer.js';
|
|
21
21
|
export { FederationSearch } from './search/federation-search.js';
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import type { CollectionClient, VectorCollectionInfo, VectorDocument, VectorSearchQuery, VectorSearchResult, VectorStoreInterface } from './types.js';
|
|
2
|
+
interface SearchRow {
|
|
3
|
+
id: string;
|
|
4
|
+
content: string;
|
|
5
|
+
metadata_json: string | null;
|
|
6
|
+
distance: number;
|
|
7
|
+
}
|
|
8
|
+
export interface SQLiteVecClientOptions {
|
|
9
|
+
dbPath: string;
|
|
10
|
+
}
|
|
11
|
+
export declare class SQLiteVecClient implements CollectionClient {
|
|
12
|
+
readonly dbPath: string;
|
|
13
|
+
private readonly db;
|
|
14
|
+
constructor(options: SQLiteVecClientOptions);
|
|
15
|
+
getOrCreateCollection(name: string, metadata?: Record<string, unknown>): Promise<VectorCollectionInfo>;
|
|
16
|
+
listCollections(): Promise<VectorCollectionInfo[]>;
|
|
17
|
+
deleteCollection(name: string): Promise<void>;
|
|
18
|
+
upsert(collectionName: string, documents: VectorDocument[]): void;
|
|
19
|
+
deleteWhere(collectionName: string, where: Record<string, string | number | boolean>): void;
|
|
20
|
+
deleteByFilePath(collectionName: string, filePath: string): void;
|
|
21
|
+
search(collectionName: string, query: VectorSearchQuery): SearchRow[];
|
|
22
|
+
private getCollection;
|
|
23
|
+
private requireCollection;
|
|
24
|
+
private tableName;
|
|
25
|
+
private parseMetadata;
|
|
26
|
+
private metadataString;
|
|
27
|
+
}
|
|
28
|
+
export declare class SQLiteVecVectorStore implements VectorStoreInterface {
|
|
29
|
+
private readonly client;
|
|
30
|
+
readonly collectionName: string;
|
|
31
|
+
constructor(client: SQLiteVecClient, collectionName: string);
|
|
32
|
+
ensureCollection(metadata?: Record<string, unknown>): Promise<void>;
|
|
33
|
+
upsert(documents: VectorDocument[]): Promise<void>;
|
|
34
|
+
deleteByFilePath(filePath: string): Promise<void>;
|
|
35
|
+
search(query: VectorSearchQuery): Promise<VectorSearchResult[]>;
|
|
36
|
+
private parseDocumentMetadata;
|
|
37
|
+
}
|
|
38
|
+
export {};
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
import Database from 'better-sqlite3';
|
|
2
|
+
import * as crypto from 'node:crypto';
|
|
3
|
+
import * as fs from 'node:fs';
|
|
4
|
+
import * as path from 'node:path';
|
|
5
|
+
import * as sqliteVec from 'sqlite-vec';
|
|
6
|
+
export class SQLiteVecClient {
|
|
7
|
+
dbPath;
|
|
8
|
+
db;
|
|
9
|
+
constructor(options) {
|
|
10
|
+
this.dbPath = options.dbPath;
|
|
11
|
+
fs.mkdirSync(path.dirname(options.dbPath), { recursive: true });
|
|
12
|
+
this.db = new Database(options.dbPath);
|
|
13
|
+
this.db.pragma('journal_mode = WAL');
|
|
14
|
+
sqliteVec.load(this.db);
|
|
15
|
+
this.db.exec(`
|
|
16
|
+
CREATE TABLE IF NOT EXISTS vector_collections (
|
|
17
|
+
name TEXT PRIMARY KEY,
|
|
18
|
+
table_name TEXT NOT NULL UNIQUE,
|
|
19
|
+
dimension INTEGER NOT NULL,
|
|
20
|
+
metadata_json TEXT,
|
|
21
|
+
created_at INTEGER NOT NULL
|
|
22
|
+
);
|
|
23
|
+
CREATE TABLE IF NOT EXISTS vector_documents (
|
|
24
|
+
rowid INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
25
|
+
collection_name TEXT NOT NULL,
|
|
26
|
+
id TEXT NOT NULL,
|
|
27
|
+
content TEXT NOT NULL,
|
|
28
|
+
metadata_json TEXT,
|
|
29
|
+
file_path TEXT,
|
|
30
|
+
vector_rowid INTEGER,
|
|
31
|
+
created_at INTEGER NOT NULL,
|
|
32
|
+
updated_at INTEGER NOT NULL,
|
|
33
|
+
UNIQUE(collection_name, id)
|
|
34
|
+
);
|
|
35
|
+
CREATE INDEX IF NOT EXISTS idx_vector_documents_collection_file ON vector_documents(collection_name, file_path);
|
|
36
|
+
`);
|
|
37
|
+
try {
|
|
38
|
+
this.db.exec('ALTER TABLE vector_documents ADD COLUMN vector_rowid INTEGER');
|
|
39
|
+
}
|
|
40
|
+
catch { /* column already exists */ }
|
|
41
|
+
}
|
|
42
|
+
async getOrCreateCollection(name, metadata = {}) {
|
|
43
|
+
const existing = this.getCollection(name);
|
|
44
|
+
if (existing)
|
|
45
|
+
return { name: existing.name, metadata: this.parseMetadata(existing.metadata_json) };
|
|
46
|
+
const dimension = Number(metadata.embedding_dimension ?? 384);
|
|
47
|
+
const tableName = this.tableName(name);
|
|
48
|
+
this.db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS ${tableName} USING vec0(embedding float[${dimension}])`);
|
|
49
|
+
this.db.prepare(`
|
|
50
|
+
INSERT INTO vector_collections (name, table_name, dimension, metadata_json, created_at)
|
|
51
|
+
VALUES (?, ?, ?, ?, ?)
|
|
52
|
+
`).run(name, tableName, dimension, JSON.stringify(metadata), Date.now());
|
|
53
|
+
return { name, metadata };
|
|
54
|
+
}
|
|
55
|
+
async listCollections() {
|
|
56
|
+
return this.db.prepare('SELECT name, metadata_json FROM vector_collections ORDER BY name').all()
|
|
57
|
+
.map(row => {
|
|
58
|
+
const record = row;
|
|
59
|
+
return { name: record.name, metadata: this.parseMetadata(record.metadata_json) };
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
async deleteCollection(name) {
|
|
63
|
+
const collection = this.getCollection(name);
|
|
64
|
+
if (!collection)
|
|
65
|
+
return;
|
|
66
|
+
this.db.exec(`DROP TABLE IF EXISTS ${collection.table_name}`);
|
|
67
|
+
this.db.prepare('DELETE FROM vector_documents WHERE collection_name = ?').run(name);
|
|
68
|
+
this.db.prepare('DELETE FROM vector_collections WHERE name = ?').run(name);
|
|
69
|
+
}
|
|
70
|
+
upsert(collectionName, documents) {
|
|
71
|
+
if (documents.length === 0)
|
|
72
|
+
return;
|
|
73
|
+
const collection = this.requireCollection(collectionName);
|
|
74
|
+
const deleteVec = this.db.prepare(`DELETE FROM ${collection.table_name} WHERE rowid = ?`);
|
|
75
|
+
const deleteDoc = this.db.prepare('DELETE FROM vector_documents WHERE collection_name = ? AND id = ?');
|
|
76
|
+
const insertDoc = this.db.prepare(`
|
|
77
|
+
INSERT INTO vector_documents (collection_name, id, content, metadata_json, file_path, created_at, updated_at)
|
|
78
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
79
|
+
`);
|
|
80
|
+
const updateVectorRowid = this.db.prepare('UPDATE vector_documents SET vector_rowid = ? WHERE rowid = ?');
|
|
81
|
+
const insertVec = this.db.prepare(`INSERT INTO ${collection.table_name} (embedding) VALUES (vec_f32(?))`);
|
|
82
|
+
const existingRow = this.db.prepare('SELECT rowid, vector_rowid FROM vector_documents WHERE collection_name = ? AND id = ?');
|
|
83
|
+
const transaction = this.db.transaction((items) => {
|
|
84
|
+
for (const document of items) {
|
|
85
|
+
const existing = existingRow.get(collectionName, document.id);
|
|
86
|
+
if (existing?.vector_rowid)
|
|
87
|
+
deleteVec.run(existing.vector_rowid);
|
|
88
|
+
deleteDoc.run(collectionName, document.id);
|
|
89
|
+
const now = Date.now();
|
|
90
|
+
const metadataJson = JSON.stringify(document.metadata);
|
|
91
|
+
const filePath = this.metadataString(document.metadata.file_path);
|
|
92
|
+
const documentResult = insertDoc.run(collectionName, document.id, document.content, metadataJson, filePath, now, now);
|
|
93
|
+
const vectorResult = insertVec.run(JSON.stringify(document.embedding));
|
|
94
|
+
updateVectorRowid.run(Number(vectorResult.lastInsertRowid), Number(documentResult.lastInsertRowid));
|
|
95
|
+
}
|
|
96
|
+
});
|
|
97
|
+
transaction(documents);
|
|
98
|
+
}
|
|
99
|
+
deleteWhere(collectionName, where) {
|
|
100
|
+
if (typeof where.file_path !== 'string')
|
|
101
|
+
return;
|
|
102
|
+
this.deleteByFilePath(collectionName, where.file_path);
|
|
103
|
+
}
|
|
104
|
+
deleteByFilePath(collectionName, filePath) {
|
|
105
|
+
const collection = this.getCollection(collectionName);
|
|
106
|
+
if (!collection)
|
|
107
|
+
return;
|
|
108
|
+
const rows = this.db.prepare('SELECT vector_rowid FROM vector_documents WHERE collection_name = ? AND file_path = ?').all(collectionName, filePath);
|
|
109
|
+
const deleteVec = this.db.prepare(`DELETE FROM ${collection.table_name} WHERE rowid = ?`);
|
|
110
|
+
const deleteDocs = this.db.prepare('DELETE FROM vector_documents WHERE collection_name = ? AND file_path = ?');
|
|
111
|
+
const transaction = this.db.transaction(() => {
|
|
112
|
+
for (const row of rows) {
|
|
113
|
+
if (row.vector_rowid)
|
|
114
|
+
deleteVec.run(row.vector_rowid);
|
|
115
|
+
}
|
|
116
|
+
deleteDocs.run(collectionName, filePath);
|
|
117
|
+
});
|
|
118
|
+
transaction();
|
|
119
|
+
}
|
|
120
|
+
search(collectionName, query) {
|
|
121
|
+
const collection = this.getCollection(collectionName);
|
|
122
|
+
if (!collection)
|
|
123
|
+
return [];
|
|
124
|
+
const filePath = query.where?.file_path;
|
|
125
|
+
if (typeof filePath === 'string') {
|
|
126
|
+
return this.db.prepare(`
|
|
127
|
+
SELECT d.id, d.content, d.metadata_json, v.distance
|
|
128
|
+
FROM (
|
|
129
|
+
SELECT rowid, distance
|
|
130
|
+
FROM ${collection.table_name}
|
|
131
|
+
WHERE embedding MATCH vec_f32(?) AND k = ?
|
|
132
|
+
) v
|
|
133
|
+
JOIN vector_documents d ON d.vector_rowid = v.rowid
|
|
134
|
+
WHERE d.file_path = ?
|
|
135
|
+
ORDER BY v.distance
|
|
136
|
+
`).all(JSON.stringify(query.queryEmbedding), query.topK, filePath);
|
|
137
|
+
}
|
|
138
|
+
return this.db.prepare(`
|
|
139
|
+
SELECT d.id, d.content, d.metadata_json, v.distance
|
|
140
|
+
FROM (
|
|
141
|
+
SELECT rowid, distance
|
|
142
|
+
FROM ${collection.table_name}
|
|
143
|
+
WHERE embedding MATCH vec_f32(?) AND k = ?
|
|
144
|
+
) v
|
|
145
|
+
JOIN vector_documents d ON d.vector_rowid = v.rowid
|
|
146
|
+
ORDER BY v.distance
|
|
147
|
+
`).all(JSON.stringify(query.queryEmbedding), query.topK);
|
|
148
|
+
}
|
|
149
|
+
getCollection(name) {
|
|
150
|
+
return this.db.prepare('SELECT name, table_name, dimension, metadata_json FROM vector_collections WHERE name = ?').get(name);
|
|
151
|
+
}
|
|
152
|
+
requireCollection(name) {
|
|
153
|
+
const collection = this.getCollection(name);
|
|
154
|
+
if (!collection)
|
|
155
|
+
throw new Error(`SQLite vec collection not found: ${name}`);
|
|
156
|
+
return collection;
|
|
157
|
+
}
|
|
158
|
+
tableName(collectionName) {
|
|
159
|
+
const hash = crypto.createHash('sha256').update(collectionName).digest('hex').slice(0, 16);
|
|
160
|
+
return `vec_${hash}`;
|
|
161
|
+
}
|
|
162
|
+
parseMetadata(metadataJson) {
|
|
163
|
+
if (!metadataJson)
|
|
164
|
+
return undefined;
|
|
165
|
+
return JSON.parse(metadataJson);
|
|
166
|
+
}
|
|
167
|
+
metadataString(value) {
|
|
168
|
+
return typeof value === 'string' ? value : null;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
export class SQLiteVecVectorStore {
|
|
172
|
+
client;
|
|
173
|
+
collectionName;
|
|
174
|
+
constructor(client, collectionName) {
|
|
175
|
+
this.client = client;
|
|
176
|
+
this.collectionName = collectionName;
|
|
177
|
+
}
|
|
178
|
+
async ensureCollection(metadata) {
|
|
179
|
+
await this.client.getOrCreateCollection(this.collectionName, metadata);
|
|
180
|
+
}
|
|
181
|
+
async upsert(documents) {
|
|
182
|
+
this.client.upsert(this.collectionName, documents);
|
|
183
|
+
}
|
|
184
|
+
async deleteByFilePath(filePath) {
|
|
185
|
+
this.client.deleteByFilePath(this.collectionName, filePath);
|
|
186
|
+
}
|
|
187
|
+
async search(query) {
|
|
188
|
+
return this.client.search(this.collectionName, query).map(row => ({
|
|
189
|
+
collection: this.collectionName,
|
|
190
|
+
score: 1 / (1 + Number(row.distance ?? 0)),
|
|
191
|
+
document: {
|
|
192
|
+
id: row.id,
|
|
193
|
+
content: row.content,
|
|
194
|
+
metadata: this.parseDocumentMetadata(row.metadata_json),
|
|
195
|
+
},
|
|
196
|
+
}));
|
|
197
|
+
}
|
|
198
|
+
parseDocumentMetadata(metadataJson) {
|
|
199
|
+
if (!metadataJson)
|
|
200
|
+
return {};
|
|
201
|
+
return JSON.parse(metadataJson);
|
|
202
|
+
}
|
|
203
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@customize-agent/knowledge",
|
|
3
|
-
"version": "2.1
|
|
3
|
+
"version": "2.2.1",
|
|
4
4
|
"description": "Local knowledge base infrastructure for customize-agent",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -8,9 +8,15 @@
|
|
|
8
8
|
"exports": {
|
|
9
9
|
".": {
|
|
10
10
|
"types": "./dist/index.d.ts",
|
|
11
|
-
"import": "./dist/index.js"
|
|
11
|
+
"import": "./dist/index.js",
|
|
12
|
+
"default": "./dist/index.js"
|
|
12
13
|
}
|
|
13
14
|
},
|
|
15
|
+
"scripts": {
|
|
16
|
+
"build": "tsc",
|
|
17
|
+
"typecheck": "tsc --noEmit",
|
|
18
|
+
"lint": "eslint src/"
|
|
19
|
+
},
|
|
14
20
|
"license": "MIT",
|
|
15
21
|
"author": "Pan-jijian",
|
|
16
22
|
"repository": {
|
|
@@ -32,6 +38,7 @@
|
|
|
32
38
|
],
|
|
33
39
|
"dependencies": {
|
|
34
40
|
"better-sqlite3": "^12.10.0",
|
|
41
|
+
"sqlite-vec": "^0.1.9",
|
|
35
42
|
"fast-glob": "^3.3.3",
|
|
36
43
|
"jszip": "^3.10.1",
|
|
37
44
|
"@napi-rs/canvas": "^0.1.82",
|
|
@@ -44,10 +51,5 @@
|
|
|
44
51
|
"devDependencies": {
|
|
45
52
|
"@types/better-sqlite3": "^7.6.13",
|
|
46
53
|
"@types/node": "^25.9.3"
|
|
47
|
-
},
|
|
48
|
-
"scripts": {
|
|
49
|
-
"build": "tsc",
|
|
50
|
-
"typecheck": "tsc --noEmit",
|
|
51
|
-
"lint": "eslint src/"
|
|
52
54
|
}
|
|
53
|
-
}
|
|
55
|
+
}
|
|
@@ -1,41 +0,0 @@
|
|
|
1
|
-
import type { CollectionClient, VectorCollectionInfo, VectorDocument, VectorSearchQuery, VectorSearchResult, VectorStoreInterface } from './types.js';
|
|
2
|
-
interface ChromaQueryResponse {
|
|
3
|
-
ids?: string[][];
|
|
4
|
-
documents?: string[][];
|
|
5
|
-
metadatas?: Array<Array<Record<string, unknown>>>;
|
|
6
|
-
distances?: number[][];
|
|
7
|
-
}
|
|
8
|
-
export interface ChromaClientOptions {
|
|
9
|
-
baseUrl?: string;
|
|
10
|
-
tenant?: string;
|
|
11
|
-
database?: string;
|
|
12
|
-
}
|
|
13
|
-
export declare class ChromaHttpClient implements CollectionClient {
|
|
14
|
-
readonly baseUrl: string;
|
|
15
|
-
readonly tenant: string;
|
|
16
|
-
readonly database: string;
|
|
17
|
-
private readonly collectionIds;
|
|
18
|
-
constructor(options?: ChromaClientOptions);
|
|
19
|
-
heartbeat(): Promise<boolean>;
|
|
20
|
-
getOrCreateCollection(name: string, metadata?: Record<string, unknown>): Promise<VectorCollectionInfo>;
|
|
21
|
-
listCollections(): Promise<VectorCollectionInfo[]>;
|
|
22
|
-
deleteCollection(name: string): Promise<void>;
|
|
23
|
-
upsert(collectionName: string, documents: VectorDocument[]): Promise<void>;
|
|
24
|
-
deleteWhere(collectionName: string, where: Record<string, string | number | boolean>): Promise<void>;
|
|
25
|
-
query(collectionName: string, query: VectorSearchQuery): Promise<ChromaQueryResponse>;
|
|
26
|
-
private getCollectionId;
|
|
27
|
-
private collectionsPath;
|
|
28
|
-
private request;
|
|
29
|
-
private toCollectionInfo;
|
|
30
|
-
}
|
|
31
|
-
export declare class ChromaVectorStore implements VectorStoreInterface {
|
|
32
|
-
private readonly client;
|
|
33
|
-
readonly collectionName: string;
|
|
34
|
-
constructor(client: ChromaHttpClient, collectionName: string);
|
|
35
|
-
ensureCollection(metadata?: Record<string, unknown>): Promise<void>;
|
|
36
|
-
upsert(documents: VectorDocument[]): Promise<void>;
|
|
37
|
-
deleteByFilePath(filePath: string): Promise<void>;
|
|
38
|
-
search(query: VectorSearchQuery): Promise<VectorSearchResult[]>;
|
|
39
|
-
private toMetadata;
|
|
40
|
-
}
|
|
41
|
-
export {};
|
|
@@ -1,162 +0,0 @@
|
|
|
1
|
-
export class ChromaHttpClient {
|
|
2
|
-
baseUrl;
|
|
3
|
-
tenant;
|
|
4
|
-
database;
|
|
5
|
-
collectionIds = new Map();
|
|
6
|
-
constructor(options = {}) {
|
|
7
|
-
this.baseUrl = options.baseUrl ?? process.env.CHROMA_URL ?? process.env.CHROMA_BASE_URL ?? 'http://localhost:17322';
|
|
8
|
-
this.tenant = options.tenant ?? 'default_tenant';
|
|
9
|
-
this.database = options.database ?? 'default_database';
|
|
10
|
-
}
|
|
11
|
-
async heartbeat() {
|
|
12
|
-
try {
|
|
13
|
-
await this.request('/api/v2/heartbeat');
|
|
14
|
-
return true;
|
|
15
|
-
}
|
|
16
|
-
catch {
|
|
17
|
-
return false;
|
|
18
|
-
}
|
|
19
|
-
}
|
|
20
|
-
async getOrCreateCollection(name, metadata = {}) {
|
|
21
|
-
const body = { name, get_or_create: true };
|
|
22
|
-
if (Object.keys(metadata).length > 0)
|
|
23
|
-
body.metadata = metadata;
|
|
24
|
-
const response = await this.request(this.collectionsPath(), {
|
|
25
|
-
method: 'POST',
|
|
26
|
-
body: JSON.stringify(body),
|
|
27
|
-
}, 10000);
|
|
28
|
-
if (response.id)
|
|
29
|
-
this.collectionIds.set(name, response.id);
|
|
30
|
-
return this.toCollectionInfo(response);
|
|
31
|
-
}
|
|
32
|
-
async listCollections() {
|
|
33
|
-
const response = await this.request(this.collectionsPath(), {}, 10000);
|
|
34
|
-
for (const collection of response)
|
|
35
|
-
if (collection.id)
|
|
36
|
-
this.collectionIds.set(collection.name, collection.id);
|
|
37
|
-
return response.map(collection => this.toCollectionInfo(collection));
|
|
38
|
-
}
|
|
39
|
-
async deleteCollection(name) {
|
|
40
|
-
await this.request(`${this.collectionsPath()}/${encodeURIComponent(name)}`, { method: 'DELETE' });
|
|
41
|
-
this.collectionIds.delete(name);
|
|
42
|
-
}
|
|
43
|
-
async upsert(collectionName, documents) {
|
|
44
|
-
if (documents.length === 0)
|
|
45
|
-
return;
|
|
46
|
-
const collectionId = await this.getCollectionId(collectionName);
|
|
47
|
-
await this.request(`${this.collectionsPath()}/${encodeURIComponent(collectionId)}/upsert`, {
|
|
48
|
-
method: 'POST',
|
|
49
|
-
body: JSON.stringify({
|
|
50
|
-
ids: documents.map(document => document.id),
|
|
51
|
-
embeddings: documents.map(document => document.embedding),
|
|
52
|
-
documents: documents.map(document => document.content),
|
|
53
|
-
metadatas: documents.map(document => document.metadata),
|
|
54
|
-
}),
|
|
55
|
-
}, 30000);
|
|
56
|
-
}
|
|
57
|
-
async deleteWhere(collectionName, where) {
|
|
58
|
-
const collectionId = await this.getCollectionId(collectionName);
|
|
59
|
-
await this.request(`${this.collectionsPath()}/${encodeURIComponent(collectionId)}/delete`, {
|
|
60
|
-
method: 'POST',
|
|
61
|
-
body: JSON.stringify({ where }),
|
|
62
|
-
}, 10000);
|
|
63
|
-
}
|
|
64
|
-
async query(collectionName, query) {
|
|
65
|
-
const collectionId = await this.getCollectionId(collectionName);
|
|
66
|
-
return this.request(`${this.collectionsPath()}/${encodeURIComponent(collectionId)}/query`, {
|
|
67
|
-
method: 'POST',
|
|
68
|
-
body: JSON.stringify({
|
|
69
|
-
query_embeddings: [query.queryEmbedding],
|
|
70
|
-
n_results: query.topK,
|
|
71
|
-
where: query.where,
|
|
72
|
-
include: ['documents', 'metadatas', 'distances'],
|
|
73
|
-
}),
|
|
74
|
-
}, 10000);
|
|
75
|
-
}
|
|
76
|
-
async getCollectionId(name) {
|
|
77
|
-
const cached = this.collectionIds.get(name);
|
|
78
|
-
if (cached)
|
|
79
|
-
return cached;
|
|
80
|
-
const collection = await this.getOrCreateCollection(name);
|
|
81
|
-
if (!collection.id)
|
|
82
|
-
throw new Error(`ChromaDB collection has no id: ${name}`);
|
|
83
|
-
this.collectionIds.set(name, collection.id);
|
|
84
|
-
return collection.id;
|
|
85
|
-
}
|
|
86
|
-
collectionsPath() {
|
|
87
|
-
return `/api/v2/tenants/${encodeURIComponent(this.tenant)}/databases/${encodeURIComponent(this.database)}/collections`;
|
|
88
|
-
}
|
|
89
|
-
async request(path, init = {}, timeoutMs = 3000) {
|
|
90
|
-
const controller = new AbortController();
|
|
91
|
-
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
|
92
|
-
let response;
|
|
93
|
-
try {
|
|
94
|
-
response = await fetch(`${this.baseUrl}${path}`, {
|
|
95
|
-
...init,
|
|
96
|
-
signal: controller.signal,
|
|
97
|
-
headers: {
|
|
98
|
-
'content-type': 'application/json',
|
|
99
|
-
...(init.headers ?? {}),
|
|
100
|
-
},
|
|
101
|
-
});
|
|
102
|
-
}
|
|
103
|
-
finally {
|
|
104
|
-
clearTimeout(timeout);
|
|
105
|
-
}
|
|
106
|
-
if (!response.ok) {
|
|
107
|
-
throw new Error(`ChromaDB request failed: ${response.status} ${response.statusText}`);
|
|
108
|
-
}
|
|
109
|
-
if (response.status === 204)
|
|
110
|
-
return undefined;
|
|
111
|
-
return await response.json();
|
|
112
|
-
}
|
|
113
|
-
toCollectionInfo(collection) {
|
|
114
|
-
return {
|
|
115
|
-
id: collection.id,
|
|
116
|
-
name: collection.name,
|
|
117
|
-
metadata: collection.metadata,
|
|
118
|
-
};
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
export class ChromaVectorStore {
|
|
122
|
-
client;
|
|
123
|
-
collectionName;
|
|
124
|
-
constructor(client, collectionName) {
|
|
125
|
-
this.client = client;
|
|
126
|
-
this.collectionName = collectionName;
|
|
127
|
-
}
|
|
128
|
-
async ensureCollection(metadata) {
|
|
129
|
-
await this.client.getOrCreateCollection(this.collectionName, metadata);
|
|
130
|
-
}
|
|
131
|
-
async upsert(documents) {
|
|
132
|
-
await this.client.upsert(this.collectionName, documents);
|
|
133
|
-
}
|
|
134
|
-
async deleteByFilePath(filePath) {
|
|
135
|
-
await this.client.deleteWhere(this.collectionName, { file_path: filePath });
|
|
136
|
-
}
|
|
137
|
-
async search(query) {
|
|
138
|
-
const response = await this.client.query(this.collectionName, query);
|
|
139
|
-
const ids = response.ids?.[0] ?? [];
|
|
140
|
-
const documents = response.documents?.[0] ?? [];
|
|
141
|
-
const metadatas = response.metadatas?.[0] ?? [];
|
|
142
|
-
const distances = response.distances?.[0] ?? [];
|
|
143
|
-
return ids.map((id, index) => ({
|
|
144
|
-
collection: this.collectionName,
|
|
145
|
-
score: 1 - Number(distances[index] ?? 1),
|
|
146
|
-
document: {
|
|
147
|
-
id,
|
|
148
|
-
content: documents[index] ?? '',
|
|
149
|
-
metadata: this.toMetadata(metadatas[index] ?? {}),
|
|
150
|
-
},
|
|
151
|
-
}));
|
|
152
|
-
}
|
|
153
|
-
toMetadata(metadata) {
|
|
154
|
-
const result = {};
|
|
155
|
-
for (const [key, value] of Object.entries(metadata)) {
|
|
156
|
-
if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean' || value === null) {
|
|
157
|
-
result[key] = value;
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
return result;
|
|
161
|
-
}
|
|
162
|
-
}
|