@matperez/coderag 0.1.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +154 -0
- package/dist/.tsbuildinfo +1 -0
- package/dist/ast-chunking.d.ts +40 -0
- package/dist/ast-chunking.d.ts.map +1 -0
- package/dist/ast-chunking.js +88 -0
- package/dist/ast-chunking.js.map +1 -0
- package/dist/ast-chunking.test.d.ts +5 -0
- package/dist/ast-chunking.test.d.ts.map +1 -0
- package/dist/ast-chunking.test.js +173 -0
- package/dist/ast-chunking.test.js.map +1 -0
- package/dist/code-tokenizer.d.ts +62 -0
- package/dist/code-tokenizer.d.ts.map +1 -0
- package/dist/code-tokenizer.js +129 -0
- package/dist/code-tokenizer.js.map +1 -0
- package/dist/code-tokenizer.test.d.ts +5 -0
- package/dist/code-tokenizer.test.d.ts.map +1 -0
- package/dist/code-tokenizer.test.js +96 -0
- package/dist/code-tokenizer.test.js.map +1 -0
- package/dist/db/client-pg.d.ts +16 -0
- package/dist/db/client-pg.d.ts.map +1 -0
- package/dist/db/client-pg.js +38 -0
- package/dist/db/client-pg.js.map +1 -0
- package/dist/db/client.d.ts +36 -0
- package/dist/db/client.d.ts.map +1 -0
- package/dist/db/client.js +81 -0
- package/dist/db/client.js.map +1 -0
- package/dist/db/migrations-pg.d.ts +6 -0
- package/dist/db/migrations-pg.d.ts.map +1 -0
- package/dist/db/migrations-pg.js +88 -0
- package/dist/db/migrations-pg.js.map +1 -0
- package/dist/db/migrations.d.ts +9 -0
- package/dist/db/migrations.d.ts.map +1 -0
- package/dist/db/migrations.js +164 -0
- package/dist/db/migrations.js.map +1 -0
- package/dist/db/schema-pg.d.ts +611 -0
- package/dist/db/schema-pg.d.ts.map +1 -0
- package/dist/db/schema-pg.js +66 -0
- package/dist/db/schema-pg.js.map +1 -0
- package/dist/db/schema.d.ts +630 -0
- package/dist/db/schema.d.ts.map +1 -0
- package/dist/db/schema.js +85 -0
- package/dist/db/schema.js.map +1 -0
- package/dist/embeddings.d.ts +92 -0
- package/dist/embeddings.d.ts.map +1 -0
- package/dist/embeddings.js +275 -0
- package/dist/embeddings.js.map +1 -0
- package/dist/embeddings.test.d.ts +5 -0
- package/dist/embeddings.test.d.ts.map +1 -0
- package/dist/embeddings.test.js +255 -0
- package/dist/embeddings.test.js.map +1 -0
- package/dist/hybrid-search.d.ts +47 -0
- package/dist/hybrid-search.d.ts.map +1 -0
- package/dist/hybrid-search.js +215 -0
- package/dist/hybrid-search.js.map +1 -0
- package/dist/hybrid-search.test.d.ts +5 -0
- package/dist/hybrid-search.test.d.ts.map +1 -0
- package/dist/hybrid-search.test.js +252 -0
- package/dist/hybrid-search.test.js.map +1 -0
- package/dist/incremental-tfidf.d.ts +77 -0
- package/dist/incremental-tfidf.d.ts.map +1 -0
- package/dist/incremental-tfidf.js +248 -0
- package/dist/incremental-tfidf.js.map +1 -0
- package/dist/incremental-tfidf.test.d.ts +5 -0
- package/dist/incremental-tfidf.test.d.ts.map +1 -0
- package/dist/incremental-tfidf.test.js +276 -0
- package/dist/incremental-tfidf.test.js.map +1 -0
- package/dist/index.d.ts +18 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +19 -0
- package/dist/index.js.map +1 -0
- package/dist/indexer.d.ts +205 -0
- package/dist/indexer.d.ts.map +1 -0
- package/dist/indexer.js +1331 -0
- package/dist/indexer.js.map +1 -0
- package/dist/indexer.test.d.ts +12 -0
- package/dist/indexer.test.d.ts.map +1 -0
- package/dist/indexer.test.js +471 -0
- package/dist/indexer.test.js.map +1 -0
- package/dist/language-config.d.ts +54 -0
- package/dist/language-config.d.ts.map +1 -0
- package/dist/language-config.js +75 -0
- package/dist/language-config.js.map +1 -0
- package/dist/search-cache.d.ts +63 -0
- package/dist/search-cache.d.ts.map +1 -0
- package/dist/search-cache.js +118 -0
- package/dist/search-cache.js.map +1 -0
- package/dist/search-cache.test.d.ts +5 -0
- package/dist/search-cache.test.d.ts.map +1 -0
- package/dist/search-cache.test.js +194 -0
- package/dist/search-cache.test.js.map +1 -0
- package/dist/storage-factory.d.ts +11 -0
- package/dist/storage-factory.d.ts.map +1 -0
- package/dist/storage-factory.js +17 -0
- package/dist/storage-factory.js.map +1 -0
- package/dist/storage-persistent-pg.d.ts +75 -0
- package/dist/storage-persistent-pg.d.ts.map +1 -0
- package/dist/storage-persistent-pg.js +579 -0
- package/dist/storage-persistent-pg.js.map +1 -0
- package/dist/storage-persistent-pg.test.d.ts +7 -0
- package/dist/storage-persistent-pg.test.d.ts.map +1 -0
- package/dist/storage-persistent-pg.test.js +90 -0
- package/dist/storage-persistent-pg.test.js.map +1 -0
- package/dist/storage-persistent-types.d.ts +110 -0
- package/dist/storage-persistent-types.d.ts.map +1 -0
- package/dist/storage-persistent-types.js +5 -0
- package/dist/storage-persistent-types.js.map +1 -0
- package/dist/storage-persistent.d.ts +231 -0
- package/dist/storage-persistent.d.ts.map +1 -0
- package/dist/storage-persistent.js +897 -0
- package/dist/storage-persistent.js.map +1 -0
- package/dist/storage-persistent.test.d.ts +5 -0
- package/dist/storage-persistent.test.d.ts.map +1 -0
- package/dist/storage-persistent.test.js +325 -0
- package/dist/storage-persistent.test.js.map +1 -0
- package/dist/storage.d.ts +63 -0
- package/dist/storage.d.ts.map +1 -0
- package/dist/storage.js +67 -0
- package/dist/storage.js.map +1 -0
- package/dist/storage.test.d.ts +5 -0
- package/dist/storage.test.d.ts.map +1 -0
- package/dist/storage.test.js +157 -0
- package/dist/storage.test.js.map +1 -0
- package/dist/tfidf.d.ts +97 -0
- package/dist/tfidf.d.ts.map +1 -0
- package/dist/tfidf.js +308 -0
- package/dist/tfidf.js.map +1 -0
- package/dist/tfidf.test.d.ts +5 -0
- package/dist/tfidf.test.d.ts.map +1 -0
- package/dist/tfidf.test.js +181 -0
- package/dist/tfidf.test.js.map +1 -0
- package/dist/utils.d.ts +61 -0
- package/dist/utils.d.ts.map +1 -0
- package/dist/utils.js +264 -0
- package/dist/utils.js.map +1 -0
- package/dist/utils.test.d.ts +5 -0
- package/dist/utils.test.d.ts.map +1 -0
- package/dist/utils.test.js +94 -0
- package/dist/utils.test.js.map +1 -0
- package/dist/vector-storage.d.ts +120 -0
- package/dist/vector-storage.d.ts.map +1 -0
- package/dist/vector-storage.js +264 -0
- package/dist/vector-storage.js.map +1 -0
- package/dist/vector-storage.test.d.ts +5 -0
- package/dist/vector-storage.test.d.ts.map +1 -0
- package/dist/vector-storage.test.js +345 -0
- package/dist/vector-storage.test.js.map +1 -0
- package/package.json +85 -0
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vector Storage using LanceDB
|
|
3
|
+
* High-performance embedded vector database
|
|
4
|
+
*/
|
|
5
|
+
/**
|
|
6
|
+
* Vector Document
|
|
7
|
+
*/
|
|
8
|
+
export interface VectorDocument {
|
|
9
|
+
readonly id: string;
|
|
10
|
+
readonly embedding: readonly number[];
|
|
11
|
+
readonly metadata: {
|
|
12
|
+
readonly type: 'code' | 'knowledge';
|
|
13
|
+
readonly language?: string;
|
|
14
|
+
readonly content?: string;
|
|
15
|
+
readonly category?: string;
|
|
16
|
+
readonly path?: string;
|
|
17
|
+
readonly [key: string]: unknown;
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
/**
|
|
21
|
+
* Vector Search Result
|
|
22
|
+
*/
|
|
23
|
+
export interface VectorSearchResult {
|
|
24
|
+
readonly doc: VectorDocument;
|
|
25
|
+
readonly similarity: number;
|
|
26
|
+
readonly distance: number;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Vector Storage Options
|
|
30
|
+
*/
|
|
31
|
+
export interface VectorStorageOptions {
|
|
32
|
+
readonly dimensions: number;
|
|
33
|
+
readonly dbPath?: string;
|
|
34
|
+
readonly tableName?: string;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Vector Storage Statistics
|
|
38
|
+
*/
|
|
39
|
+
export interface VectorStorageStats {
|
|
40
|
+
readonly totalDocuments: number;
|
|
41
|
+
readonly dimensions: number;
|
|
42
|
+
readonly indexSize: number;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* Vector Storage with LanceDB
|
|
46
|
+
*/
|
|
47
|
+
export declare class VectorStorage {
|
|
48
|
+
private db;
|
|
49
|
+
private table;
|
|
50
|
+
private dimensions;
|
|
51
|
+
private dbPath;
|
|
52
|
+
private tableName;
|
|
53
|
+
private initialized;
|
|
54
|
+
constructor(options: VectorStorageOptions);
|
|
55
|
+
/**
|
|
56
|
+
* Initialize the database connection
|
|
57
|
+
*/
|
|
58
|
+
private ensureInitialized;
|
|
59
|
+
/**
|
|
60
|
+
* Create table if it doesn't exist
|
|
61
|
+
*/
|
|
62
|
+
private ensureTable;
|
|
63
|
+
/**
|
|
64
|
+
* Convert VectorDocument to LanceDB record
|
|
65
|
+
*/
|
|
66
|
+
private docToRecord;
|
|
67
|
+
/**
|
|
68
|
+
* Convert LanceDB record to VectorDocument
|
|
69
|
+
*/
|
|
70
|
+
private recordToDoc;
|
|
71
|
+
/**
|
|
72
|
+
* Add document to vector storage
|
|
73
|
+
*/
|
|
74
|
+
addDocument(doc: VectorDocument): Promise<void>;
|
|
75
|
+
/**
|
|
76
|
+
* Add multiple documents in batch
|
|
77
|
+
*/
|
|
78
|
+
addDocuments(docs: readonly VectorDocument[]): Promise<void>;
|
|
79
|
+
/**
|
|
80
|
+
* Update document (delete + add)
|
|
81
|
+
*/
|
|
82
|
+
updateDocument(doc: VectorDocument): Promise<void>;
|
|
83
|
+
/**
|
|
84
|
+
* Delete document
|
|
85
|
+
*/
|
|
86
|
+
deleteDocument(docId: string): Promise<boolean>;
|
|
87
|
+
/**
|
|
88
|
+
* Search for similar vectors
|
|
89
|
+
*/
|
|
90
|
+
search(queryVector: readonly number[], options?: {
|
|
91
|
+
readonly k?: number;
|
|
92
|
+
readonly minScore?: number;
|
|
93
|
+
readonly filter?: (doc: VectorDocument) => boolean;
|
|
94
|
+
}): Promise<VectorSearchResult[]>;
|
|
95
|
+
/**
|
|
96
|
+
* Get document by ID
|
|
97
|
+
*/
|
|
98
|
+
getDocument(docId: string): Promise<VectorDocument | undefined>;
|
|
99
|
+
/**
|
|
100
|
+
* Get all documents
|
|
101
|
+
*/
|
|
102
|
+
getAllDocuments(): Promise<readonly VectorDocument[]>;
|
|
103
|
+
/**
|
|
104
|
+
* Check if document exists
|
|
105
|
+
*/
|
|
106
|
+
hasDocument(docId: string): Promise<boolean>;
|
|
107
|
+
/**
|
|
108
|
+
* Get statistics
|
|
109
|
+
*/
|
|
110
|
+
getStats(): Promise<VectorStorageStats>;
|
|
111
|
+
/**
|
|
112
|
+
* Clear all data
|
|
113
|
+
*/
|
|
114
|
+
clear(): Promise<void>;
|
|
115
|
+
/**
|
|
116
|
+
* Close database connection and release resources
|
|
117
|
+
*/
|
|
118
|
+
close(): Promise<void>;
|
|
119
|
+
}
|
|
120
|
+
//# sourceMappingURL=vector-storage.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vector-storage.d.ts","sourceRoot":"","sources":["../src/vector-storage.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH;;GAEG;AACH,MAAM,WAAW,cAAc;IAC9B,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;IACnB,QAAQ,CAAC,SAAS,EAAE,SAAS,MAAM,EAAE,CAAA;IACrC,QAAQ,CAAC,QAAQ,EAAE;QAClB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,CAAA;QACnC,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;QAC1B,QAAQ,CAAC,OAAO,CAAC,EAAE,MAAM,CAAA;QACzB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;QAC1B,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAA;QACtB,QAAQ,EAAE,GAAG,EAAE,MAAM,GAAG,OAAO,CAAA;KAC/B,CAAA;CACD;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC,QAAQ,CAAC,GAAG,EAAE,cAAc,CAAA;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAA;CACzB;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACpC,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;CAC3B;AAED;;GAEG;AACH,MAAM,WAAW,kBAAkB;IAClC,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAA;IAC/B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;CAC1B;AAeD;;GAEG;AACH,qBAAa,aAAa;IACzB,OAAO,CAAC,EAAE,CAAkC;IAC5C,OAAO,CAAC,KAAK,CAA6B;IAC1C,OAAO,CAAC,UAAU,CAAQ;IAC1B,OAAO,CAAC,MAAM,CAAQ;IACtB,OAAO,CAAC,SAAS,CAAQ;IACzB,OAAO,CAAC,WAAW,CAAiB;gBAExB,OAAO,EAAE,oBAAoB;IAMzC;;OAEG;YACW,iBAAiB;IAc/B;;OAEG;YACW,WAAW;IAwBzB;;OAEG;IACH,OAAO,CAAC,WAAW;IAcnB;;OAEG;IACH,OAAO,CAAC,WAAW;IAgBnB;;OAEG;IACG,WAAW,CAAC,GAAG,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBrD;;OAEG;IACG,YAAY,CAAC,IAAI,EAAE,SAAS,cAAc,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAgBlE;;OAEG;IACG,cAAc,CAAC,GAAG,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAKxD;;OAEG;IACG,cAAc,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAYrD;;OAEG;IACG,MAAM,CACX,WAAW,EAAE,SAAS,MAAM,EAAE,EAC9B,OAAO,GAAE;QACR,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,CAAA;QACnB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;QAC1B,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,EAAE,cAAc,KAAK,OAAO,CAAA;KAC7C,GACJ,OAAO,CAAC,kBAAkB,EAAE,CAAC;IA4ChC;;OAEG;IACG,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC;IAgBrE;;OAEG;IACG,eAAe,IAAI,OAAO,CAAC,SAAS,cAAc,EAAE,CAAC;IAS3D;;OAEG;IACG,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,OAAO,CAAC;IAclD;;OAEG;IACG,QAAQ,IAAI,OAAO,CAAC,kBAAkB,CAAC;IAoB7C;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAS5B;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAa5B"}
|
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Vector Storage using LanceDB
|
|
3
|
+
* High-performance embedded vector database
|
|
4
|
+
*/
|
|
5
|
+
import * as lancedb from '@lancedb/lancedb';
|
|
6
|
+
/**
|
|
7
|
+
* Vector Storage with LanceDB
|
|
8
|
+
*/
|
|
9
|
+
export class VectorStorage {
|
|
10
|
+
db = null;
|
|
11
|
+
table = null;
|
|
12
|
+
dimensions;
|
|
13
|
+
dbPath;
|
|
14
|
+
tableName;
|
|
15
|
+
initialized = false;
|
|
16
|
+
constructor(options) {
|
|
17
|
+
this.dimensions = options.dimensions;
|
|
18
|
+
this.dbPath = options.dbPath || ':memory:';
|
|
19
|
+
this.tableName = options.tableName || 'vectors';
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Initialize the database connection
|
|
23
|
+
*/
|
|
24
|
+
async ensureInitialized() {
|
|
25
|
+
if (this.initialized)
|
|
26
|
+
return;
|
|
27
|
+
this.db = await lancedb.connect(this.dbPath);
|
|
28
|
+
// Check if table exists
|
|
29
|
+
const tables = await this.db.tableNames();
|
|
30
|
+
if (tables.includes(this.tableName)) {
|
|
31
|
+
this.table = await this.db.openTable(this.tableName);
|
|
32
|
+
}
|
|
33
|
+
this.initialized = true;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Create table if it doesn't exist
|
|
37
|
+
*/
|
|
38
|
+
async ensureTable(sampleVector) {
|
|
39
|
+
await this.ensureInitialized();
|
|
40
|
+
if (this.table)
|
|
41
|
+
return;
|
|
42
|
+
// Create table with first record as schema
|
|
43
|
+
// Use empty strings instead of null to help LanceDB infer types
|
|
44
|
+
const initialRecord = {
|
|
45
|
+
id: '__schema__',
|
|
46
|
+
vector: sampleVector,
|
|
47
|
+
type: 'code',
|
|
48
|
+
language: '',
|
|
49
|
+
content: '',
|
|
50
|
+
category: '',
|
|
51
|
+
path: '',
|
|
52
|
+
metadata_json: '{}',
|
|
53
|
+
};
|
|
54
|
+
this.table = await this.db?.createTable(this.tableName, [initialRecord]);
|
|
55
|
+
// Delete the schema record
|
|
56
|
+
await this.table.delete('id = "__schema__"');
|
|
57
|
+
}
|
|
58
|
+
/**
|
|
59
|
+
* Convert VectorDocument to LanceDB record
|
|
60
|
+
*/
|
|
61
|
+
docToRecord(doc) {
|
|
62
|
+
const { type, language, content, category, path, ...rest } = doc.metadata;
|
|
63
|
+
return {
|
|
64
|
+
id: doc.id,
|
|
65
|
+
vector: doc.embedding,
|
|
66
|
+
type,
|
|
67
|
+
language: language || '',
|
|
68
|
+
content: content || '',
|
|
69
|
+
category: category || '',
|
|
70
|
+
path: path || '',
|
|
71
|
+
metadata_json: JSON.stringify(rest),
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
/**
|
|
75
|
+
* Convert LanceDB record to VectorDocument
|
|
76
|
+
*/
|
|
77
|
+
recordToDoc(record) {
|
|
78
|
+
const extraMetadata = JSON.parse(record.metadata_json || '{}');
|
|
79
|
+
return {
|
|
80
|
+
id: record.id,
|
|
81
|
+
embedding: record.vector,
|
|
82
|
+
metadata: {
|
|
83
|
+
type: record.type,
|
|
84
|
+
...(record.language && { language: record.language }),
|
|
85
|
+
...(record.content && { content: record.content }),
|
|
86
|
+
...(record.category && { category: record.category }),
|
|
87
|
+
...(record.path && { path: record.path }),
|
|
88
|
+
...extraMetadata,
|
|
89
|
+
},
|
|
90
|
+
};
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Add document to vector storage
|
|
94
|
+
*/
|
|
95
|
+
async addDocument(doc) {
|
|
96
|
+
if (doc.embedding.length !== this.dimensions) {
|
|
97
|
+
throw new Error(`Embedding dimensions (${doc.embedding.length}) don't match index dimensions (${this.dimensions})`);
|
|
98
|
+
}
|
|
99
|
+
await this.ensureTable(doc.embedding);
|
|
100
|
+
// Check if exists
|
|
101
|
+
if (await this.hasDocument(doc.id)) {
|
|
102
|
+
throw new Error(`Document with id ${doc.id} already exists`);
|
|
103
|
+
}
|
|
104
|
+
const record = this.docToRecord(doc);
|
|
105
|
+
await this.table?.add([record]);
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Add multiple documents in batch
|
|
109
|
+
*/
|
|
110
|
+
async addDocuments(docs) {
|
|
111
|
+
if (docs.length === 0)
|
|
112
|
+
return;
|
|
113
|
+
const firstDoc = docs[0];
|
|
114
|
+
if (firstDoc.embedding.length !== this.dimensions) {
|
|
115
|
+
throw new Error(`Embedding dimensions (${firstDoc.embedding.length}) don't match index dimensions (${this.dimensions})`);
|
|
116
|
+
}
|
|
117
|
+
await this.ensureTable(firstDoc.embedding);
|
|
118
|
+
const records = docs.map((doc) => this.docToRecord(doc));
|
|
119
|
+
await this.table?.add(records);
|
|
120
|
+
}
|
|
121
|
+
/**
|
|
122
|
+
* Update document (delete + add)
|
|
123
|
+
*/
|
|
124
|
+
async updateDocument(doc) {
|
|
125
|
+
await this.deleteDocument(doc.id);
|
|
126
|
+
await this.addDocument(doc);
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Delete document
|
|
130
|
+
*/
|
|
131
|
+
async deleteDocument(docId) {
|
|
132
|
+
await this.ensureInitialized();
|
|
133
|
+
if (!this.table)
|
|
134
|
+
return false;
|
|
135
|
+
const exists = await this.hasDocument(docId);
|
|
136
|
+
if (!exists)
|
|
137
|
+
return false;
|
|
138
|
+
await this.table.delete(`id = "${docId.replace(/"/g, '\\"')}"`);
|
|
139
|
+
return true;
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* Search for similar vectors
|
|
143
|
+
*/
|
|
144
|
+
async search(queryVector, options = {}) {
|
|
145
|
+
const k = options.k || 10;
|
|
146
|
+
const minScore = options.minScore || 0;
|
|
147
|
+
if (queryVector.length !== this.dimensions) {
|
|
148
|
+
throw new Error(`Query vector dimensions (${queryVector.length}) don't match index dimensions (${this.dimensions})`);
|
|
149
|
+
}
|
|
150
|
+
await this.ensureInitialized();
|
|
151
|
+
if (!this.table)
|
|
152
|
+
return [];
|
|
153
|
+
// Search with extra results for filtering
|
|
154
|
+
const searchK = options.filter ? k * 3 : k;
|
|
155
|
+
const searchResults = await this.table
|
|
156
|
+
.search(queryVector)
|
|
157
|
+
.limit(searchK)
|
|
158
|
+
.toArray();
|
|
159
|
+
const results = [];
|
|
160
|
+
for (const result of searchResults) {
|
|
161
|
+
const distance = result._distance;
|
|
162
|
+
// LanceDB uses L2 distance by default, convert to similarity
|
|
163
|
+
// For cosine distance: similarity = 1 - distance
|
|
164
|
+
// For L2: we use 1 / (1 + distance) as approximation
|
|
165
|
+
const similarity = 1 / (1 + distance);
|
|
166
|
+
if (similarity < minScore)
|
|
167
|
+
continue;
|
|
168
|
+
const doc = this.recordToDoc(result);
|
|
169
|
+
if (options.filter && !options.filter(doc))
|
|
170
|
+
continue;
|
|
171
|
+
results.push({ doc, similarity, distance });
|
|
172
|
+
if (results.length >= k)
|
|
173
|
+
break;
|
|
174
|
+
}
|
|
175
|
+
return results;
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* Get document by ID
|
|
179
|
+
*/
|
|
180
|
+
async getDocument(docId) {
|
|
181
|
+
await this.ensureInitialized();
|
|
182
|
+
if (!this.table)
|
|
183
|
+
return undefined;
|
|
184
|
+
const results = await this.table
|
|
185
|
+
.query()
|
|
186
|
+
.where(`id = "${docId.replace(/"/g, '\\"')}"`)
|
|
187
|
+
.limit(1)
|
|
188
|
+
.toArray();
|
|
189
|
+
if (results.length === 0)
|
|
190
|
+
return undefined;
|
|
191
|
+
return this.recordToDoc(results[0]);
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Get all documents
|
|
195
|
+
*/
|
|
196
|
+
async getAllDocuments() {
|
|
197
|
+
await this.ensureInitialized();
|
|
198
|
+
if (!this.table)
|
|
199
|
+
return [];
|
|
200
|
+
const results = await this.table.query().toArray();
|
|
201
|
+
return results.map((r) => this.recordToDoc(r));
|
|
202
|
+
}
|
|
203
|
+
/**
|
|
204
|
+
* Check if document exists
|
|
205
|
+
*/
|
|
206
|
+
async hasDocument(docId) {
|
|
207
|
+
await this.ensureInitialized();
|
|
208
|
+
if (!this.table)
|
|
209
|
+
return false;
|
|
210
|
+
const results = await this.table
|
|
211
|
+
.query()
|
|
212
|
+
.where(`id = "${docId.replace(/"/g, '\\"')}"`)
|
|
213
|
+
.limit(1)
|
|
214
|
+
.toArray();
|
|
215
|
+
return results.length > 0;
|
|
216
|
+
}
|
|
217
|
+
/**
|
|
218
|
+
* Get statistics
|
|
219
|
+
*/
|
|
220
|
+
async getStats() {
|
|
221
|
+
await this.ensureInitialized();
|
|
222
|
+
if (!this.table) {
|
|
223
|
+
return {
|
|
224
|
+
totalDocuments: 0,
|
|
225
|
+
dimensions: this.dimensions,
|
|
226
|
+
indexSize: 0,
|
|
227
|
+
};
|
|
228
|
+
}
|
|
229
|
+
const count = await this.table.countRows();
|
|
230
|
+
return {
|
|
231
|
+
totalDocuments: count,
|
|
232
|
+
dimensions: this.dimensions,
|
|
233
|
+
indexSize: count,
|
|
234
|
+
};
|
|
235
|
+
}
|
|
236
|
+
/**
|
|
237
|
+
* Clear all data
|
|
238
|
+
*/
|
|
239
|
+
async clear() {
|
|
240
|
+
await this.ensureInitialized();
|
|
241
|
+
if (this.table && this.db) {
|
|
242
|
+
await this.db.dropTable(this.tableName);
|
|
243
|
+
this.table = null;
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
/**
|
|
247
|
+
* Close database connection and release resources
|
|
248
|
+
*/
|
|
249
|
+
async close() {
|
|
250
|
+
// Actually close LanceDB connection to release file handles and memory
|
|
251
|
+
if (this.db) {
|
|
252
|
+
try {
|
|
253
|
+
this.db.close();
|
|
254
|
+
}
|
|
255
|
+
catch {
|
|
256
|
+
// Ignore close errors - connection may already be closed
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
this.db = null;
|
|
260
|
+
this.table = null;
|
|
261
|
+
this.initialized = false;
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
//# sourceMappingURL=vector-storage.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vector-storage.js","sourceRoot":"","sources":["../src/vector-storage.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,OAAO,MAAM,kBAAkB,CAAA;AA0D3C;;GAEG;AACH,MAAM,OAAO,aAAa;IACjB,EAAE,GAA8B,IAAI,CAAA;IACpC,KAAK,GAAyB,IAAI,CAAA;IAClC,UAAU,CAAQ;IAClB,MAAM,CAAQ;IACd,SAAS,CAAQ;IACjB,WAAW,GAAY,KAAK,CAAA;IAEpC,YAAY,OAA6B;QACxC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAA;QACpC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,UAAU,CAAA;QAC1C,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,SAAS,CAAA;IAChD,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,iBAAiB;QAC9B,IAAI,IAAI,CAAC,WAAW;YAAE,OAAM;QAE5B,IAAI,CAAC,EAAE,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAE5C,wBAAwB;QACxB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,EAAE,CAAA;QACzC,IAAI,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YACrC,IAAI,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QACrD,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,IAAI,CAAA;IACxB,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,WAAW,CAAC,YAAsB;QAC/C,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAA;QAE9B,IAAI,IAAI,CAAC,KAAK;YAAE,OAAM;QAEtB,2CAA2C;QAC3C,gEAAgE;QAChE,MAAM,aAAa,GAAiB;YACnC,EAAE,EAAE,YAAY;YAChB,MAAM,EAAE,YAAY;YACpB,IAAI,EAAE,MAAM;YACZ,QAAQ,EAAE,EAAE;YACZ,OAAO,EAAE,EAAE;YACX,QAAQ,EAAE,EAAE;YACZ,IAAI,EAAE,EAAE;YACR,aAAa,EAAE,IAAI;SACnB,CAAA;QAED,IAAI,CAAC,KAAK,GAAG,MAAM,IAAI,CAAC,EAAE,EAAE,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC,aAAa,CAAC,CAAC,CAAA;QAExE,2BAA2B;QAC3B,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAA;IAC7C,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,GAAmB;QACtC,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,IAAI,EAAE,GAAG,GAAG,CAAC,QAAQ,CAAA;QACzE,OAAO;YACN,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,MAAM,EAAE,GAAG,CAAC,SAAqB;YACjC,IAAI;YACJ,QAAQ,EAAE,QAAQ,IAAI,EAAE;YACxB,OAAO,EAAE,OAAO,IAAI,EAAE;YACtB,QAAQ,EAAE,QAAQ,IAAI,EAAE;YACxB,IAAI,EAAE,IAAI,IAAI,EAAE;YAChB,aAAa,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;SACnC,CAAA;IACF,CAAC;IAED;;OAEG;IACK,WAAW,CAAC,MAAoB;QACvC,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,aAAa,IAAI,IAAI,CAAC,CAAA;QAC9D,OAAO;YACN,EAAE,EAAE,MAAM,CAAC,EAAE;YACb,SAAS,EAAE,MAAM,CAAC,MAAM;YACxB,QAAQ,EAAE;gBACT,IAAI,EAAE,MAAM,CAAC,IAA4B;gBACzC,GAAG,CAAC,MAAM,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;gBACrD,GAAG,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,OAAO,EAAE,CAAC;gBAClD,GAAG,CAAC,MAAM,CAAC,QAAQ,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC;gBACrD,GAAG,CAAC,MAAM,CAAC,IAAI,IAAI,EAAE,IAAI,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC;gBACzC,GAAG,aAAa;aAChB;SACD,CAAA;IACF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,GAAmB;QACpC,IAAI,GAAG,CAAC,SAAS,CAAC,MAAM,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;YAC9C,MAAM,IAAI,KAAK,CACd,yBAAyB,GAAG,CAAC,SAAS,CAAC,MAAM,mCAAmC,IAAI,CAAC,UAAU,GAAG,CAClG,CAAA;QACF,CAAC;QAED,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,SAAqB,CAAC,CAAA;QAEjD,kBAAkB;QAClB,IAAI,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;YACpC,MAAM,IAAI,KAAK,CAAC,oBAAoB,GAAG,CAAC,EAAE,iBAAiB,CAAC,CAAA;QAC7D,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAA;QACpC,MAAM,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAA;IAChC,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY,CAAC,IAA+B;QACjD,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;YAAE,OAAM;QAE7B,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;QACxB,IAAI,QAAQ,CAAC,SAAS,CAAC,MAAM,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;YACnD,MAAM,IAAI,KAAK,CACd,yBAAyB,QAAQ,CAAC,SAAS,CAAC,MAAM,mCAAmC,IAAI,CAAC,UAAU,GAAG,CACvG,CAAA;QACF,CAAC;QAED,MAAM,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,SAAqB,CAAC,CAAA;QAEtD,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAA;QACxD,MAAM,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,CAAA;IAC/B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,GAAmB;QACvC,MAAM,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;QACjC,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAA;IAC5B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,cAAc,CAAC,KAAa;QACjC,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAA;QAE9B,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO,KAAK,CAAA;QAE7B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAA;QAC5C,IAAI,CAAC,MAAM;YAAE,OAAO,KAAK,CAAA;QAEzB,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,CAAA;QAC/D,OAAO,IAAI,CAAA;IACZ,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,MAAM,CACX,WAA8B,EAC9B,UAII,EAAE;QAEN,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,IAAI,EAAE,CAAA;QACzB,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,CAAC,CAAA;QAEtC,IAAI,WAAW,CAAC,MAAM,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;YAC5C,MAAM,IAAI,KAAK,CACd,4BAA4B,WAAW,CAAC,MAAM,mCAAmC,IAAI,CAAC,UAAU,GAAG,CACnG,CAAA;QACF,CAAC;QAED,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAA;QAE9B,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,CAAA;QAE1B,0CAA0C;QAC1C,MAAM,OAAO,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;QAC1C,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,KAAK;aACpC,MAAM,CAAC,WAAuB,CAAC;aAC/B,KAAK,CAAC,OAAO,CAAC;aACd,OAAO,EAAE,CAAA;QAEX,MAAM,OAAO,GAAyB,EAAE,CAAA;QAExC,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;YACpC,MAAM,QAAQ,GAAG,MAAM,CAAC,SAAmB,CAAA;YAC3C,6DAA6D;YAC7D,iDAAiD;YACjD,qDAAqD;YACrD,MAAM,UAAU,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAA;YAErC,IAAI,UAAU,GAAG,QAAQ;gBAAE,SAAQ;YAEnC,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,MAAiC,CAAC,CAAA;YAE/D,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC;gBAAE,SAAQ;YAEpD,OAAO,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,CAAA;YAE3C,IAAI,OAAO,CAAC,MAAM,IAAI,CAAC;gBAAE,MAAK;QAC/B,CAAC;QAED,OAAO,OAAO,CAAA;IACf,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,KAAa;QAC9B,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAA;QAE9B,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO,SAAS,CAAA;QAEjC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK;aAC9B,KAAK,EAAE;aACP,KAAK,CAAC,SAAS,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC;aAC7C,KAAK,CAAC,CAAC,CAAC;aACR,OAAO,EAAE,CAAA;QAEX,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,SAAS,CAAA;QAE1C,OAAO,IAAI,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC,CAA4B,CAAC,CAAA;IAC/D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,eAAe;QACpB,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAA;QAE9B,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO,EAAE,CAAA;QAE1B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,OAAO,EAAE,CAAA;QAClD,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAA4B,CAAC,CAAC,CAAA;IAC1E,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,WAAW,CAAC,KAAa;QAC9B,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAA;QAE9B,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,OAAO,KAAK,CAAA;QAE7B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK;aAC9B,KAAK,EAAE;aACP,KAAK,CAAC,SAAS,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC;aAC7C,KAAK,CAAC,CAAC,CAAC;aACR,OAAO,EAAE,CAAA;QAEX,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAA;IAC1B,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ;QACb,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAA;QAE9B,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;YACjB,OAAO;gBACN,cAAc,EAAE,CAAC;gBACjB,UAAU,EAAE,IAAI,CAAC,UAAU;gBAC3B,SAAS,EAAE,CAAC;aACZ,CAAA;QACF,CAAC;QAED,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,CAAA;QAE1C,OAAO;YACN,cAAc,EAAE,KAAK;YACrB,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,SAAS,EAAE,KAAK;SAChB,CAAA;IACF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACV,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAA;QAE9B,IAAI,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,IAAI,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YACvC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;QAClB,CAAC;IACF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,KAAK;QACV,uEAAuE;QACvE,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;YACb,IAAI,CAAC;gBACJ,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAA;YAChB,CAAC;YAAC,MAAM,CAAC;gBACR,yDAAyD;YAC1D,CAAC;QACF,CAAC;QACD,IAAI,CAAC,EAAE,GAAG,IAAI,CAAA;QACd,IAAI,CAAC,KAAK,GAAG,IAAI,CAAA;QACjB,IAAI,CAAC,WAAW,GAAG,KAAK,CAAA;IACzB,CAAC;CACD"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vector-storage.test.d.ts","sourceRoot":"","sources":["../src/vector-storage.test.ts"],"names":[],"mappings":"AAAA;;GAEG"}
|