@mduenas/codegraph 0.6.0 → 0.6.1-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +1 -1
- package/dist/vectors/manager.d.ts +2 -2
- package/dist/vectors/manager.js +4 -4
- package/dist/vectors/search.d.ts +17 -17
- package/dist/vectors/search.d.ts.map +1 -1
- package/dist/vectors/search.js +71 -122
- package/dist/vectors/search.js.map +1 -1
- package/package.json +2 -2
package/dist/index.d.ts
CHANGED
|
@@ -95,7 +95,7 @@ export declare class VectorManager {
|
|
|
95
95
|
*/
|
|
96
96
|
getStats(): {
|
|
97
97
|
totalVectors: number;
|
|
98
|
-
|
|
98
|
+
vecEnabled: boolean;
|
|
99
99
|
modelId: string;
|
|
100
100
|
dimension: number;
|
|
101
101
|
};
|
|
@@ -104,7 +104,7 @@ export declare class VectorManager {
|
|
|
104
104
|
*/
|
|
105
105
|
clear(): void;
|
|
106
106
|
/**
|
|
107
|
-
* Rebuild the
|
|
107
|
+
* Rebuild the vec index
|
|
108
108
|
*/
|
|
109
109
|
rebuildIndex(): void;
|
|
110
110
|
/**
|
package/dist/vectors/manager.js
CHANGED
|
@@ -54,7 +54,7 @@ class VectorManager {
|
|
|
54
54
|
}
|
|
55
55
|
// Initialize embedder (downloads model if needed)
|
|
56
56
|
await this.embedder.initialize();
|
|
57
|
-
// Initialize vector search (loads sqlite-
|
|
57
|
+
// Initialize vector search (loads sqlite-vec if available)
|
|
58
58
|
await this.searchManager.initialize();
|
|
59
59
|
this.initialized = true;
|
|
60
60
|
}
|
|
@@ -240,7 +240,7 @@ class VectorManager {
|
|
|
240
240
|
getStats() {
|
|
241
241
|
return {
|
|
242
242
|
totalVectors: this.searchManager.getVectorCount(),
|
|
243
|
-
|
|
243
|
+
vecEnabled: this.searchManager.isVecEnabled(),
|
|
244
244
|
modelId: this.embedder.getModelId(),
|
|
245
245
|
dimension: this.embedder.getDimension(),
|
|
246
246
|
};
|
|
@@ -252,10 +252,10 @@ class VectorManager {
|
|
|
252
252
|
this.searchManager.clear();
|
|
253
253
|
}
|
|
254
254
|
/**
|
|
255
|
-
* Rebuild the
|
|
255
|
+
* Rebuild the vec index
|
|
256
256
|
*/
|
|
257
257
|
rebuildIndex() {
|
|
258
|
-
this.searchManager.
|
|
258
|
+
this.searchManager.rebuildVecIndex();
|
|
259
259
|
}
|
|
260
260
|
/**
|
|
261
261
|
* Release resources
|
package/dist/vectors/search.d.ts
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Vector Search
|
|
3
3
|
*
|
|
4
|
-
* Provides vector similarity search using sqlite-
|
|
5
|
-
* Falls back to brute-force cosine similarity if sqlite-
|
|
4
|
+
* Provides vector similarity search using sqlite-vec extension.
|
|
5
|
+
* Falls back to brute-force cosine similarity if sqlite-vec is not available.
|
|
6
6
|
*/
|
|
7
7
|
import Database from 'better-sqlite3';
|
|
8
8
|
import { Node } from '../types';
|
|
@@ -24,32 +24,32 @@ export interface VectorSearchOptions {
|
|
|
24
24
|
*/
|
|
25
25
|
export declare class VectorSearchManager {
|
|
26
26
|
private db;
|
|
27
|
-
private
|
|
27
|
+
private vecEnabled;
|
|
28
28
|
private embeddingDimension;
|
|
29
29
|
constructor(db: Database.Database, dimension?: number);
|
|
30
30
|
/**
|
|
31
31
|
* Initialize vector search
|
|
32
32
|
*
|
|
33
|
-
* Attempts to load sqlite-
|
|
33
|
+
* Attempts to load sqlite-vec extension. Falls back to brute-force
|
|
34
34
|
* search if the extension is not available.
|
|
35
35
|
*/
|
|
36
36
|
initialize(): Promise<void>;
|
|
37
37
|
/**
|
|
38
|
-
* Load the sqlite-
|
|
38
|
+
* Load the sqlite-vec extension
|
|
39
39
|
*/
|
|
40
|
-
private
|
|
40
|
+
private loadVecExtension;
|
|
41
41
|
/**
|
|
42
|
-
* Create the
|
|
42
|
+
* Create the vec virtual table for vector search
|
|
43
43
|
*/
|
|
44
|
-
private
|
|
44
|
+
private createVecTable;
|
|
45
45
|
/**
|
|
46
46
|
* Ensure the basic vectors table exists (for fallback mode)
|
|
47
47
|
*/
|
|
48
48
|
private ensureVectorsTable;
|
|
49
49
|
/**
|
|
50
|
-
* Check if
|
|
50
|
+
* Check if vec extension is enabled
|
|
51
51
|
*/
|
|
52
|
-
|
|
52
|
+
isVecEnabled(): boolean;
|
|
53
53
|
/**
|
|
54
54
|
* Store a vector embedding for a node
|
|
55
55
|
*
|
|
@@ -59,9 +59,9 @@ export declare class VectorSearchManager {
|
|
|
59
59
|
*/
|
|
60
60
|
storeVector(nodeId: string, embedding: Float32Array, model: string): void;
|
|
61
61
|
/**
|
|
62
|
-
* Store vector in
|
|
62
|
+
* Store vector in vec virtual table
|
|
63
63
|
*/
|
|
64
|
-
private
|
|
64
|
+
private storeInVec;
|
|
65
65
|
/**
|
|
66
66
|
* Store multiple vectors in a batch
|
|
67
67
|
*
|
|
@@ -97,9 +97,9 @@ export declare class VectorSearchManager {
|
|
|
97
97
|
score: number;
|
|
98
98
|
}>;
|
|
99
99
|
/**
|
|
100
|
-
* Search using sqlite-
|
|
100
|
+
* Search using sqlite-vec KNN search
|
|
101
101
|
*/
|
|
102
|
-
private
|
|
102
|
+
private searchWithVec;
|
|
103
103
|
/**
|
|
104
104
|
* Brute-force search using cosine similarity
|
|
105
105
|
*/
|
|
@@ -121,11 +121,11 @@ export declare class VectorSearchManager {
|
|
|
121
121
|
*/
|
|
122
122
|
clear(): void;
|
|
123
123
|
/**
|
|
124
|
-
* Rebuild
|
|
124
|
+
* Rebuild vec index from vectors table
|
|
125
125
|
*
|
|
126
|
-
* Useful after bulk operations or if
|
|
126
|
+
* Useful after bulk operations or if vec index gets out of sync.
|
|
127
127
|
*/
|
|
128
|
-
|
|
128
|
+
rebuildVecIndex(): void;
|
|
129
129
|
}
|
|
130
130
|
/**
|
|
131
131
|
* Create a vector search manager
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/vectors/search.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAGhC;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,0CAA0C;IAC1C,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,qCAAqC;IACrC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,mCAAmC;IACnC,SAAS,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;CAC5B;AAED;;;;GAIG;AACH,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,EAAE,CAAoB;IAC9B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,kBAAkB,CAAS;gBAEvB,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,SAAS,GAAE,MAA4B;IAK1E;;;;;OAKG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAsBjC;;OAEG;YACW,gBAAgB;
|
|
1
|
+
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../src/vectors/search.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,IAAI,EAAE,MAAM,UAAU,CAAC;AAGhC;;GAEG;AACH,MAAM,WAAW,mBAAmB;IAClC,0CAA0C;IAC1C,KAAK,CAAC,EAAE,MAAM,CAAC;IAEf,qCAAqC;IACrC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAElB,mCAAmC;IACnC,SAAS,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;CAC5B;AAED;;;;GAIG;AACH,qBAAa,mBAAmB;IAC9B,OAAO,CAAC,EAAE,CAAoB;IAC9B,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,kBAAkB,CAAS;gBAEvB,EAAE,EAAE,QAAQ,CAAC,QAAQ,EAAE,SAAS,GAAE,MAA4B;IAK1E;;;;;OAKG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAsBjC;;OAEG;YACW,gBAAgB;IAgB9B;;OAEG;IACH,OAAO,CAAC,cAAc;IAiBtB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IAW1B;;OAEG;IACH,YAAY,IAAI,OAAO;IAIvB;;;;;;OAMG;IACH,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAoBzE;;OAEG;IACH,OAAO,CAAC,UAAU;IAiBlB;;;;;OAKG;IACH,gBAAgB,CACd,OAAO,EAAE,KAAK,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,YAAY,CAAA;KAAE,CAAC,EAC3D,KAAK,EAAE,MAAM,GACZ,IAAI;IAuBP;;;;;OAKG;IACH,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,YAAY,GAAG,IAAI;IAe9C;;;;OAIG;IACH,YAAY,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI;IAQlC;;;;;;OAMG;IACH,MAAM,CACJ,cAAc,EAAE,YAAY,EAC5B,OAAO,GAAE,mBAAwB,GAChC,KAAK,CAAC;QAAE,MAAM,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC;IAU3C;;OAEG;IACH,OAAO,CAAC,aAAa;IA0CrB;;OAEG;IACH,OAAO,CAAC,gBAAgB;IA+BxB;;OAEG;IACH,cAAc,IAAI,MAAM;IAOxB;;OAEG;IACH,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO;IAOlC;;OAEG;IACH,iBAAiB,IAAI,MAAM,EAAE;IAO7B;;OAEG;IACH,KAAK,IAAI,IAAI;IAQb;;;;OAIG;IACH,eAAe,IAAI,IAAI;CAuBxB;AAED;;GAEG;AACH,wBAAgB,kBAAkB,CAChC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,SAAS,CAAC,EAAE,MAAM,GACjB,mBAAmB,CAErB"}
|
package/dist/vectors/search.js
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
/**
|
|
3
3
|
* Vector Search
|
|
4
4
|
*
|
|
5
|
-
* Provides vector similarity search using sqlite-
|
|
6
|
-
* Falls back to brute-force cosine similarity if sqlite-
|
|
5
|
+
* Provides vector similarity search using sqlite-vec extension.
|
|
6
|
+
* Falls back to brute-force cosine similarity if sqlite-vec is not available.
|
|
7
7
|
*/
|
|
8
8
|
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
9
9
|
if (k2 === undefined) k2 = k;
|
|
@@ -49,7 +49,7 @@ const embedder_1 = require("./embedder");
|
|
|
49
49
|
*/
|
|
50
50
|
class VectorSearchManager {
|
|
51
51
|
db;
|
|
52
|
-
|
|
52
|
+
vecEnabled = false;
|
|
53
53
|
embeddingDimension;
|
|
54
54
|
constructor(db, dimension = embedder_1.EMBEDDING_DIMENSION) {
|
|
55
55
|
this.db = db;
|
|
@@ -58,74 +58,61 @@ class VectorSearchManager {
|
|
|
58
58
|
/**
|
|
59
59
|
* Initialize vector search
|
|
60
60
|
*
|
|
61
|
-
* Attempts to load sqlite-
|
|
61
|
+
* Attempts to load sqlite-vec extension. Falls back to brute-force
|
|
62
62
|
* search if the extension is not available.
|
|
63
63
|
*/
|
|
64
64
|
async initialize() {
|
|
65
65
|
try {
|
|
66
|
-
// Try to load sqlite-
|
|
67
|
-
await this.
|
|
68
|
-
this.
|
|
69
|
-
console.log('sqlite-
|
|
70
|
-
// Create the
|
|
71
|
-
this.
|
|
66
|
+
// Try to load sqlite-vec extension
|
|
67
|
+
await this.loadVecExtension();
|
|
68
|
+
this.vecEnabled = true;
|
|
69
|
+
console.log('sqlite-vec extension loaded successfully');
|
|
70
|
+
// Create the vec virtual table
|
|
71
|
+
this.createVecTable();
|
|
72
72
|
}
|
|
73
73
|
catch (error) {
|
|
74
74
|
// Fall back to brute-force search
|
|
75
|
-
console.warn('sqlite-
|
|
76
|
-
this.
|
|
75
|
+
console.warn('sqlite-vec extension not available, falling back to brute-force search:', error instanceof Error ? error.message : String(error));
|
|
76
|
+
this.vecEnabled = false;
|
|
77
77
|
}
|
|
78
|
-
// Ensure the vectors table exists (for both
|
|
78
|
+
// Ensure the vectors table exists (for both vec and fallback modes)
|
|
79
79
|
this.ensureVectorsTable();
|
|
80
80
|
}
|
|
81
81
|
/**
|
|
82
|
-
* Load the sqlite-
|
|
82
|
+
* Load the sqlite-vec extension
|
|
83
83
|
*/
|
|
84
|
-
async
|
|
84
|
+
async loadVecExtension() {
|
|
85
85
|
try {
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
if (typeof vss.load === 'function') {
|
|
90
|
-
vss.load(this.db);
|
|
86
|
+
const sqliteVec = await Promise.resolve().then(() => __importStar(require('sqlite-vec')));
|
|
87
|
+
if (typeof sqliteVec.load === 'function') {
|
|
88
|
+
sqliteVec.load(this.db);
|
|
91
89
|
}
|
|
92
|
-
else if (typeof
|
|
93
|
-
|
|
90
|
+
else if (typeof sqliteVec.default?.load === 'function') {
|
|
91
|
+
sqliteVec.default.load(this.db);
|
|
94
92
|
}
|
|
95
93
|
else {
|
|
96
|
-
throw new Error('sqlite-
|
|
94
|
+
throw new Error('sqlite-vec load function not found');
|
|
97
95
|
}
|
|
98
96
|
}
|
|
99
97
|
catch (error) {
|
|
100
|
-
throw new Error(`Failed to load sqlite-
|
|
98
|
+
throw new Error(`Failed to load sqlite-vec: ${error instanceof Error ? error.message : String(error)}`);
|
|
101
99
|
}
|
|
102
100
|
}
|
|
103
101
|
/**
|
|
104
|
-
* Create the
|
|
102
|
+
* Create the vec virtual table for vector search
|
|
105
103
|
*/
|
|
106
|
-
|
|
104
|
+
createVecTable() {
|
|
107
105
|
// Check if the table already exists
|
|
108
106
|
const tableExists = this.db
|
|
109
|
-
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='
|
|
107
|
+
.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='vec_vectors'")
|
|
110
108
|
.get();
|
|
111
109
|
if (!tableExists) {
|
|
112
|
-
// Create
|
|
113
|
-
// vss0 is the vector search extension
|
|
110
|
+
// Create vec0 virtual table with node_id as auxiliary column
|
|
114
111
|
this.db.exec(`
|
|
115
|
-
CREATE VIRTUAL TABLE IF NOT EXISTS
|
|
116
|
-
|
|
112
|
+
CREATE VIRTUAL TABLE IF NOT EXISTS vec_vectors USING vec0(
|
|
113
|
+
node_id text,
|
|
114
|
+
embedding float[${this.embeddingDimension}]
|
|
117
115
|
);
|
|
118
|
-
`);
|
|
119
|
-
// Create mapping table to link VSS rowids to node IDs
|
|
120
|
-
this.db.exec(`
|
|
121
|
-
CREATE TABLE IF NOT EXISTS vss_map (
|
|
122
|
-
rowid INTEGER PRIMARY KEY,
|
|
123
|
-
node_id TEXT NOT NULL UNIQUE
|
|
124
|
-
);
|
|
125
|
-
`);
|
|
126
|
-
// Create index on node_id
|
|
127
|
-
this.db.exec(`
|
|
128
|
-
CREATE INDEX IF NOT EXISTS idx_vss_map_node ON vss_map(node_id);
|
|
129
116
|
`);
|
|
130
117
|
}
|
|
131
118
|
}
|
|
@@ -143,10 +130,10 @@ class VectorSearchManager {
|
|
|
143
130
|
`);
|
|
144
131
|
}
|
|
145
132
|
/**
|
|
146
|
-
* Check if
|
|
133
|
+
* Check if vec extension is enabled
|
|
147
134
|
*/
|
|
148
|
-
|
|
149
|
-
return this.
|
|
135
|
+
isVecEnabled() {
|
|
136
|
+
return this.vecEnabled;
|
|
150
137
|
}
|
|
151
138
|
/**
|
|
152
139
|
* Store a vector embedding for a node
|
|
@@ -165,47 +152,25 @@ class VectorSearchManager {
|
|
|
165
152
|
VALUES (?, ?, ?, ?)
|
|
166
153
|
`)
|
|
167
154
|
.run(nodeId, blob, model, now);
|
|
168
|
-
// Also store in
|
|
169
|
-
if (this.
|
|
170
|
-
this.
|
|
155
|
+
// Also store in vec table if enabled
|
|
156
|
+
if (this.vecEnabled) {
|
|
157
|
+
this.storeInVec(nodeId, embedding);
|
|
171
158
|
}
|
|
172
159
|
}
|
|
173
160
|
/**
|
|
174
|
-
* Store vector in
|
|
161
|
+
* Store vector in vec virtual table
|
|
175
162
|
*/
|
|
176
|
-
|
|
163
|
+
storeInVec(nodeId, embedding) {
|
|
177
164
|
try {
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
const vectorJson = JSON.stringify(Array.from(embedding));
|
|
185
|
-
this.db
|
|
186
|
-
.prepare('UPDATE vss_vectors SET embedding = ? WHERE rowid = ?')
|
|
187
|
-
.run(vectorJson, existing.rowid);
|
|
188
|
-
}
|
|
189
|
-
else {
|
|
190
|
-
// Insert new vector - get max rowid and increment
|
|
191
|
-
const maxRow = this.db
|
|
192
|
-
.prepare('SELECT MAX(rowid) as max FROM vss_map')
|
|
193
|
-
.get();
|
|
194
|
-
const newRowid = (maxRow?.max ?? 0) + 1;
|
|
195
|
-
const vectorJson = JSON.stringify(Array.from(embedding));
|
|
196
|
-
this.db
|
|
197
|
-
.prepare('INSERT INTO vss_vectors (rowid, embedding) VALUES (?, ?)')
|
|
198
|
-
.run(newRowid, vectorJson);
|
|
199
|
-
// Map the rowid to node_id
|
|
200
|
-
this.db
|
|
201
|
-
.prepare('INSERT INTO vss_map (rowid, node_id) VALUES (?, ?)')
|
|
202
|
-
.run(newRowid, nodeId);
|
|
203
|
-
}
|
|
165
|
+
const buffer = Buffer.from(embedding.buffer, embedding.byteOffset, embedding.byteLength);
|
|
166
|
+
// Delete existing entry if present, then insert
|
|
167
|
+
this.db.prepare('DELETE FROM vec_vectors WHERE node_id = ?').run(nodeId);
|
|
168
|
+
this.db
|
|
169
|
+
.prepare('INSERT INTO vec_vectors(node_id, embedding) VALUES (?, ?)')
|
|
170
|
+
.run(nodeId, buffer);
|
|
204
171
|
}
|
|
205
172
|
catch (error) {
|
|
206
|
-
|
|
207
|
-
// Fall back to brute-force search silently
|
|
208
|
-
console.warn('VSS storage failed, using brute-force search:', error instanceof Error ? error.message : String(error));
|
|
173
|
+
console.warn('vec storage failed, using brute-force search:', error instanceof Error ? error.message : String(error));
|
|
209
174
|
}
|
|
210
175
|
}
|
|
211
176
|
/**
|
|
@@ -226,8 +191,8 @@ class VectorSearchManager {
|
|
|
226
191
|
VALUES (?, ?, ?, ?)
|
|
227
192
|
`)
|
|
228
193
|
.run(entry.nodeId, blob, model, now);
|
|
229
|
-
if (this.
|
|
230
|
-
this.
|
|
194
|
+
if (this.vecEnabled) {
|
|
195
|
+
this.storeInVec(entry.nodeId, entry.embedding);
|
|
231
196
|
}
|
|
232
197
|
}
|
|
233
198
|
})();
|
|
@@ -254,15 +219,8 @@ class VectorSearchManager {
|
|
|
254
219
|
*/
|
|
255
220
|
deleteVector(nodeId) {
|
|
256
221
|
this.db.prepare('DELETE FROM vectors WHERE node_id = ?').run(nodeId);
|
|
257
|
-
if (this.
|
|
258
|
-
|
|
259
|
-
const mapping = this.db
|
|
260
|
-
.prepare('SELECT rowid FROM vss_map WHERE node_id = ?')
|
|
261
|
-
.get(nodeId);
|
|
262
|
-
if (mapping) {
|
|
263
|
-
this.db.prepare('DELETE FROM vss_vectors WHERE rowid = ?').run(mapping.rowid);
|
|
264
|
-
this.db.prepare('DELETE FROM vss_map WHERE node_id = ?').run(nodeId);
|
|
265
|
-
}
|
|
222
|
+
if (this.vecEnabled) {
|
|
223
|
+
this.db.prepare('DELETE FROM vec_vectors WHERE node_id = ?').run(nodeId);
|
|
266
224
|
}
|
|
267
225
|
}
|
|
268
226
|
/**
|
|
@@ -274,37 +232,30 @@ class VectorSearchManager {
|
|
|
274
232
|
*/
|
|
275
233
|
search(queryEmbedding, options = {}) {
|
|
276
234
|
const { limit = 10, minScore = 0 } = options;
|
|
277
|
-
if (this.
|
|
278
|
-
return this.
|
|
235
|
+
if (this.vecEnabled) {
|
|
236
|
+
return this.searchWithVec(queryEmbedding, limit, minScore);
|
|
279
237
|
}
|
|
280
238
|
else {
|
|
281
239
|
return this.searchBruteForce(queryEmbedding, limit, minScore);
|
|
282
240
|
}
|
|
283
241
|
}
|
|
284
242
|
/**
|
|
285
|
-
* Search using sqlite-
|
|
243
|
+
* Search using sqlite-vec KNN search
|
|
286
244
|
*/
|
|
287
|
-
|
|
245
|
+
searchWithVec(queryEmbedding, limit, minScore) {
|
|
288
246
|
try {
|
|
289
|
-
const
|
|
290
|
-
// Sanitize limit to prevent SQL injection (ensure it's a positive integer)
|
|
247
|
+
const buffer = Buffer.from(queryEmbedding.buffer, queryEmbedding.byteOffset, queryEmbedding.byteLength);
|
|
291
248
|
const safeLimit = Math.max(1, Math.floor(limit));
|
|
292
|
-
// Use
|
|
293
|
-
// The distance is L2 (euclidean), we need to convert to similarity score
|
|
294
|
-
// Note: sqlite-vss requires LIMIT to be a literal, not a parameter
|
|
249
|
+
// Use vec0 KNN search with MATCH syntax
|
|
295
250
|
const rows = this.db
|
|
296
251
|
.prepare(`
|
|
297
|
-
SELECT
|
|
298
|
-
FROM
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
WHERE vss_search(embedding, ?)
|
|
302
|
-
LIMIT ${safeLimit}
|
|
303
|
-
) v
|
|
304
|
-
JOIN vss_map m ON m.rowid = v.rowid
|
|
252
|
+
SELECT node_id, distance
|
|
253
|
+
FROM vec_vectors
|
|
254
|
+
WHERE embedding MATCH ?
|
|
255
|
+
AND k = ${safeLimit}
|
|
305
256
|
`)
|
|
306
|
-
.all(
|
|
307
|
-
// Convert
|
|
257
|
+
.all(buffer);
|
|
258
|
+
// Convert distance to similarity score (1 / (1 + distance))
|
|
308
259
|
return rows
|
|
309
260
|
.map((row) => ({
|
|
310
261
|
nodeId: row.node_id,
|
|
@@ -313,8 +264,8 @@ class VectorSearchManager {
|
|
|
313
264
|
.filter((r) => r.score >= minScore);
|
|
314
265
|
}
|
|
315
266
|
catch (error) {
|
|
316
|
-
//
|
|
317
|
-
console.warn('
|
|
267
|
+
// vec search failed, fall back to brute force
|
|
268
|
+
console.warn('vec search failed, using brute-force:', error instanceof Error ? error.message : String(error));
|
|
318
269
|
return this.searchBruteForce(queryEmbedding, limit, minScore);
|
|
319
270
|
}
|
|
320
271
|
}
|
|
@@ -371,23 +322,21 @@ class VectorSearchManager {
|
|
|
371
322
|
*/
|
|
372
323
|
clear() {
|
|
373
324
|
this.db.prepare('DELETE FROM vectors').run();
|
|
374
|
-
if (this.
|
|
375
|
-
this.db.prepare('DELETE FROM
|
|
376
|
-
this.db.prepare('DELETE FROM vss_map').run();
|
|
325
|
+
if (this.vecEnabled) {
|
|
326
|
+
this.db.prepare('DELETE FROM vec_vectors').run();
|
|
377
327
|
}
|
|
378
328
|
}
|
|
379
329
|
/**
|
|
380
|
-
* Rebuild
|
|
330
|
+
* Rebuild vec index from vectors table
|
|
381
331
|
*
|
|
382
|
-
* Useful after bulk operations or if
|
|
332
|
+
* Useful after bulk operations or if vec index gets out of sync.
|
|
383
333
|
*/
|
|
384
|
-
|
|
385
|
-
if (!this.
|
|
334
|
+
rebuildVecIndex() {
|
|
335
|
+
if (!this.vecEnabled) {
|
|
386
336
|
return;
|
|
387
337
|
}
|
|
388
|
-
// Clear
|
|
389
|
-
this.db.prepare('DELETE FROM
|
|
390
|
-
this.db.prepare('DELETE FROM vss_map').run();
|
|
338
|
+
// Clear vec table
|
|
339
|
+
this.db.prepare('DELETE FROM vec_vectors').run();
|
|
391
340
|
// Reload from vectors table
|
|
392
341
|
const rows = this.db
|
|
393
342
|
.prepare('SELECT node_id, embedding FROM vectors')
|
|
@@ -395,7 +344,7 @@ class VectorSearchManager {
|
|
|
395
344
|
this.db.transaction(() => {
|
|
396
345
|
for (const row of rows) {
|
|
397
346
|
const embedding = new Float32Array(row.embedding.buffer.slice(row.embedding.byteOffset, row.embedding.byteOffset + row.embedding.byteLength));
|
|
398
|
-
this.
|
|
347
|
+
this.storeInVec(row.node_id, embedding);
|
|
399
348
|
}
|
|
400
349
|
})();
|
|
401
350
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search.js","sourceRoot":"","sources":["../../src/vectors/search.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
|
|
1
|
+
{"version":3,"file":"search.js","sourceRoot":"","sources":["../../src/vectors/search.ts"],"names":[],"mappings":";AAAA;;;;;GAKG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAyZH,gDAKC;AA1ZD,yCAA+D;AAgB/D;;;;GAIG;AACH,MAAa,mBAAmB;IACtB,EAAE,CAAoB;IACtB,UAAU,GAAG,KAAK,CAAC;IACnB,kBAAkB,CAAS;IAEnC,YAAY,EAAqB,EAAE,YAAoB,8BAAmB;QACxE,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC;QACb,IAAI,CAAC,kBAAkB,GAAG,SAAS,CAAC;IACtC,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU;QACd,IAAI,CAAC;YACH,mCAAmC;YACnC,MAAM,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC9B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;YACvB,OAAO,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC;YAExD,+BAA+B;YAC/B,IAAI,CAAC,cAAc,EAAE,CAAC;QACxB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,kCAAkC;YAClC,OAAO,CAAC,IAAI,CACV,yEAAyE,EACzE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CACvD,CAAC;YACF,IAAI,CAAC,UAAU,GAAG,KAAK,CAAC;QAC1B,CAAC;QAED,oEAAoE;QACpE,IAAI,CAAC,kBAAkB,EAAE,CAAC;IAC5B,CAAC;IAED;;OAEG;IACK,KAAK,CAAC,gBAAgB;QAC5B,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,wDAAa,YAAY,GAAC,CAAC;YAE7C,IAAI,OAAO,SAAS,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;gBACzC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC1B,CAAC;iBAAM,IAAI,OAAO,SAAS,CAAC,OAAO,EAAE,IAAI,KAAK,UAAU,EAAE,CAAC;gBACzD,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAClC,CAAC;iBAAM,CAAC;gBACN,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;YACxD,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,8BAA8B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC1G,CAAC;IACH,CAAC;IAED;;OAEG;IACK,cAAc;QACpB,oCAAoC;QACpC,MAAM,WAAW,GAAG,IAAI,CAAC,EAAE;aACxB,OAAO,CAAC,0EAA0E,CAAC;aACnF,GAAG,EAAE,CAAC;QAET,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,6DAA6D;YAC7D,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;4BAGS,IAAI,CAAC,kBAAkB;;OAE5C,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED;;OAEG;IACK,kBAAkB;QACxB,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC;;;;;;;KAOZ,CAAC,CAAC;IACL,CAAC;IAED;;OAEG;IACH,YAAY;QACV,OAAO,IAAI,CAAC,UAAU,CAAC;IACzB,CAAC;IAED;;;;;;OAMG;IACH,WAAW,CAAC,MAAc,EAAE,SAAuB,EAAE,KAAa;QAChE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,uDAAuD;QACvD,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;QAC3C,IAAI,CAAC,EAAE;aACJ,OAAO,CACN;;;OAGD,CACA;aACA,GAAG,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;QAEjC,qCAAqC;QACrC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IAED;;OAEG;IACK,UAAU,CAAC,MAAc,EAAE,SAAuB;QACxD,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,UAAU,EAAE,SAAS,CAAC,UAAU,CAAC,CAAC;YAEzF,gDAAgD;YAChD,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,2CAA2C,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACzE,IAAI,CAAC,EAAE;iBACJ,OAAO,CAAC,2DAA2D,CAAC;iBACpE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QACzB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,IAAI,CACV,+CAA+C,EAC/C,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CACvD,CAAC;QACJ,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,gBAAgB,CACd,OAA2D,EAC3D,KAAa;QAEb,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,2CAA2C;QAC3C,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YACvB,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;gBAC5B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;gBACjD,IAAI,CAAC,EAAE;qBACJ,OAAO,CACN;;;WAGD,CACA;qBACA,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;gBAEvC,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;oBACpB,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;gBACjD,CAAC;YACH,CAAC;QACH,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;IAED;;;;;OAKG;IACH,SAAS,CAAC,MAAc;QACtB,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,OAAO,CAAC,iDAAiD,CAAC;aAC1D,GAAG,CAAC,MAAM,CAAsC,CAAC;QAEpD,IAAI,CAAC,GAAG,EAAE,CAAC;YACT,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,IAAI,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAChD,GAAG,CAAC,SAAS,CAAC,UAAU,EACxB,GAAG,CAAC,SAAS,CAAC,UAAU,GAAG,GAAG,CAAC,SAAS,CAAC,UAAU,CACpD,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IACH,YAAY,CAAC,MAAc;QACzB,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,uCAAuC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAErE,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,2CAA2C,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC;IAED;;;;;;OAMG;IACH,MAAM,CACJ,cAA4B,EAC5B,UAA+B,EAAE;QAEjC,MAAM,EAAE,KAAK,GAAG,EAAE,EAAE,QAAQ,GAAG,CAAC,EAAE,GAAG,OAAO,CAAC;QAE7C,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,OAAO,IAAI,CAAC,aAAa,CAAC,cAAc,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC7D,CAAC;aAAM,CAAC;YACN,OAAO,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IAED;;OAEG;IACK,aAAa,CACnB,cAA4B,EAC5B,KAAa,EACb,QAAgB;QAEhB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CACxB,cAAc,CAAC,MAAM,EACrB,cAAc,CAAC,UAAU,EACzB,cAAc,CAAC,UAAU,CAC1B,CAAC;YACF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC;YAEjD,wCAAwC;YACxC,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;iBACjB,OAAO,CACN;;;;sBAIY,SAAS;SACtB,CACA;iBACA,GAAG,CAAC,MAAM,CAAiD,CAAC;YAE/D,4DAA4D;YAC5D,OAAO,IAAI;iBACR,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;gBACb,MAAM,EAAE,GAAG,CAAC,OAAO;gBACnB,KAAK,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC;aAC9B,CAAC,CAAC;iBACF,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,QAAQ,CAAC,CAAC;QACxC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,8CAA8C;YAC9C,OAAO,CAAC,IAAI,CACV,uCAAuC,EACvC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CACvD,CAAC;YACF,OAAO,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IAED;;OAEG;IACK,gBAAgB,CACtB,cAA4B,EAC5B,KAAa,EACb,QAAgB;QAEhB,kBAAkB;QAClB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,OAAO,CAAC,wCAAwC,CAAC;aACjD,GAAG,EAAmD,CAAC;QAE1D,uCAAuC;QACvC,MAAM,OAAO,GAA6C,EAAE,CAAC;QAE7D,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;YACvB,MAAM,SAAS,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAC3D,GAAG,CAAC,SAAS,CAAC,UAAU,EACxB,GAAG,CAAC,SAAS,CAAC,UAAU,GAAG,GAAG,CAAC,SAAS,CAAC,UAAU,CACpD,CAAC,CAAC;YAEH,MAAM,KAAK,GAAG,uBAAY,CAAC,gBAAgB,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;YAEvE,IAAI,KAAK,IAAI,QAAQ,EAAE,CAAC;gBACtB,OAAO,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC;YAC/C,CAAC;QACH,CAAC;QAED,qCAAqC;QACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QAC1C,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IACjC,CAAC;IAED;;OAEG;IACH,cAAc;QACZ,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE;aACnB,OAAO,CAAC,uCAAuC,CAAC;aAChD,GAAG,EAAuB,CAAC;QAC9B,OAAO,MAAM,CAAC,KAAK,CAAC;IACtB,CAAC;IAED;;OAEG;IACH,SAAS,CAAC,MAAc;QACtB,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE;aACnB,OAAO,CAAC,iDAAiD,CAAC;aAC1D,GAAG,CAAC,MAAM,CAAC,CAAC;QACf,OAAO,CAAC,CAAC,MAAM,CAAC;IAClB,CAAC;IAED;;OAEG;IACH,iBAAiB;QACf,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,OAAO,CAAC,6BAA6B,CAAC;aACtC,GAAG,EAAgC,CAAC;QACvC,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IACpC,CAAC;IAED;;OAEG;IACH,KAAK;QACH,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC,GAAG,EAAE,CAAC;QAE7C,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpB,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,yBAAyB,CAAC,CAAC,GAAG,EAAE,CAAC;QACnD,CAAC;IACH,CAAC;IAED;;;;OAIG;IACH,eAAe;QACb,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;YACrB,OAAO;QACT,CAAC;QAED,kBAAkB;QAClB,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,yBAAyB,CAAC,CAAC,GAAG,EAAE,CAAC;QAEjD,4BAA4B;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE;aACjB,OAAO,CAAC,wCAAwC,CAAC;aACjD,GAAG,EAAmD,CAAC;QAE1D,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;YACvB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,MAAM,SAAS,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAC3D,GAAG,CAAC,SAAS,CAAC,UAAU,EACxB,GAAG,CAAC,SAAS,CAAC,UAAU,GAAG,GAAG,CAAC,SAAS,CAAC,UAAU,CACpD,CAAC,CAAC;gBACH,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;YAC1C,CAAC;QACH,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;CACF;AA3XD,kDA2XC;AAED;;GAEG;AACH,SAAgB,kBAAkB,CAChC,EAAqB,EACrB,SAAkB;IAElB,OAAO,IAAI,mBAAmB,CAAC,EAAE,EAAE,SAAS,CAAC,CAAC;AAChD,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mduenas/codegraph",
|
|
3
|
-
"version": "0.6.0",
|
|
3
|
+
"version": "0.6.1-beta.0",
|
|
4
4
|
"description": "Supercharge Claude Code with semantic code intelligence. 30% fewer tokens, 25% fewer tool calls, 100% local.",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -37,7 +37,7 @@
|
|
|
37
37
|
"better-sqlite3": "^11.0.0",
|
|
38
38
|
"commander": "^14.0.2",
|
|
39
39
|
"figlet": "^1.8.0",
|
|
40
|
-
"sqlite-
|
|
40
|
+
"sqlite-vec": "^0.1.6",
|
|
41
41
|
"tree-sitter": "^0.22.4",
|
|
42
42
|
"tree-sitter-c": "^0.23.4",
|
|
43
43
|
"tree-sitter-c-sharp": "^0.23.1",
|