@ninjaxtools/slopdex 0.18.0 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/.gitignore +2 -0
  2. package/README.md +136 -48
  3. package/binary-install.js +348 -0
  4. package/binary.js +124 -0
  5. package/install.js +4 -0
  6. package/npm-shrinkwrap.json +52 -0
  7. package/package.json +78 -62
  8. package/run-slopdex.js +4 -0
  9. package/.agents/skills/slopdex/SKILL.md +0 -241
  10. package/dist/chunk-6LHZCHEB.js +0 -1112
  11. package/dist/chunk-6LHZCHEB.js.map +0 -1
  12. package/dist/chunk-BXWO2KMC.js +0 -20
  13. package/dist/chunk-BXWO2KMC.js.map +0 -1
  14. package/dist/chunk-DKRB2XT5.js +0 -33
  15. package/dist/chunk-DKRB2XT5.js.map +0 -1
  16. package/dist/chunk-DTI7SXGL.js +0 -109
  17. package/dist/chunk-DTI7SXGL.js.map +0 -1
  18. package/dist/chunk-IQU3YVZ3.js +0 -85
  19. package/dist/chunk-IQU3YVZ3.js.map +0 -1
  20. package/dist/chunk-JAVPHCZH.js +0 -24
  21. package/dist/chunk-JAVPHCZH.js.map +0 -1
  22. package/dist/chunk-KQRS5P4U.js +0 -10
  23. package/dist/chunk-KQRS5P4U.js.map +0 -1
  24. package/dist/chunk-KUV6PKKL.js +0 -425
  25. package/dist/chunk-KUV6PKKL.js.map +0 -1
  26. package/dist/chunk-MKUTTXZB.js +0 -74
  27. package/dist/chunk-MKUTTXZB.js.map +0 -1
  28. package/dist/chunk-N6D66ASM.js +0 -2217
  29. package/dist/chunk-N6D66ASM.js.map +0 -1
  30. package/dist/chunk-OIKBO3NJ.js +0 -133
  31. package/dist/chunk-OIKBO3NJ.js.map +0 -1
  32. package/dist/chunk-S225GYCL.js +0 -79
  33. package/dist/chunk-S225GYCL.js.map +0 -1
  34. package/dist/chunk-TSURHRFF.js +0 -240
  35. package/dist/chunk-TSURHRFF.js.map +0 -1
  36. package/dist/chunk-VH5VGRCI.js +0 -103
  37. package/dist/chunk-VH5VGRCI.js.map +0 -1
  38. package/dist/cli.d.ts +0 -1
  39. package/dist/cli.js +0 -1130
  40. package/dist/cli.js.map +0 -1
  41. package/dist/code-index-YEFTNG7E.js +0 -14
  42. package/dist/code-index-YEFTNG7E.js.map +0 -1
  43. package/dist/cross-search-FVKDMGP7.js +0 -10
  44. package/dist/cross-search-FVKDMGP7.js.map +0 -1
  45. package/dist/database-KAPV2YMP.js +0 -14
  46. package/dist/database-KAPV2YMP.js.map +0 -1
  47. package/dist/hosted-GHIUHKOU.js +0 -13
  48. package/dist/hosted-GHIUHKOU.js.map +0 -1
  49. package/dist/index.d.ts +0 -681
  50. package/dist/index.js +0 -102
  51. package/dist/index.js.map +0 -1
  52. package/dist/jina-43RL7C6M.js +0 -12
  53. package/dist/jina-43RL7C6M.js.map +0 -1
  54. package/dist/openai-CYUPNBGB.js +0 -12
  55. package/dist/openai-CYUPNBGB.js.map +0 -1
  56. package/dist/openai-EC6THXKV.js +0 -21
  57. package/dist/openai-EC6THXKV.js.map +0 -1
  58. package/dist/openai-TCRXM66O.js +0 -11
  59. package/dist/openai-TCRXM66O.js.map +0 -1
  60. package/docs/implementation.md +0 -162
  61. package/docs/reference.md +0 -187
  62. package/scripts/install-opencode-skill.mjs +0 -11
@@ -1,1112 +0,0 @@
1
- import {
2
- CodeIndexError,
3
- IncompatibleIndexError
4
- } from "./chunk-DKRB2XT5.js";
5
-
6
- // src/storage/database.ts
7
- import { mkdirSync } from "node:fs";
8
- import path from "node:path";
9
- import { DatabaseSync } from "node:sqlite";
10
- import * as sqliteVec from "sqlite-vec";
11
- var SCHEMA_VERSION = "10";
12
- var KNN_TIE_OVERFETCH = 32;
13
- var VEC0_MAX_K = 4096;
14
- var VEC0_MAX_DIMENSIONS = 8192;
15
- var IndexDatabase = class {
16
- #db;
17
- #rootDir;
18
- #indexPath;
19
- #profile;
20
- #readOnly;
21
- constructor(indexPath, rootDir, profile, readOnly = false) {
22
- if (!readOnly) mkdirSync(path.dirname(indexPath), { recursive: true });
23
- this.#indexPath = indexPath;
24
- this.#rootDir = rootDir;
25
- this.#profile = profile;
26
- this.#readOnly = readOnly;
27
- this.#db = new DatabaseSync(indexPath, { allowExtension: true, readOnly });
28
- try {
29
- sqliteVec.load(this.#db);
30
- this.#db.enableLoadExtension(false);
31
- const regexes = /* @__PURE__ */ new Map();
32
- this.#db.function("slopdex_regexp", { deterministic: true }, (pattern, value) => {
33
- if (typeof pattern !== "string" || typeof value !== "string") return 0;
34
- let regex = regexes.get(pattern);
35
- if (!regex) {
36
- regex = new RegExp(pattern);
37
- regexes.set(pattern, regex);
38
- }
39
- return regex.test(value) ? 1 : 0;
40
- });
41
- if (readOnly) {
42
- this.#db.exec("PRAGMA foreign_keys=ON");
43
- } else {
44
- this.#db.exec("PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL; PRAGMA foreign_keys=ON;");
45
- this.#initializeSchema();
46
- }
47
- this.#validateMetadata(readOnly);
48
- } catch (error) {
49
- this.#db.close();
50
- throw error;
51
- }
52
- }
53
- close() {
54
- this.#db.close();
55
- }
56
- get isReadOnly() {
57
- return this.#readOnly;
58
- }
59
- #initializeSchema() {
60
- const existingVersion = this.#metadataTableExists() ? this.#metadata("schema_version") : null;
61
- if (existingVersion && existingVersion !== "6" && existingVersion !== "7" && existingVersion !== "8" && existingVersion !== "9" && existingVersion !== SCHEMA_VERSION) {
62
- throw new IncompatibleIndexError(`Unsupported index schema version ${existingVersion}.`);
63
- }
64
- this.#db.exec(`
65
- CREATE TABLE IF NOT EXISTS metadata (
66
- key TEXT PRIMARY KEY,
67
- value TEXT NOT NULL
68
- );
69
- CREATE TABLE IF NOT EXISTS files (
70
- path TEXT PRIMARY KEY,
71
- content_hash TEXT NOT NULL,
72
- blob_oid TEXT,
73
- source_mode TEXT NOT NULL CHECK(source_mode IN ('git', 'working-tree')),
74
- indexed_commit TEXT,
75
- previous_path TEXT,
76
- language TEXT NOT NULL,
77
- byte_size INTEGER NOT NULL,
78
- file_description_path TEXT,
79
- file_description_content_hash TEXT,
80
- file_description TEXT,
81
- file_description_embedding_id INTEGER REFERENCES embeddings(id)
82
- );
83
- CREATE TABLE IF NOT EXISTS embeddings (
84
- id INTEGER PRIMARY KEY,
85
- embedding_key TEXT NOT NULL UNIQUE,
86
- vector BLOB NOT NULL
87
- );
88
- CREATE TABLE IF NOT EXISTS description_cache (
89
- description_key TEXT PRIMARY KEY,
90
- description TEXT NOT NULL
91
- );
92
- CREATE TABLE IF NOT EXISTS parse_cache (
93
- parse_key TEXT PRIMARY KEY,
94
- result TEXT NOT NULL
95
- );
96
- CREATE TABLE IF NOT EXISTS functions (
97
- id INTEGER PRIMARY KEY,
98
- path TEXT NOT NULL REFERENCES files(path) ON DELETE CASCADE ON UPDATE CASCADE,
99
- language TEXT NOT NULL,
100
- kind TEXT NOT NULL,
101
- name TEXT NOT NULL,
102
- qualified_name TEXT NOT NULL,
103
- signature TEXT,
104
- identity_key TEXT NOT NULL UNIQUE,
105
- start_line INTEGER NOT NULL,
106
- start_column INTEGER NOT NULL,
107
- end_line INTEGER NOT NULL,
108
- end_column INTEGER NOT NULL,
109
- line_count INTEGER NOT NULL,
110
- source TEXT NOT NULL,
111
- source_hash TEXT NOT NULL,
112
- embedding_input TEXT NOT NULL,
113
- first_seen_commit TEXT,
114
- last_seen_commit TEXT,
115
- source_mode TEXT NOT NULL CHECK(source_mode IN ('git', 'working-tree')),
116
- embedding_id INTEGER NOT NULL REFERENCES embeddings(id),
117
- description TEXT,
118
- description_embedding_id INTEGER REFERENCES embeddings(id)
119
- );
120
- CREATE TABLE IF NOT EXISTS callable_provenance (
121
- identity_key TEXT NOT NULL,
122
- source_hash TEXT NOT NULL,
123
- first_seen_commit TEXT NOT NULL,
124
- PRIMARY KEY(identity_key, source_hash)
125
- );
126
- CREATE INDEX IF NOT EXISTS functions_path ON functions(path);
127
- CREATE INDEX IF NOT EXISTS functions_embedding ON functions(embedding_id);
128
- CREATE INDEX IF NOT EXISTS functions_description_embedding ON functions(description_embedding_id);
129
- CREATE TABLE IF NOT EXISTS indexing_errors (
130
- id INTEGER PRIMARY KEY,
131
- path TEXT NOT NULL REFERENCES files(path) ON DELETE CASCADE ON UPDATE CASCADE,
132
- diagnostic TEXT NOT NULL
133
- );
134
- CREATE INDEX IF NOT EXISTS indexing_errors_path ON indexing_errors(path);
135
- INSERT OR IGNORE INTO callable_provenance(identity_key, source_hash, first_seen_commit)
136
- SELECT identity_key, source_hash, first_seen_commit FROM functions WHERE first_seen_commit IS NOT NULL;
137
- `);
138
- if (existingVersion === "6" || existingVersion === "7" || existingVersion === null) {
139
- this.#db.exec("BEGIN IMMEDIATE");
140
- try {
141
- if (existingVersion === "6") {
142
- this.#db.exec(`
143
- ALTER TABLE files ADD COLUMN file_description_path TEXT;
144
- ALTER TABLE files ADD COLUMN file_description_content_hash TEXT;
145
- ALTER TABLE files ADD COLUMN file_description TEXT;
146
- ALTER TABLE files ADD COLUMN file_description_embedding_id INTEGER REFERENCES embeddings(id);
147
- `);
148
- }
149
- const storedProfile = this.#metadata("embedding_profile");
150
- const dimensions = storedProfile ? Number(JSON.parse(storedProfile).dimensions) : this.#profile.dimensions;
151
- if (!Number.isInteger(dimensions) || dimensions <= 0) throw new IncompatibleIndexError("Index has an invalid embedding profile.");
152
- if (dimensions <= VEC0_MAX_DIMENSIONS) this.#db.exec(functionVectorsSchema(dimensions));
153
- this.#setMetadata("schema_version", SCHEMA_VERSION);
154
- this.#db.exec("COMMIT");
155
- } catch (error) {
156
- this.#db.exec("ROLLBACK");
157
- throw error;
158
- }
159
- } else if (existingVersion === "8") {
160
- this.#db.exec("BEGIN IMMEDIATE");
161
- try {
162
- this.#db.exec(similarityCacheSchema());
163
- this.#setMetadata("schema_version", SCHEMA_VERSION);
164
- this.#db.exec("COMMIT");
165
- } catch (error) {
166
- this.#db.exec("ROLLBACK");
167
- throw error;
168
- }
169
- } else if (existingVersion === "9") {
170
- this.#db.exec("BEGIN IMMEDIATE");
171
- try {
172
- this.#db.exec(`
173
- ALTER TABLE similarity_cache_state ADD COLUMN floor REAL NOT NULL DEFAULT -1;
174
- ALTER TABLE similarity_cache_state ADD COLUMN stored_count INTEGER NOT NULL DEFAULT 0;
175
- ALTER TABLE similarity_cache_state ADD COLUMN complete INTEGER NOT NULL DEFAULT 0;
176
- UPDATE similarity_cache_state SET stored_count = (
177
- SELECT COUNT(*) FROM similarity_cache c
178
- WHERE c.source_id = similarity_cache_state.function_id
179
- AND c.similarity_mode = similarity_cache_state.similarity_mode
180
- );
181
- UPDATE similarity_cache_state SET complete = (stored_count < cached_width);
182
- `);
183
- this.#setMetadata("schema_version", SCHEMA_VERSION);
184
- this.#db.exec("COMMIT");
185
- } catch (error) {
186
- this.#db.exec("ROLLBACK");
187
- throw error;
188
- }
189
- }
190
- this.#db.exec("CREATE INDEX IF NOT EXISTS files_description_embedding ON files(file_description_embedding_id);");
191
- this.#db.exec(similarityCacheSchema());
192
- }
193
- #metadataTableExists() {
194
- return Boolean(this.#db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'metadata'").get());
195
- }
196
- #metadata(key) {
197
- if (!this.#metadataTableExists()) return null;
198
- const row = this.#db.prepare("SELECT value FROM metadata WHERE key = ?").get(key);
199
- return row?.value ?? null;
200
- }
201
- #setMetadata(key, value) {
202
- this.#db.prepare(`
203
- INSERT INTO metadata(key, value) VALUES (?, ?)
204
- ON CONFLICT(key) DO UPDATE SET value = excluded.value
205
- `).run(key, value);
206
- }
207
- #deleteMetadata(key) {
208
- this.#db.prepare("DELETE FROM metadata WHERE key = ?").run(key);
209
- }
210
- #validateMetadata(readOnly) {
211
- const schemaVersion = this.#metadata("schema_version");
212
- if (readOnly && !schemaVersion) throw new IncompatibleIndexError("Target file is not a slopdex index.");
213
- if (schemaVersion && schemaVersion !== SCHEMA_VERSION) {
214
- throw new IncompatibleIndexError(`Unsupported index schema version ${schemaVersion}.`);
215
- }
216
- const storedRoot = this.#metadata("root_dir");
217
- if (readOnly && !storedRoot) throw new IncompatibleIndexError("Target index has no repository identity.");
218
- if (storedRoot && path.resolve(storedRoot) !== this.#rootDir) {
219
- throw new IncompatibleIndexError(`Index belongs to ${storedRoot}, not ${this.#rootDir}.`);
220
- }
221
- const profileJson = JSON.stringify(this.#profile);
222
- const storedProfile = this.#metadata("embedding_profile");
223
- if (readOnly && !storedProfile) throw new IncompatibleIndexError("Target index has no embedding profile.");
224
- if (storedProfile && storedProfile !== profileJson) {
225
- throw new IncompatibleIndexError("Embedding provider, model, dimensions, or strategy differs from this index.");
226
- }
227
- if (readOnly) return;
228
- this.#transaction(() => {
229
- this.#setMetadata("schema_version", SCHEMA_VERSION);
230
- this.#setMetadata("root_dir", this.#rootDir);
231
- this.#setMetadata("embedding_profile", profileJson);
232
- if (!this.#metadata("generation")) this.#setMetadata("generation", "0");
233
- });
234
- }
235
- #transaction(operation) {
236
- this.#db.exec("BEGIN IMMEDIATE");
237
- try {
238
- const result = operation();
239
- this.#db.exec("COMMIT");
240
- return result;
241
- } catch (error) {
242
- this.#db.exec("ROLLBACK");
243
- throw error;
244
- }
245
- }
246
- getCheckpoint() {
247
- return this.#metadata("git_checkpoint");
248
- }
249
- getGeneration() {
250
- return Number(this.#metadata("generation") ?? 0);
251
- }
252
- descriptionsEnabled() {
253
- return this.#metadata("descriptions_enabled") === "true";
254
- }
255
- descriptionProfile() {
256
- const value = this.#metadata("description_profile");
257
- return value ? JSON.parse(value) : null;
258
- }
259
- cachedEmbedding(key) {
260
- const row = this.#db.prepare("SELECT vector FROM embeddings WHERE embedding_key = ?").get(key);
261
- return row ? bufferVector(row.vector) : void 0;
262
- }
263
- storeEmbedding(key, vector) {
264
- if (this.#readOnly) return;
265
- this.#db.prepare("INSERT OR IGNORE INTO embeddings(embedding_key, vector) VALUES (?, ?)").run(key, vectorBuffer(vector));
266
- }
267
- cachedDescription(key) {
268
- const row = this.#db.prepare("SELECT description FROM description_cache WHERE description_key = ?").get(key);
269
- return row?.description;
270
- }
271
- liveFunctionDescriptions(identityKeys) {
272
- const result = /* @__PURE__ */ new Map();
273
- for (let index = 0; index < identityKeys.length; index += 500) {
274
- const batch = identityKeys.slice(index, index + 500);
275
- if (batch.length === 0) continue;
276
- const rows = this.#db.prepare(`
277
- SELECT identity_key AS identityKey, description, source_hash AS sourceHash
278
- FROM functions WHERE identity_key IN (${batch.map(() => "?").join(", ")})
279
- `).all(...batch);
280
- for (const row of rows) result.set(row.identityKey, { description: row.description, sourceHash: row.sourceHash });
281
- }
282
- return result;
283
- }
284
- liveFileContentHashes(filePaths) {
285
- const result = /* @__PURE__ */ new Map();
286
- for (let index = 0; index < filePaths.length; index += 500) {
287
- const batch = filePaths.slice(index, index + 500);
288
- if (batch.length === 0) continue;
289
- const rows = this.#db.prepare(`
290
- SELECT path, content_hash AS contentHash FROM files WHERE path IN (${batch.map(() => "?").join(", ")})
291
- `).all(...batch);
292
- for (const row of rows) result.set(row.path, row.contentHash);
293
- }
294
- return result;
295
- }
296
- updateDescriptionProfile(profile, expectedGeneration) {
297
- this.#transaction(() => {
298
- if (this.getGeneration() !== expectedGeneration) throw new CodeIndexError("Index changed while descriptions were being prepared; retry the update.");
299
- this.#setMetadata("description_profile", JSON.stringify(profile));
300
- this.#setMetadata("generation", String(expectedGeneration + 1));
301
- });
302
- }
303
- storeDescription(key, description) {
304
- if (this.#readOnly) return description;
305
- this.#db.prepare("INSERT OR IGNORE INTO description_cache(description_key, description) VALUES (?, ?)").run(key, description);
306
- return this.#db.prepare("SELECT description FROM description_cache WHERE description_key = ?").get(key).description;
307
- }
308
- cachedParse(key) {
309
- const row = this.#db.prepare("SELECT result FROM parse_cache WHERE parse_key = ?").get(key);
310
- return row ? JSON.parse(row.result) : void 0;
311
- }
312
- storeParse(key, result) {
313
- if (this.#readOnly) return;
314
- this.#db.prepare("INSERT OR IGNORE INTO parse_cache(parse_key, result) VALUES (?, ?)").run(key, JSON.stringify(result));
315
- }
316
- #embeddingId(key, vector) {
317
- if (vector) this.storeEmbedding(key, vector);
318
- const row = this.#db.prepare("SELECT id FROM embeddings WHERE embedding_key = ?").get(key);
319
- if (!row) throw new CodeIndexError("Missing cached embedding.");
320
- return row.id;
321
- }
322
- enableDescriptions(values, files, profile, expectedGeneration) {
323
- this.#transaction(() => {
324
- if (this.getGeneration() !== expectedGeneration) throw new CodeIndexError("Index changed while descriptions were being prepared; retry the update.");
325
- for (const file of files) this.#updateFileDescription(file);
326
- for (const { id, description } of values) {
327
- this.#db.prepare("UPDATE functions SET description = ?, description_embedding_id = ? WHERE id = ?").run(description.description, this.#embeddingId(description.key, description.vector), id);
328
- }
329
- this.#setMetadata("descriptions_enabled", "true");
330
- this.#setMetadata("description_profile", JSON.stringify(profile));
331
- this.#setMetadata("generation", String(expectedGeneration + 1));
332
- });
333
- }
334
- updateDescriptions(files, includeCallables, expectedGeneration, profile) {
335
- this.#transaction(() => {
336
- if (this.getGeneration() !== expectedGeneration) throw new CodeIndexError("Index changed while descriptions were being prepared; retry the update.");
337
- if (profile) this.#setMetadata("description_profile", JSON.stringify(profile));
338
- if (files.length === 0) return;
339
- for (const file of files) {
340
- this.#updateFileDescription(file);
341
- this.#replaceCachedDescription(file.fileDescription.descriptionKey, file.fileDescription.value.description);
342
- if (!includeCallables) continue;
343
- for (const callable of file.callables) {
344
- if (!callable.description) throw new CodeIndexError(`Missing description for ${callable.qualifiedName}.`);
345
- if (!callable.descriptionKey) throw new CodeIndexError(`Missing description cache key for ${callable.qualifiedName}.`);
346
- this.#db.prepare("UPDATE functions SET description = ?, description_embedding_id = ? WHERE identity_key = ?").run(
347
- callable.description.description,
348
- this.#embeddingId(callable.description.key, callable.description.vector),
349
- callable.identityKey
350
- );
351
- this.#replaceCachedDescription(callable.descriptionKey, callable.description.description);
352
- }
353
- }
354
- this.#setMetadata("generation", String(expectedGeneration + 1));
355
- });
356
- }
357
- #updateFileDescription(file) {
358
- if (!file.fileDescription) throw new CodeIndexError(`Missing file description for ${file.path}.`);
359
- this.#db.prepare(`
360
- UPDATE files
361
- SET file_description_path = ?, file_description_content_hash = ?,
362
- file_description = ?, file_description_embedding_id = ?
363
- WHERE path = ?
364
- `).run(
365
- file.fileDescription.path,
366
- file.fileDescription.contentHash,
367
- file.fileDescription.value.description,
368
- this.#embeddingId(file.fileDescription.value.key, file.fileDescription.value.vector),
369
- file.path
370
- );
371
- }
372
- #replaceCachedDescription(key, description) {
373
- this.#db.prepare(`
374
- INSERT INTO description_cache(description_key, description) VALUES (?, ?)
375
- ON CONFLICT(description_key) DO UPDATE SET description = excluded.description
376
- `).run(key, description);
377
- }
378
- disableDescriptions() {
379
- if (!this.descriptionsEnabled()) return;
380
- this.#transaction(() => {
381
- this.#setMetadata("descriptions_enabled", "false");
382
- this.#setMetadata("generation", String(this.getGeneration() + 1));
383
- });
384
- }
385
- getWorkingTreeFiles() {
386
- return this.#db.prepare(`
387
- SELECT path, previous_path AS previousPath
388
- FROM files WHERE source_mode = 'working-tree' ORDER BY path
389
- `).all();
390
- }
391
- getFileStates() {
392
- return this.#db.prepare(`
393
- SELECT path, content_hash AS contentHash, blob_oid AS blobOid,
394
- source_mode AS sourceMode, previous_path AS previousPath, language,
395
- NOT EXISTS (
396
- SELECT 1 FROM indexing_errors ie
397
- WHERE ie.path = files.path
398
- AND json_extract(ie.diagnostic, '$.code') IN ('read-error', 'file-too-large')
399
- ) AS describable,
400
- file_description AS fileDescription,
401
- file_description_path AS fileDescriptionPath,
402
- file_description_content_hash AS fileDescriptionContentHash
403
- FROM files ORDER BY path
404
- `).all();
405
- }
406
- fileDescription(filePath) {
407
- const row = this.#db.prepare(`
408
- SELECT file_description AS description, file_description_path AS path,
409
- file_description_content_hash AS contentHash
410
- FROM files WHERE path = ? AND file_description IS NOT NULL
411
- `).get(filePath);
412
- return row;
413
- }
414
- functionDescription(identityKey) {
415
- const row = this.#db.prepare("SELECT description FROM functions WHERE identity_key = ?").get(identityKey);
416
- return row?.description ?? void 0;
417
- }
418
- applyUpdate(options) {
419
- return this.#transaction(() => {
420
- if (options.expectedCheckpoint !== void 0 && this.getCheckpoint() !== options.expectedCheckpoint) {
421
- throw new CodeIndexError("Git checkpoint changed while the update was being prepared.");
422
- }
423
- if (options.expectedGeneration !== void 0 && this.getGeneration() !== options.expectedGeneration) {
424
- throw new CodeIndexError("Index changed while the update was being prepared; retry the update.");
425
- }
426
- const stats = {
427
- filesUpdated: 0,
428
- filesDeleted: 0,
429
- functionsAdded: 0,
430
- functionsUpdated: 0,
431
- functionsDeleted: 0,
432
- embeddingsCreated: options.embeddingsCreated ?? 0,
433
- checkpoint: options.checkpoint !== void 0 ? options.checkpoint : this.getCheckpoint()
434
- };
435
- const deleteFile = this.#db.prepare("DELETE FROM files WHERE path = ?");
436
- const countFunctions = this.#db.prepare("SELECT COUNT(*) AS count FROM functions WHERE path = ?");
437
- const applyStage = (files, deletePaths) => {
438
- stats.filesUpdated += files.length;
439
- const uniqueDeletes = new Set(deletePaths);
440
- for (const file of files) {
441
- uniqueDeletes.delete(file.path);
442
- if (file.previousPath) uniqueDeletes.delete(file.previousPath);
443
- if (file.replacePath) uniqueDeletes.delete(file.replacePath);
444
- }
445
- for (const filePath of uniqueDeletes) {
446
- const count = Number(countFunctions.get(filePath).count);
447
- const result = deleteFile.run(filePath);
448
- if (result.changes > 0) {
449
- stats.filesDeleted += 1;
450
- stats.functionsDeleted += count;
451
- }
452
- }
453
- for (const file of files) this.#replaceFile(file, stats);
454
- };
455
- applyStage(options.files, options.deletePaths);
456
- if (options.workingTree) applyStage(options.workingTree.files, options.workingTree.deletePaths);
457
- const generation = this.getGeneration() + 1;
458
- this.#setMetadata("generation", String(generation));
459
- if (options.completeDiagnosticsScan) this.#deleteMetadata("diagnostics_scan_pending");
460
- if (options.checkpoint === null) this.#deleteMetadata("git_checkpoint");
461
- else if (options.checkpoint !== void 0) this.#setMetadata("git_checkpoint", options.checkpoint);
462
- return stats;
463
- });
464
- }
465
- #replaceFile(file, stats) {
466
- const sourcePath = file.replacePath ?? file.previousPath ?? file.path;
467
- const oldRows = this.#rowsForPath(sourcePath);
468
- const existingFile = this.#db.prepare(`
469
- SELECT previous_path, file_description_path, file_description_content_hash,
470
- file_description, file_description_embedding_id
471
- FROM files WHERE path = ?
472
- `).get(sourcePath);
473
- const oldMatches = reconcileFunctions(file.callables, oldRows);
474
- if (sourcePath !== file.path) {
475
- const displacedRows = this.#rowsForPath(file.path);
476
- const displaced = this.#db.prepare("DELETE FROM files WHERE path = ?").run(file.path);
477
- if (displaced.changes > 0) {
478
- stats.filesDeleted += 1;
479
- stats.functionsDeleted += displacedRows.length;
480
- }
481
- this.#db.prepare("DELETE FROM files WHERE path = ?").run(sourcePath);
482
- } else {
483
- this.#db.prepare("DELETE FROM files WHERE path = ?").run(file.path);
484
- }
485
- this.#db.prepare(`
486
- INSERT INTO files(
487
- path, content_hash, blob_oid, source_mode, indexed_commit, previous_path, language, byte_size,
488
- file_description_path, file_description_content_hash, file_description, file_description_embedding_id
489
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
490
- `).run(
491
- file.path,
492
- file.contentHash,
493
- file.blobOid,
494
- file.sourceMode,
495
- file.indexedCommit,
496
- file.sourceMode === "working-tree" ? file.previousPath ?? existingFile?.previous_path ?? null : null,
497
- file.language,
498
- file.byteSize,
499
- file.fileDescription?.path ?? (file.unavailable ? null : existingFile?.file_description_path) ?? null,
500
- file.fileDescription?.contentHash ?? (file.unavailable ? null : existingFile?.file_description_content_hash) ?? null,
501
- file.fileDescription?.value.description ?? (file.unavailable ? null : existingFile?.file_description) ?? null,
502
- file.fileDescription ? this.#embeddingId(file.fileDescription.value.key, file.fileDescription.value.vector) : file.unavailable ? null : existingFile?.file_description_embedding_id ?? null
503
- );
504
- const insertError = this.#db.prepare("INSERT INTO indexing_errors(path, diagnostic) VALUES (?, ?)");
505
- for (const error of file.errors) insertError.run(file.path, JSON.stringify(error));
506
- const usedIds = /* @__PURE__ */ new Set();
507
- const orderedCallables = [
508
- ...file.callables.filter((callable) => oldMatches.has(callable)),
509
- ...file.callables.filter((callable) => !oldMatches.has(callable))
510
- ];
511
- for (const callable of orderedCallables) {
512
- if (callable.vector) {
513
- this.storeEmbedding(callable.embeddingKey, callable.vector);
514
- }
515
- const embedding = this.#db.prepare("SELECT id FROM embeddings WHERE embedding_key = ?").get(callable.embeddingKey);
516
- if (!embedding) throw new CodeIndexError(`Missing embedding for ${callable.qualifiedName}.`);
517
- const old = oldMatches.get(callable);
518
- if (this.descriptionsEnabled() && !callable.description) throw new CodeIndexError(`Missing description for ${callable.qualifiedName}.`);
519
- const descriptionId = callable.description ? this.#embeddingId(callable.description.key, callable.description.vector) : null;
520
- if (old) usedIds.add(old.id);
521
- const remembered = this.#db.prepare(`
522
- SELECT first_seen_commit FROM callable_provenance WHERE identity_key = ? AND source_hash = ?
523
- `).get(callable.identityKey, callable.sourceHash);
524
- const firstSeenCommit = old?.first_seen_commit ?? remembered?.first_seen_commit ?? file.indexedCommit;
525
- this.#db.prepare(`
526
- INSERT INTO functions(
527
- id, path, language, kind, name, qualified_name, signature, identity_key,
528
- start_line, start_column, end_line, end_column, line_count, source, source_hash,
529
- embedding_input, first_seen_commit, last_seen_commit, source_mode, embedding_id, description, description_embedding_id
530
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
531
- `).run(
532
- old?.id ?? null,
533
- file.path,
534
- callable.language,
535
- callable.kind,
536
- callable.name,
537
- callable.qualifiedName,
538
- callable.signature,
539
- callable.identityKey,
540
- callable.startLine,
541
- callable.startColumn,
542
- callable.endLine,
543
- callable.endColumn,
544
- callable.lineCount,
545
- callable.source,
546
- callable.sourceHash,
547
- callable.embeddingInput,
548
- firstSeenCommit,
549
- file.indexedCommit,
550
- file.sourceMode,
551
- embedding.id,
552
- callable.description?.description ?? null,
553
- descriptionId
554
- );
555
- if (old) stats.functionsUpdated += 1;
556
- else stats.functionsAdded += 1;
557
- if (firstSeenCommit) {
558
- this.#db.prepare(`
559
- INSERT OR IGNORE INTO callable_provenance(identity_key, source_hash, first_seen_commit) VALUES (?, ?, ?)
560
- `).run(callable.identityKey, callable.sourceHash, firstSeenCommit);
561
- }
562
- }
563
- stats.functionsDeleted += oldRows.length - usedIds.size;
564
- }
565
- #rowsForPath(filePath) {
566
- return this.#db.prepare("SELECT * FROM functions WHERE path = ? ORDER BY start_line, start_column, id").all(filePath);
567
- }
568
- allFunctions() {
569
- return this.#db.prepare("SELECT * FROM functions ORDER BY path, start_line, start_column, id").all().map(toIndexedFunction);
570
- }
571
- allFilePaths() {
572
- return this.#db.prepare("SELECT path FROM files ORDER BY path").all().map((row) => row.path);
573
- }
574
- previousPath(filePath) {
575
- const row = this.#db.prepare("SELECT previous_path FROM files WHERE path = ?").get(filePath);
576
- return row?.previous_path ?? null;
577
- }
578
- functionsForPaths(paths) {
579
- const results = [];
580
- for (const filePath of paths) results.push(...this.#rowsForPath(filePath).map(toIndexedFunction));
581
- return results;
582
- }
583
- searchVector(vector, options) {
584
- const functionDescription = options.descriptionVector !== void 0;
585
- const fileDescription = options.fileDescriptionVector !== void 0;
586
- const fused = functionDescription || fileDescription;
587
- if (functionDescription && options.descriptions) throw new CodeIndexError("Description-only search cannot also supply a separate function-description vector.");
588
- const scoreCount = 1 + Number(functionDescription) + Number(fileDescription);
589
- const scoreExpression = ["base_similarity"].concat(functionDescription ? ["description_similarity"] : []).concat(fileDescription ? ["file_description_similarity"] : []).join(" + ");
590
- const excludePaths = options.excludePaths ?? [];
591
- if (options.limit !== void 0 && !options.descriptions && !fused && options.maxSimilarity === void 0 && options.nameRegex === void 0 && excludePaths.length <= 1 && options.limit <= VEC0_MAX_K && this.#profile.dimensions <= VEC0_MAX_DIMENSIONS) {
592
- const excludeIdFilter = options.excludeId === void 0 ? "" : "AND function_id_filter != ?";
593
- const excludePathFilter = excludePaths.length === 0 ? "" : "AND path != ?";
594
- const candidateLimit = Math.min(options.limit + KNN_TIE_OVERFETCH, VEC0_MAX_K);
595
- const rows2 = this.#db.prepare(`
596
- WITH nearest AS MATERIALIZED (
597
- SELECT function_id, distance
598
- FROM function_vectors
599
- WHERE embedding MATCH ? AND k = ?
600
- AND line_count >= ?
601
- ${excludeIdFilter}
602
- ${excludePathFilter}
603
- )
604
- SELECT f.*, 1.0 - nearest.distance AS similarity, 1.0 - nearest.distance AS base_similarity
605
- FROM nearest JOIN functions f ON f.id = nearest.function_id
606
- WHERE 1.0 - nearest.distance >= ?
607
- ORDER BY similarity DESC, f.id ASC
608
- `).all(
609
- vectorBuffer(vector),
610
- candidateLimit,
611
- options.minLines ?? 1,
612
- ...options.excludeId === void 0 ? [] : [options.excludeId],
613
- ...excludePaths,
614
- options.minSimilarity
615
- );
616
- const limit = options.limit;
617
- const boundary = rows2[limit - 1];
618
- const last = rows2.at(-1);
619
- const ambiguousTie = rows2.length === candidateLimit && boundary && last && Math.abs(boundary.similarity - last.similarity) <= Number.EPSILON;
620
- if (!ambiguousTie) {
621
- return rows2.slice(0, limit).map((row) => ({ function: toIndexedFunction(row), similarity: row.similarity }));
622
- }
623
- }
624
- const pathFilter = excludePaths.length > 0 ? `AND f.path NOT IN (${excludePaths.map(() => "?").join(", ")})` : "";
625
- const rows = this.#db.prepare(`
626
- WITH scores AS (
627
- SELECT f.id AS function_id, 1.0 - vec_distance_cosine(e.vector, ?) AS base_similarity
628
- ${functionDescription ? ", 1.0 - vec_distance_cosine(d.vector, ?) AS description_similarity" : ""}
629
- ${fileDescription ? ", 1.0 - vec_distance_cosine(fd.vector, ?) AS file_description_similarity" : ""}
630
- FROM functions f
631
- JOIN embeddings e ON e.id = f.${options.descriptions ? "description_embedding_id" : "embedding_id"}
632
- ${functionDescription ? "JOIN embeddings d ON d.id = f.description_embedding_id" : ""}
633
- ${fileDescription ? "JOIN files described_file ON described_file.path = f.path JOIN embeddings fd ON fd.id = described_file.file_description_embedding_id" : ""}
634
- WHERE (? IS NULL OR f.id != ?)
635
- ${pathFilter}
636
- AND f.line_count >= ?
637
- AND (? IS NULL OR slopdex_regexp(?, f.qualified_name))
638
- ), ranked AS (
639
- SELECT *, ${fused ? `(${scoreExpression}) / ${scoreCount}.0` : "base_similarity"} AS similarity
640
- FROM scores
641
- ), selected AS (
642
- SELECT * FROM ranked
643
- WHERE similarity >= ? AND (? IS NULL OR similarity < ?)
644
- ORDER BY similarity DESC, function_id ASC
645
- LIMIT ?
646
- )
647
- SELECT f.*, selected.similarity, selected.base_similarity
648
- ${functionDescription ? ", selected.description_similarity" : ""}
649
- ${fileDescription ? ", selected.file_description_similarity" : ""}
650
- FROM selected JOIN functions f ON f.id = selected.function_id
651
- ORDER BY selected.similarity DESC, f.id ASC
652
- `).all(
653
- vectorBuffer(vector),
654
- ...options.descriptionVector ? [vectorBuffer(options.descriptionVector)] : [],
655
- ...options.fileDescriptionVector ? [vectorBuffer(options.fileDescriptionVector)] : [],
656
- options.excludeId ?? null,
657
- options.excludeId ?? null,
658
- ...excludePaths,
659
- options.minLines ?? 1,
660
- options.nameRegex ?? null,
661
- options.nameRegex ?? null,
662
- options.minSimilarity,
663
- options.maxSimilarity ?? null,
664
- options.maxSimilarity ?? null,
665
- options.limit ?? -1
666
- );
667
- return rows.map((row) => ({
668
- function: toIndexedFunction(row),
669
- similarity: row.similarity,
670
- ...fused && !options.descriptions ? { codeSimilarity: row.base_similarity } : {},
671
- ...fused ? { descriptionSimilarity: options.descriptions ? row.base_similarity : row.description_similarity } : {},
672
- ...fileDescription ? { fileDescriptionSimilarity: row.file_description_similarity } : {}
673
- }));
674
- }
675
- vectorForFunction(id, kind = "code") {
676
- const row = this.#db.prepare(`
677
- SELECT e.vector FROM functions f
678
- JOIN embeddings e ON e.id = f.${kind === "description" ? "description_embedding_id" : "embedding_id"} WHERE f.id = ?
679
- `).get(id);
680
- if (!row) throw new CodeIndexError(`Function ${id} does not exist or has no ${kind} embedding.`);
681
- return bufferVector(row.vector);
682
- }
683
- vectorForFile(filePath) {
684
- const row = this.#db.prepare(`
685
- SELECT e.vector FROM files f
686
- JOIN embeddings e ON e.id = f.file_description_embedding_id WHERE f.path = ?
687
- `).get(filePath);
688
- if (!row) throw new CodeIndexError(`File ${filePath} does not exist or has no description embedding.`);
689
- return bufferVector(row.vector);
690
- }
691
- fileVectorForFunction(id) {
692
- const row = this.#db.prepare(`
693
- SELECT e.vector FROM functions fn
694
- JOIN files f ON f.path = fn.path
695
- JOIN embeddings e ON e.id = f.file_description_embedding_id
696
- WHERE fn.id = ?
697
- `).get(id);
698
- if (!row) throw new CodeIndexError(`Function ${id} does not exist or its file has no description embedding.`);
699
- return bufferVector(row.vector);
700
- }
701
- similarityCacheTriples() {
702
- return this.#db.prepare(`
703
- SELECT f.id AS functionId, f.embedding_id AS codeEmbeddingId,
704
- f.description_embedding_id AS descriptionEmbeddingId,
705
- files.file_description_embedding_id AS fileDescriptionEmbeddingId
706
- FROM functions f JOIN files ON files.path = f.path
707
- ORDER BY f.id
708
- `).all();
709
- }
710
- similarityCacheStates(mode) {
711
- const rows = this.#db.prepare(`
712
- SELECT function_id AS functionId, code_embedding_id AS codeEmbeddingId,
713
- description_embedding_id AS descriptionEmbeddingId,
714
- file_description_embedding_id AS fileDescriptionEmbeddingId,
715
- cached_width AS cachedWidth, generation, floor,
716
- stored_count AS storedCount, complete
717
- FROM similarity_cache_state WHERE similarity_mode = ?
718
- `).all(mode);
719
- return new Map(rows.map((row) => [row.functionId, {
720
- codeEmbeddingId: row.codeEmbeddingId,
721
- descriptionEmbeddingId: row.descriptionEmbeddingId,
722
- fileDescriptionEmbeddingId: row.fileDescriptionEmbeddingId,
723
- cachedWidth: row.cachedWidth,
724
- generation: row.generation,
725
- floor: row.floor,
726
- storedCount: Number(row.storedCount),
727
- complete: row.complete !== 0
728
- }]));
729
- }
730
- cachedSimilarityNeighbors(sourceId, mode, filter) {
731
- const excludePaths = filter?.excludePaths ?? [];
732
- const pathFilter = excludePaths.length > 0 ? `AND f.path NOT IN (${excludePaths.map(() => "?").join(", ")})` : "";
733
- const rows = this.#db.prepare(`
734
- SELECT f.*, c.similarity, c.similarity AS base_similarity,
735
- c.code_similarity AS code_similarity,
736
- c.description_similarity AS description_similarity,
737
- c.file_description_similarity AS file_description_similarity
738
- FROM similarity_cache c JOIN functions f ON f.id = c.target_id
739
- WHERE c.source_id = ? AND c.similarity_mode = ?
740
- AND c.similarity >= ?
741
- AND (? IS NULL OR c.similarity < ?)
742
- AND f.line_count >= ?
743
- AND (? IS NULL OR slopdex_regexp(?, f.qualified_name))
744
- ${pathFilter}
745
- ORDER BY c.similarity DESC, f.id ASC
746
- ${filter ? "LIMIT ?" : ""}
747
- `).all(
748
- sourceId,
749
- mode,
750
- filter?.minSimilarity ?? -1,
751
- filter?.maxSimilarity ?? null,
752
- filter?.maxSimilarity ?? null,
753
- filter?.minLines ?? 1,
754
- filter?.nameRegex ?? null,
755
- filter?.nameRegex ?? null,
756
- ...excludePaths,
757
- ...filter ? [filter.limit] : []
758
- );
759
- const fused = mode !== "code";
760
- return rows.map((row) => ({
761
- function: toIndexedFunction(row),
762
- similarity: row.similarity,
763
- ...fused && row.code_similarity !== null ? { codeSimilarity: row.code_similarity } : {},
764
- ...fused && row.description_similarity !== null ? { descriptionSimilarity: row.description_similarity } : {},
765
- ...fused && row.file_description_similarity !== null ? { fileDescriptionSimilarity: row.file_description_similarity } : {}
766
- }));
767
- }
768
- storeSimilarityNeighbors(sourceId, mode, neighbors, state) {
769
- if (this.#readOnly) return;
770
- this.#transaction(() => {
771
- this.#db.prepare("DELETE FROM similarity_cache WHERE source_id = ? AND similarity_mode = ?").run(sourceId, mode);
772
- const insert = this.#db.prepare(`
773
- INSERT INTO similarity_cache(
774
- source_id, target_id, similarity_mode, similarity,
775
- code_similarity, description_similarity, file_description_similarity
776
- ) VALUES (?, ?, ?, ?, ?, ?, ?)
777
- `);
778
- for (const neighbor of neighbors) {
779
- insert.run(
780
- sourceId,
781
- neighbor.function.id,
782
- mode,
783
- neighbor.similarity,
784
- neighbor.codeSimilarity ?? null,
785
- neighbor.descriptionSimilarity ?? null,
786
- neighbor.fileDescriptionSimilarity ?? null
787
- );
788
- }
789
- this.#db.prepare(`
790
- INSERT INTO similarity_cache_state(
791
- function_id, code_embedding_id, description_embedding_id,
792
- file_description_embedding_id, similarity_mode, cached_width, generation,
793
- floor, stored_count, complete
794
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
795
- ON CONFLICT(function_id, similarity_mode) DO UPDATE SET
796
- code_embedding_id = excluded.code_embedding_id,
797
- description_embedding_id = excluded.description_embedding_id,
798
- file_description_embedding_id = excluded.file_description_embedding_id,
799
- cached_width = excluded.cached_width,
800
- generation = excluded.generation,
801
- floor = excluded.floor,
802
- stored_count = excluded.stored_count,
803
- complete = excluded.complete
804
- `).run(
805
- sourceId,
806
- state.codeEmbeddingId,
807
- state.descriptionEmbeddingId,
808
- state.fileDescriptionEmbeddingId,
809
- mode,
810
- state.cachedWidth,
811
- state.generation,
812
- state.floor,
813
- neighbors.length,
814
- state.complete ? 1 : 0
815
- );
816
- });
817
- }
818
- similarityCacheInfo() {
819
- const sources = this.#db.prepare("SELECT COUNT(*) AS count FROM similarity_cache_state").get();
820
- const pairs = this.#db.prepare("SELECT COUNT(*) AS count FROM similarity_cache").get();
821
- return { cachedSources: Number(sources.count), cachedPairs: Number(pairs.count) };
822
- }
823
- similarityCacheCounts(mode) {
824
- const rows = this.#db.prepare(`
825
- SELECT source_id AS sourceId, COUNT(*) AS count
826
- FROM similarity_cache WHERE similarity_mode = ? GROUP BY source_id
827
- `).all(mode);
828
- return new Map(rows.map((row) => [row.sourceId, Number(row.count)]));
829
- }
830
- touchSimilarityCacheState(functionId, mode, state) {
831
- if (this.#readOnly) return;
832
- this.#db.prepare(`
833
- INSERT INTO similarity_cache_state(
834
- function_id, code_embedding_id, description_embedding_id,
835
- file_description_embedding_id, similarity_mode, cached_width, generation,
836
- floor, stored_count, complete
837
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
838
- ON CONFLICT(function_id, similarity_mode) DO UPDATE SET
839
- code_embedding_id = excluded.code_embedding_id,
840
- description_embedding_id = excluded.description_embedding_id,
841
- file_description_embedding_id = excluded.file_description_embedding_id,
842
- cached_width = excluded.cached_width,
843
- generation = excluded.generation,
844
- floor = excluded.floor,
845
- stored_count = excluded.stored_count,
846
- complete = excluded.complete
847
- `).run(
848
- functionId,
849
- state.codeEmbeddingId,
850
- state.descriptionEmbeddingId,
851
- state.fileDescriptionEmbeddingId,
852
- mode,
853
- state.cachedWidth,
854
- state.generation,
855
- state.floor,
856
- state.storedCount,
857
- state.complete ? 1 : 0
858
- );
859
- }
860
- status() {
861
- const functionCount = Number(this.#db.prepare("SELECT COUNT(*) AS count FROM functions").get().count);
862
- const fileCount = Number(this.#db.prepare("SELECT COUNT(*) AS count FROM files").get().count);
863
- const describableFileCount = Number(this.#db.prepare(`
864
- SELECT COUNT(*) AS count FROM files f
865
- WHERE NOT EXISTS (
866
- SELECT 1 FROM indexing_errors ie
867
- WHERE ie.path = f.path
868
- AND json_extract(ie.diagnostic, '$.code') IN ('read-error', 'file-too-large')
869
- )
870
- `).get().count);
871
- return {
872
- rootDir: this.#rootDir,
873
- indexPath: this.#indexPath,
874
- functionCount,
875
- fileCount,
876
- generation: this.getGeneration(),
877
- gitCheckpoint: this.getCheckpoint(),
878
- embeddingProfile: this.#profile,
879
- descriptionsEnabled: this.descriptionsEnabled(),
880
- descriptionCount: Number(this.#db.prepare("SELECT COUNT(*) AS count FROM functions WHERE description_embedding_id IS NOT NULL").get().count),
881
- fileDescriptionCount: Number(this.#db.prepare("SELECT COUNT(*) AS count FROM files WHERE file_description_embedding_id IS NOT NULL").get().count),
882
- describableFileCount,
883
- staleFileDescriptionCount: Number(this.#db.prepare(`
884
- SELECT COUNT(*) AS count FROM files f
885
- WHERE (file_description_path IS NULL OR file_description_path != path
886
- OR file_description_content_hash IS NULL OR file_description_content_hash != content_hash)
887
- AND NOT EXISTS (
888
- SELECT 1 FROM indexing_errors ie
889
- WHERE ie.path = f.path
890
- AND json_extract(ie.diagnostic, '$.code') IN ('read-error', 'file-too-large')
891
- )
892
- `).get().count),
893
- descriptionProfile: this.descriptionProfile(),
894
- indexingErrorCount: Number(this.#db.prepare("SELECT COUNT(*) AS count FROM indexing_errors").get().count),
895
- failedFileCount: this.filesWithErrors().length
896
- };
897
- }
898
- indexErrors() {
899
- return errorsFromDatabase(this.#db);
900
- }
901
- filesWithErrors() {
902
- return this.#db.prepare("SELECT DISTINCT path FROM indexing_errors").all().map((row) => row.path);
903
- }
904
- needsDiagnosticsScan() {
905
- return this.#metadata("diagnostics_scan_pending") === "true";
906
- }
907
- };
908
- function readIndexErrors(indexPath) {
909
- const database = new DatabaseSync(indexPath, { readOnly: true });
910
- try {
911
- if (!database.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'indexing_errors'").get()) return [];
912
- return errorsFromDatabase(database);
913
- } finally {
914
- database.close();
915
- }
916
- }
917
- function resetIndexState(indexPath, rootDir, profile) {
918
- const database = new DatabaseSync(indexPath, { allowExtension: true });
919
- try {
920
- sqliteVec.load(database);
921
- database.enableLoadExtension(false);
922
- database.exec("PRAGMA journal_mode=WAL; PRAGMA synchronous=FULL; PRAGMA foreign_keys=ON;");
923
- const metadata = new Map(database.prepare("SELECT key, value FROM metadata").all().map((row) => [row.key, row.value]));
924
- if (metadata.get("schema_version") !== SCHEMA_VERSION) {
925
- throw new IncompatibleIndexError(`Unsupported index schema version ${metadata.get("schema_version") ?? "unknown"}.`);
926
- }
927
- if (path.resolve(metadata.get("root_dir") ?? "") !== path.resolve(rootDir)) {
928
- throw new IncompatibleIndexError("Index belongs to a different repository.");
929
- }
930
- if (!Number.isInteger(profile.dimensions) || profile.dimensions <= 0) {
931
- throw new CodeIndexError("Embedding dimensions must be a positive integer.");
932
- }
933
- database.exec("BEGIN IMMEDIATE");
934
- try {
935
- database.exec(`
936
- DELETE FROM files;
937
- DROP TRIGGER IF EXISTS functions_vector_insert;
938
- DROP TRIGGER IF EXISTS functions_vector_delete;
939
- DROP TRIGGER IF EXISTS functions_vector_update;
940
- DROP TABLE IF EXISTS function_vectors;
941
- DELETE FROM callable_provenance;
942
- DELETE FROM metadata;
943
- `);
944
- if (profile.dimensions <= VEC0_MAX_DIMENSIONS) database.exec(functionVectorsSchema(profile.dimensions));
945
- const setMetadata = database.prepare("INSERT INTO metadata(key, value) VALUES (?, ?)");
946
- setMetadata.run("schema_version", SCHEMA_VERSION);
947
- setMetadata.run("root_dir", path.resolve(rootDir));
948
- setMetadata.run("embedding_profile", JSON.stringify({
949
- provider: profile.provider,
950
- model: profile.model,
951
- dimensions: profile.dimensions,
952
- strategyVersion: profile.strategyVersion ?? "callable-v2"
953
- }));
954
- setMetadata.run("generation", "0");
955
- database.exec("COMMIT");
956
- } catch (error) {
957
- database.exec("ROLLBACK");
958
- throw error;
959
- }
960
- } finally {
961
- database.close();
962
- }
963
- }
964
- function readIndexErrorCounts(indexPath) {
965
- const database = new DatabaseSync(indexPath, { readOnly: true });
966
- try {
967
- if (!database.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'indexing_errors'").get()) {
968
- return { errors: 0, files: 0 };
969
- }
970
- const counts = database.prepare("SELECT COUNT(*) AS errors, COUNT(DISTINCT path) AS files FROM indexing_errors").get();
971
- return { errors: Number(counts.errors), files: Number(counts.files) };
972
- } finally {
973
- database.close();
974
- }
975
- }
976
- function errorsFromDatabase(database) {
977
- const rows = database.prepare(`
978
- SELECT e.id, e.diagnostic, f.source_mode, f.indexed_commit
979
- FROM indexing_errors e JOIN files f ON f.path = e.path
980
- ORDER BY e.path, e.id
981
- `).all();
982
- return rows.map((row) => ({
983
- ...JSON.parse(row.diagnostic),
984
- id: row.id,
985
- sourceMode: row.source_mode,
986
- indexedCommit: row.indexed_commit
987
- }));
988
- }
989
- function similarityCacheSchema() {
990
- return `
991
- CREATE TABLE IF NOT EXISTS similarity_cache (
992
- source_id INTEGER NOT NULL REFERENCES functions(id) ON DELETE CASCADE,
993
- target_id INTEGER NOT NULL REFERENCES functions(id) ON DELETE CASCADE,
994
- similarity_mode TEXT NOT NULL,
995
- similarity REAL NOT NULL,
996
- code_similarity REAL,
997
- description_similarity REAL,
998
- file_description_similarity REAL,
999
- PRIMARY KEY (source_id, target_id, similarity_mode)
1000
- );
1001
- CREATE INDEX IF NOT EXISTS similarity_cache_source ON similarity_cache(source_id, similarity_mode, similarity DESC);
1002
- CREATE INDEX IF NOT EXISTS similarity_cache_target ON similarity_cache(target_id, similarity_mode);
1003
- CREATE TABLE IF NOT EXISTS similarity_cache_state (
1004
- function_id INTEGER NOT NULL REFERENCES functions(id) ON DELETE CASCADE,
1005
- similarity_mode TEXT NOT NULL,
1006
- code_embedding_id INTEGER NOT NULL,
1007
- description_embedding_id INTEGER,
1008
- file_description_embedding_id INTEGER,
1009
- cached_width INTEGER NOT NULL,
1010
- generation INTEGER NOT NULL,
1011
- floor REAL NOT NULL DEFAULT -1,
1012
- stored_count INTEGER NOT NULL DEFAULT 0,
1013
- complete INTEGER NOT NULL DEFAULT 0,
1014
- PRIMARY KEY (function_id, similarity_mode)
1015
- );
1016
- `;
1017
- }
1018
- function functionVectorsSchema(dimensions) {
1019
- return `
1020
- CREATE VIRTUAL TABLE function_vectors USING vec0(
1021
- function_id INTEGER PRIMARY KEY,
1022
- embedding FLOAT[${dimensions}] distance_metric=cosine,
1023
- line_count INTEGER,
1024
- path TEXT,
1025
- function_id_filter INTEGER
1026
- );
1027
- INSERT INTO function_vectors(function_id, embedding, line_count, path, function_id_filter)
1028
- SELECT f.id, e.vector, f.line_count, f.path, f.id
1029
- FROM functions f JOIN embeddings e ON e.id = f.embedding_id;
1030
- CREATE TRIGGER functions_vector_insert AFTER INSERT ON functions BEGIN
1031
- INSERT INTO function_vectors(function_id, embedding, line_count, path, function_id_filter)
1032
- SELECT new.id, e.vector, new.line_count, new.path, new.id FROM embeddings e WHERE e.id = new.embedding_id;
1033
- END;
1034
- CREATE TRIGGER functions_vector_delete AFTER DELETE ON functions BEGIN
1035
- DELETE FROM function_vectors WHERE function_id = old.id;
1036
- END;
1037
- CREATE TRIGGER functions_vector_update AFTER UPDATE OF embedding_id, line_count, path ON functions BEGIN
1038
- DELETE FROM function_vectors WHERE function_id = old.id;
1039
- INSERT INTO function_vectors(function_id, embedding, line_count, path, function_id_filter)
1040
- SELECT new.id, e.vector, new.line_count, new.path, new.id FROM embeddings e WHERE e.id = new.embedding_id;
1041
- END;
1042
- `;
1043
- }
1044
- function vectorBuffer(vector) {
1045
- const values = Float32Array.from(vector);
1046
- return new Uint8Array(values.buffer);
1047
- }
1048
- function bufferVector(vector) {
1049
- return Array.from(new Float32Array(vector.buffer, vector.byteOffset, vector.byteLength / 4));
1050
- }
1051
- function toIndexedFunction(row) {
1052
- return {
1053
- id: row.id,
1054
- path: row.path,
1055
- language: row.language,
1056
- kind: row.kind,
1057
- name: row.name,
1058
- qualifiedName: row.qualified_name,
1059
- signature: row.signature,
1060
- identityKey: row.identity_key,
1061
- startLine: row.start_line,
1062
- startColumn: row.start_column,
1063
- endLine: row.end_line,
1064
- endColumn: row.end_column,
1065
- lineCount: row.line_count,
1066
- source: row.source,
1067
- sourceHash: row.source_hash,
1068
- embeddingInput: row.embedding_input,
1069
- firstSeenCommit: row.first_seen_commit,
1070
- lastSeenCommit: row.last_seen_commit,
1071
- sourceMode: row.source_mode,
1072
- embeddingId: row.embedding_id,
1073
- description: row.description,
1074
- descriptionEmbeddingId: row.description_embedding_id
1075
- };
1076
- }
1077
- function reconcileFunctions(current, previous) {
1078
- const matches = /* @__PURE__ */ new Map();
1079
- const keys = /* @__PURE__ */ new Set([
1080
- ...current.map((callable) => `${callable.qualifiedName}\0${callable.kind}`),
1081
- ...previous.map((callable) => `${callable.qualified_name}\0${callable.kind}`)
1082
- ]);
1083
- for (const key of keys) {
1084
- const currentGroup = current.filter((callable) => `${callable.qualifiedName}\0${callable.kind}` === key);
1085
- const previousGroup = previous.filter((callable) => `${callable.qualified_name}\0${callable.kind}` === key);
1086
- const unmatchedPrevious = new Set(previousGroup);
1087
- const unmatchedCurrent = new Set(currentGroup);
1088
- for (const callable of currentGroup) {
1089
- const exact = [...unmatchedPrevious].find((old) => old.source_hash === callable.sourceHash);
1090
- if (exact) {
1091
- matches.set(callable, exact);
1092
- unmatchedPrevious.delete(exact);
1093
- unmatchedCurrent.delete(callable);
1094
- }
1095
- }
1096
- for (const callable of unmatchedCurrent) {
1097
- const old = unmatchedPrevious.values().next().value;
1098
- if (!old) break;
1099
- matches.set(callable, old);
1100
- unmatchedPrevious.delete(old);
1101
- }
1102
- }
1103
- return matches;
1104
- }
1105
-
1106
- export {
1107
- IndexDatabase,
1108
- readIndexErrors,
1109
- resetIndexState,
1110
- readIndexErrorCounts
1111
- };
1112
- //# sourceMappingURL=chunk-6LHZCHEB.js.map