@ninjaxtools/slopdex 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js ADDED
@@ -0,0 +1,1413 @@
1
+ // src/code-index.ts
2
+ import { lstat, readFile } from "node:fs/promises";
3
+ import path6 from "node:path";
4
+
5
+ // src/errors.ts
6
+ var CodeIndexError = class extends Error {
7
+ constructor(message, options) {
8
+ super(message, options);
9
+ this.name = "CodeIndexError";
10
+ }
11
+ };
12
+ var GitDivergenceError = class extends CodeIndexError {
13
+ constructor(checkpoint, target) {
14
+ super(`Stored Git checkpoint ${checkpoint} is not an ancestor of ${target}; rebuild the index.`);
15
+ this.name = "GitDivergenceError";
16
+ }
17
+ };
18
+ var IncompatibleIndexError = class extends CodeIndexError {
19
+ constructor(message) {
20
+ super(message);
21
+ this.name = "IncompatibleIndexError";
22
+ }
23
+ };
24
+
25
+ // src/git/repository.ts
26
+ import { spawn } from "node:child_process";
27
+ import path from "node:path";
28
+ var GitRepository = class {
29
+ constructor(rootDir) {
30
+ this.rootDir = rootDir;
31
+ }
32
+ rootDir;
33
+ async #run(args, allowExitOne = false) {
34
+ return await new Promise((resolve, reject) => {
35
+ const child = spawn("git", ["-C", this.rootDir, ...args], { stdio: ["ignore", "pipe", "pipe"] });
36
+ const stdout = [];
37
+ const stderr = [];
38
+ child.stdout.on("data", (value) => stdout.push(value));
39
+ child.stderr.on("data", (value) => stderr.push(value));
40
+ child.on("error", reject);
41
+ child.on("close", (code) => {
42
+ const exitCode = code ?? -1;
43
+ if (exitCode === 0 || allowExitOne && exitCode === 1) {
44
+ resolve({ stdout: Buffer.concat(stdout), code: exitCode });
45
+ return;
46
+ }
47
+ reject(new CodeIndexError(`Git command failed: ${Buffer.concat(stderr).toString("utf8").trim()}`));
48
+ });
49
+ });
50
+ }
51
+ async assertRepository() {
52
+ const result = await this.#run(["rev-parse", "--is-inside-work-tree"]);
53
+ if (result.stdout.toString("utf8").trim() !== "true") {
54
+ throw new CodeIndexError(`${this.rootDir} is not a Git worktree.`);
55
+ }
56
+ const topLevel = (await this.#run(["rev-parse", "--show-toplevel"])).stdout.toString("utf8").trim();
57
+ if (path.resolve(topLevel) !== path.resolve(this.rootDir)) {
58
+ throw new CodeIndexError(`rootDir must be the Git worktree root: ${topLevel}`);
59
+ }
60
+ }
61
+ async assertCleanWorkingTree(excludePaths = []) {
62
+ const pathspecs = [".", ...excludePaths.map((filePath) => `:(top,literal,exclude)${filePath}`)];
63
+ const result = await this.#run([
64
+ "status",
65
+ "--porcelain=v1",
66
+ "-z",
67
+ "--untracked-files=all",
68
+ "--ignore-submodules=none",
69
+ "--",
70
+ ...pathspecs
71
+ ]);
72
+ if (result.stdout.length > 0) {
73
+ throw new CodeIndexError("Cannot update from Git: the working tree has uncommitted changes. Commit or stash them first.");
74
+ }
75
+ }
76
+ async resolveCommit(ref) {
77
+ const result = await this.#run(["rev-parse", "--verify", "--end-of-options", `${ref}^{commit}`]);
78
+ const oid = result.stdout.toString("ascii").trim();
79
+ if (!/^[0-9a-f]{40,64}$/.test(oid)) throw new CodeIndexError(`Could not resolve Git commit: ${ref}`);
80
+ return oid;
81
+ }
82
+ async isAncestor(ancestor, descendant) {
83
+ const result = await this.#run(["merge-base", "--is-ancestor", ancestor, descendant], true);
84
+ return result.code === 0;
85
+ }
86
+ async listTree(commit) {
87
+ const result = await this.#run(["ls-tree", "-r", "-z", "-l", "--full-tree", commit, "--"]);
88
+ const entries = /* @__PURE__ */ new Map();
89
+ for (const record of splitNull(result.stdout)) {
90
+ const tab = record.indexOf(" ");
91
+ if (tab < 0) continue;
92
+ const [mode, type, oid, sizeText] = record.slice(0, tab).trim().split(/\s+/);
93
+ const filePath = record.slice(tab + 1);
94
+ const size = Number(sizeText);
95
+ if ((mode === "100644" || mode === "100755") && type === "blob" && oid && Number.isSafeInteger(size)) {
96
+ entries.set(filePath, { path: filePath, oid, mode, size });
97
+ }
98
+ }
99
+ return entries;
100
+ }
101
+ async readBlob(oid) {
102
+ return (await this.#run(["cat-file", "blob", oid])).stdout;
103
+ }
104
+ async diff(base, target) {
105
+ const result = await this.#run(["diff", "--name-status", "-z", "--find-renames", `${base}..${target}`, "--"]);
106
+ const values = splitNull(result.stdout);
107
+ const changes = [];
108
+ for (let index = 0; index < values.length; ) {
109
+ const rawStatus = values[index++];
110
+ if (!rawStatus) continue;
111
+ const status = rawStatus[0];
112
+ if (status === "R" || status === "C") {
113
+ const oldPath = values[index++];
114
+ const filePath = values[index++];
115
+ if (oldPath && filePath) changes.push({ status, oldPath, path: filePath });
116
+ } else if (status === "A" || status === "M" || status === "D" || status === "T") {
117
+ const filePath = values[index++];
118
+ if (filePath) changes.push({ status, path: filePath });
119
+ } else {
120
+ throw new CodeIndexError(`Unsupported Git diff status: ${rawStatus}`);
121
+ }
122
+ }
123
+ return changes;
124
+ }
125
+ };
126
+ function splitNull(buffer) {
127
+ const value = buffer.toString("utf8");
128
+ const parts = value.split("\0");
129
+ if (parts.at(-1) === "") parts.pop();
130
+ return parts;
131
+ }
132
+
133
+ // src/parser/callable-parser.ts
134
+ import path3 from "node:path";
135
+ import Parser from "tree-sitter";
136
+ import JavaScript from "tree-sitter-javascript";
137
+ import TypeScript from "tree-sitter-typescript";
138
+
139
+ // src/utils.ts
140
+ import { createHash } from "node:crypto";
141
+ import path2 from "node:path";
142
+ function sha256(value) {
143
+ return createHash("sha256").update(value).digest("hex");
144
+ }
145
+ function normalizeRelativePath(rootDir, inputPath) {
146
+ const absolute = path2.resolve(rootDir, inputPath);
147
+ const relative = path2.relative(rootDir, absolute);
148
+ if (relative === "" || relative === ".") {
149
+ throw new CodeIndexError(`Expected a file path, received repository root: ${inputPath}`);
150
+ }
151
+ if (relative.startsWith(`..${path2.sep}`) || path2.isAbsolute(relative)) {
152
+ throw new CodeIndexError(`Path escapes repository root: ${inputPath}`);
153
+ }
154
+ return relative.split(path2.sep).join("/");
155
+ }
156
+ function throwIfAborted(signal) {
157
+ signal?.throwIfAborted();
158
+ }
159
+ function chunk(values, size) {
160
+ const batches = [];
161
+ for (let index = 0; index < values.length; index += size) {
162
+ batches.push(values.slice(index, index + size));
163
+ }
164
+ return batches;
165
+ }
166
+ function assertPositiveInteger(value, name) {
167
+ if (!Number.isInteger(value) || value < 1) {
168
+ throw new CodeIndexError(`${name} must be a positive integer.`);
169
+ }
170
+ }
171
+ function normalizeEmbeddingVector(value, dimensions) {
172
+ if (!Array.isArray(value) || value.length !== dimensions) {
173
+ throw new CodeIndexError(`Expected a finite ${dimensions}-dimensional embedding vector.`);
174
+ }
175
+ const converted = [];
176
+ for (const component of value) {
177
+ if (typeof component !== "number" || !Number.isFinite(component)) {
178
+ throw new CodeIndexError(`Expected a finite ${dimensions}-dimensional embedding vector.`);
179
+ }
180
+ const normalized = Math.fround(component);
181
+ if (!Number.isFinite(normalized)) {
182
+ throw new CodeIndexError(`Expected a finite ${dimensions}-dimensional embedding vector.`);
183
+ }
184
+ converted.push(normalized);
185
+ }
186
+ if (!converted.some((component) => component !== 0)) {
187
+ throw new CodeIndexError("Embedding vector must not be all zeroes.");
188
+ }
189
+ return converted;
190
+ }
191
+
192
+ // src/parser/callable-parser.ts
193
+ var parsers = /* @__PURE__ */ new Map();
194
+ function languageForPath(filePath) {
195
+ switch (path3.extname(filePath).toLowerCase()) {
196
+ case ".ts":
197
+ case ".mts":
198
+ case ".cts":
199
+ return "typescript";
200
+ case ".tsx":
201
+ return "tsx";
202
+ case ".js":
203
+ case ".mjs":
204
+ case ".cjs":
205
+ return "javascript";
206
+ case ".jsx":
207
+ return "jsx";
208
+ default:
209
+ return null;
210
+ }
211
+ }
212
+ function getParser(language) {
213
+ const existing = parsers.get(language);
214
+ if (existing) return existing;
215
+ const parser = new Parser();
216
+ if (language === "typescript") parser.setLanguage(TypeScript.typescript);
217
+ else if (language === "tsx") parser.setLanguage(TypeScript.tsx);
218
+ else parser.setLanguage(JavaScript);
219
+ parsers.set(language, parser);
220
+ return parser;
221
+ }
222
+ function parseTree(parser, content, relativePath) {
223
+ try {
224
+ return parser.parse(content, void 0, { bufferSize: Buffer.byteLength(content) + 1 });
225
+ } catch (error) {
226
+ const message = error instanceof Error ? error.message : String(error);
227
+ throw new CodeIndexError(`Cannot index ${relativePath}: tree-sitter failed: ${message}`, { cause: error });
228
+ }
229
+ }
230
+ function maskSyntax(value, replacement) {
231
+ return replacement + value.slice(replacement.length).replace(/[^\r\n]/g, " ");
232
+ }
233
+ function recoverUnsupportedTypeScriptSyntax(parser, tree, content, relativePath) {
234
+ const errors = [];
235
+ const collectErrors = (node) => {
236
+ if (node.isError || node.isMissing) errors.push(node);
237
+ for (const child of node.children) collectErrors(child);
238
+ };
239
+ collectErrors(tree.rootNode);
240
+ if (errors.length === 0) return null;
241
+ let recoveredContent = content;
242
+ let changed = false;
243
+ const taggedTemplateTypeArguments = /* @__PURE__ */ new Map();
244
+ for (const error of errors) {
245
+ const parent = error.parent;
246
+ if (error.isMissing && error.type === "!" && parent?.type === "type_arguments" && /^\s*`/.test(content.slice(parent.endIndex))) {
247
+ taggedTemplateTypeArguments.set(parent.startIndex, parent);
248
+ }
249
+ }
250
+ for (const node of [...taggedTemplateTypeArguments.values()].toSorted((left, right) => right.startIndex - left.startIndex)) {
251
+ const value = recoveredContent.slice(node.startIndex, node.endIndex);
252
+ recoveredContent = `${recoveredContent.slice(0, node.startIndex)}${value.replace(/[^\r\n]/g, " ")}${recoveredContent.slice(node.endIndex)}`;
253
+ changed = true;
254
+ }
255
+ for (const node of errors.filter((candidate) => {
256
+ const parent = candidate.parent;
257
+ return !candidate.isMissing && candidate.type === "ERROR" && candidate.text === "type" && parent?.type === "export_statement" && content.slice(parent.startIndex, candidate.startIndex).trim() === "export" && /^\s*\*/.test(content.slice(candidate.endIndex, parent.endIndex));
258
+ }).toSorted((left, right) => right.startIndex - left.startIndex)) {
259
+ recoveredContent = `${recoveredContent.slice(0, node.startIndex)} ${recoveredContent.slice(node.endIndex)}`;
260
+ changed = true;
261
+ }
262
+ recoveredContent = recoveredContent.replace(
263
+ /\bimport\s*\(\s*(["'])(?:\\.|(?!\1)[^\\])*\1\s*\)/g,
264
+ (value) => {
265
+ changed = true;
266
+ return maskSyntax(value, "X");
267
+ }
268
+ );
269
+ if (!changed) return null;
270
+ const recoveredTree = parseTree(parser, recoveredContent, relativePath);
271
+ return recoveredTree.rootNode.hasError ? null : recoveredTree;
272
+ }
273
+ function textFor(content, node) {
274
+ return content.slice(node.startIndex, node.endIndex);
275
+ }
276
+ function fieldText(content, node, field) {
277
+ const child = node.childForFieldName(field);
278
+ return child ? textFor(content, child) : null;
279
+ }
280
+ function callableKind(node, name) {
281
+ if (name === "constructor") return "constructor";
282
+ if (node.type.includes("generator") || /^\s*(?:async\s+)?\*/.test(node.text)) return "generator";
283
+ return node.type === "method_definition" ? "method" : "function";
284
+ }
285
+ function bindingName(content, node) {
286
+ const nameNode = node.childForFieldName("name");
287
+ if (!nameNode || !["identifier", "property_identifier", "private_property_identifier"].includes(nameNode.type)) {
288
+ return null;
289
+ }
290
+ return textFor(content, nameNode).replace(/^#/, "");
291
+ }
292
+ function assignmentName(content, node) {
293
+ const left = node.childForFieldName("left");
294
+ if (!left) return null;
295
+ const value = textFor(content, left).trim();
296
+ const match = value.match(/(?:^|[.#])([A-Za-z_$][\w$]*)$/);
297
+ return match?.[1] ?? null;
298
+ }
299
+ function objectScopeName(content, node) {
300
+ const parent = node.parent;
301
+ if (!parent) return null;
302
+ if (parent.type === "variable_declarator" && parent.childForFieldName("value")?.id === node.id) {
303
+ return bindingName(content, parent);
304
+ }
305
+ if (parent.type === "pair" && parent.childForFieldName("value")?.id === node.id) {
306
+ return fieldText(content, parent, "key")?.replace(/["']/g, "") ?? null;
307
+ }
308
+ if (parent.type === "assignment_expression" && parent.childForFieldName("right")?.id === node.id) {
309
+ return assignmentName(content, parent);
310
+ }
311
+ return null;
312
+ }
313
+ function walk(node, content, scope, candidates) {
314
+ if (node.type === "class_declaration" || node.type === "class") {
315
+ const name = fieldText(content, node, "name") ?? objectScopeName(content, node);
316
+ const nextScope = name ? [...scope, name] : scope;
317
+ for (const child of node.namedChildren) walk(child, content, nextScope, candidates);
318
+ return;
319
+ }
320
+ if (node.type === "internal_module" || node.type === "module") {
321
+ const name = fieldText(content, node, "name")?.replace(/["']/g, "");
322
+ const nextScope = name ? [...scope, name] : scope;
323
+ for (const child of node.namedChildren) walk(child, content, nextScope, candidates);
324
+ return;
325
+ }
326
+ if (node.type === "object") {
327
+ const name = objectScopeName(content, node);
328
+ const nextScope = name ? [...scope, name] : scope;
329
+ for (const child of node.namedChildren) walk(child, content, nextScope, candidates);
330
+ return;
331
+ }
332
+ if (["function_declaration", "generator_function_declaration"].includes(node.type)) {
333
+ const name = fieldText(content, node, "name");
334
+ const body = node.childForFieldName("body");
335
+ if (name && body) {
336
+ candidates.push({ node, name, kind: callableKind(node, name), scope });
337
+ walk(body, content, [...scope, name], candidates);
338
+ }
339
+ return;
340
+ }
341
+ if (node.type === "method_definition") {
342
+ const name = fieldText(content, node, "name")?.replace(/^#/, "");
343
+ const body = node.childForFieldName("body");
344
+ if (name && body) {
345
+ candidates.push({ node, name, kind: callableKind(node, name), scope });
346
+ walk(body, content, [...scope, name], candidates);
347
+ }
348
+ return;
349
+ }
350
+ if (node.type === "public_field_definition" || node.type === "field_definition") {
351
+ const name = (fieldText(content, node, "name") ?? fieldText(content, node, "property"))?.replace(/^#/, "");
352
+ const value = node.childForFieldName("value");
353
+ if (name && value && ["arrow_function", "function_expression", "generator_function"].includes(value.type)) {
354
+ const kind = callableKind(value, name) === "generator" ? "generator" : "method";
355
+ candidates.push({ node: value, name, kind, scope });
356
+ const body = value.childForFieldName("body");
357
+ if (body) walk(body, content, [...scope, name], candidates);
358
+ return;
359
+ }
360
+ }
361
+ if (node.type === "variable_declarator") {
362
+ const name = bindingName(content, node);
363
+ const value = node.childForFieldName("value");
364
+ if (name && value && ["arrow_function", "function_expression", "generator_function"].includes(value.type)) {
365
+ candidates.push({ node: value, name, kind: callableKind(value, name), scope });
366
+ const body = value.childForFieldName("body");
367
+ if (body) walk(body, content, [...scope, name], candidates);
368
+ return;
369
+ }
370
+ }
371
+ if (node.type === "pair") {
372
+ const name = fieldText(content, node, "key")?.replace(/["']/g, "");
373
+ const value = node.childForFieldName("value");
374
+ if (name && value && ["arrow_function", "function_expression", "generator_function"].includes(value.type)) {
375
+ candidates.push({ node: value, name, kind: callableKind(value, name), scope });
376
+ const body = value.childForFieldName("body");
377
+ if (body) walk(body, content, [...scope, name], candidates);
378
+ return;
379
+ }
380
+ }
381
+ if (node.type === "assignment_expression") {
382
+ const name = assignmentName(content, node);
383
+ const value = node.childForFieldName("right");
384
+ if (name && value && ["arrow_function", "function_expression", "generator_function"].includes(value.type)) {
385
+ candidates.push({ node: value, name, kind: callableKind(value, name), scope });
386
+ const body = value.childForFieldName("body");
387
+ if (body) walk(body, content, [...scope, name], candidates);
388
+ return;
389
+ }
390
+ }
391
+ if (["function_expression", "generator_function"].includes(node.type)) {
392
+ const name = fieldText(content, node, "name");
393
+ const body = node.childForFieldName("body");
394
+ if (name && body) {
395
+ candidates.push({ node, name, kind: callableKind(node, name), scope });
396
+ walk(body, content, [...scope, name], candidates);
397
+ return;
398
+ }
399
+ }
400
+ for (const child of node.namedChildren) walk(child, content, scope, candidates);
401
+ }
402
+ function signatureFor(content, node, name) {
403
+ const parameters = node.childForFieldName("parameters");
404
+ if (!parameters) return name;
405
+ const returnType = node.childForFieldName("return_type");
406
+ return `${name}${textFor(content, parameters)}${returnType ? textFor(content, returnType) : ""}`;
407
+ }
408
+ function parseCallables(relativePath, content, onWarning = console.warn) {
409
+ const language = languageForPath(relativePath);
410
+ if (!language) return [];
411
+ const parser = getParser(language);
412
+ const parsedTree = parseTree(parser, content, relativePath);
413
+ const tree = parsedTree.rootNode.hasError ? recoverUnsupportedTypeScriptSyntax(parser, parsedTree, content, relativePath) ?? parsedTree : parsedTree;
414
+ if (tree.rootNode.hasError) {
415
+ onWarning(`Cannot fully parse ${relativePath}: tree-sitter reported syntax errors; indexing recoverable callables only.`);
416
+ }
417
+ const candidates = [];
418
+ walk(tree.rootNode, content, [], candidates);
419
+ candidates.sort((left, right) => left.node.startIndex - right.node.startIndex);
420
+ const occurrences = /* @__PURE__ */ new Map();
421
+ return candidates.map(({ node, name, kind, scope }) => {
422
+ const qualifiedName = [...scope, name].join(".");
423
+ const baseIdentity = `${relativePath}\0${qualifiedName}\0${kind}`;
424
+ const occurrence = occurrences.get(baseIdentity) ?? 0;
425
+ occurrences.set(baseIdentity, occurrence + 1);
426
+ const identityKey = sha256(`${baseIdentity}\0${occurrence}`);
427
+ const source = textFor(content, node);
428
+ const signature = signatureFor(content, node, name);
429
+ const embeddingInput = [
430
+ `language: ${language}`,
431
+ `kind: ${kind}`,
432
+ `symbol: ${qualifiedName}`,
433
+ signature ? `signature: ${signature}` : null,
434
+ "source:",
435
+ source
436
+ ].filter((value) => value !== null).join("\n");
437
+ return {
438
+ path: relativePath,
439
+ language,
440
+ kind,
441
+ name,
442
+ qualifiedName,
443
+ signature,
444
+ identityKey,
445
+ startLine: node.startPosition.row + 1,
446
+ startColumn: node.startPosition.column + 1,
447
+ endLine: node.endPosition.row + 1,
448
+ endColumn: node.endPosition.column + 1,
449
+ source,
450
+ sourceHash: sha256(source),
451
+ embeddingInput
452
+ };
453
+ });
454
+ }
455
+
456
+ // src/source-policy.ts
457
+ import path4 from "node:path";
458
+ var DEFAULT_EXCLUDED_SEGMENTS = /* @__PURE__ */ new Set([
459
+ ".git",
460
+ ".slopdex",
461
+ "node_modules",
462
+ "dist",
463
+ "build",
464
+ "coverage",
465
+ "vendor",
466
+ "generated"
467
+ ]);
468
+ var SourcePolicy = class {
469
+ #include;
470
+ #exclude;
471
+ constructor(include = [], exclude = []) {
472
+ this.#include = include;
473
+ this.#exclude = exclude;
474
+ }
475
+ includes(relativePath) {
476
+ const normalized = relativePath.replaceAll("\\", "/");
477
+ if (!languageForPath(normalized)) return false;
478
+ if (normalized.split("/").some((segment) => DEFAULT_EXCLUDED_SEGMENTS.has(segment))) return false;
479
+ if (this.#exclude.some((pattern) => path4.matchesGlob(normalized, pattern))) return false;
480
+ return this.#include.length === 0 || this.#include.some((pattern) => path4.matchesGlob(normalized, pattern));
481
+ }
482
+ };
483
+
484
+ // src/storage/database.ts
485
+ import { mkdirSync } from "node:fs";
486
+ import path5 from "node:path";
487
+ import { DatabaseSync } from "node:sqlite";
488
+ import * as sqliteVec from "sqlite-vec";
489
+ var SCHEMA_VERSION = "1";
490
+ var IndexDatabase = class {
491
+ #db;
492
+ #rootDir;
493
+ #indexPath;
494
+ #profile;
495
+ constructor(indexPath, rootDir, profile, readOnly = false) {
496
+ if (!readOnly) mkdirSync(path5.dirname(indexPath), { recursive: true });
497
+ this.#indexPath = indexPath;
498
+ this.#rootDir = rootDir;
499
+ this.#profile = profile;
500
+ this.#db = new DatabaseSync(indexPath, { allowExtension: true, readOnly });
501
+ sqliteVec.load(this.#db);
502
+ this.#db.enableLoadExtension(false);
503
+ if (readOnly) {
504
+ this.#db.exec("PRAGMA foreign_keys=ON");
505
+ } else {
506
+ this.#db.exec("PRAGMA journal_mode=WAL; PRAGMA synchronous=NORMAL; PRAGMA foreign_keys=ON;");
507
+ this.#migrate();
508
+ }
509
+ this.#validateMetadata(readOnly);
510
+ }
511
+ close() {
512
+ this.#db.close();
513
+ }
514
+ #migrate() {
515
+ this.#db.exec(`
516
+ CREATE TABLE IF NOT EXISTS metadata (
517
+ key TEXT PRIMARY KEY,
518
+ value TEXT NOT NULL
519
+ );
520
+ CREATE TABLE IF NOT EXISTS files (
521
+ path TEXT PRIMARY KEY,
522
+ content_hash TEXT NOT NULL,
523
+ blob_oid TEXT,
524
+ source_mode TEXT NOT NULL CHECK(source_mode IN ('git', 'working-tree')),
525
+ indexed_commit TEXT,
526
+ previous_path TEXT,
527
+ language TEXT NOT NULL,
528
+ byte_size INTEGER NOT NULL
529
+ );
530
+ CREATE TABLE IF NOT EXISTS embeddings (
531
+ id INTEGER PRIMARY KEY,
532
+ embedding_key TEXT NOT NULL UNIQUE,
533
+ vector BLOB NOT NULL
534
+ );
535
+ CREATE TABLE IF NOT EXISTS functions (
536
+ id INTEGER PRIMARY KEY,
537
+ path TEXT NOT NULL REFERENCES files(path) ON DELETE CASCADE ON UPDATE CASCADE,
538
+ language TEXT NOT NULL,
539
+ kind TEXT NOT NULL,
540
+ name TEXT NOT NULL,
541
+ qualified_name TEXT NOT NULL,
542
+ signature TEXT,
543
+ identity_key TEXT NOT NULL UNIQUE,
544
+ start_line INTEGER NOT NULL,
545
+ start_column INTEGER NOT NULL,
546
+ end_line INTEGER NOT NULL,
547
+ end_column INTEGER NOT NULL,
548
+ source TEXT NOT NULL,
549
+ source_hash TEXT NOT NULL,
550
+ embedding_input TEXT NOT NULL,
551
+ first_seen_commit TEXT,
552
+ last_seen_commit TEXT,
553
+ source_mode TEXT NOT NULL CHECK(source_mode IN ('git', 'working-tree')),
554
+ embedding_id INTEGER NOT NULL REFERENCES embeddings(id)
555
+ );
556
+ CREATE INDEX IF NOT EXISTS functions_path ON functions(path);
557
+ CREATE INDEX IF NOT EXISTS functions_embedding ON functions(embedding_id);
558
+ `);
559
+ const fileColumns = this.#db.prepare("PRAGMA table_info(files)").all();
560
+ if (!fileColumns.some((column) => column.name === "previous_path")) {
561
+ this.#db.exec("ALTER TABLE files ADD COLUMN previous_path TEXT");
562
+ }
563
+ }
564
+ #metadata(key) {
565
+ const row = this.#db.prepare("SELECT value FROM metadata WHERE key = ?").get(key);
566
+ return row?.value ?? null;
567
+ }
568
+ #setMetadata(key, value) {
569
+ this.#db.prepare(`
570
+ INSERT INTO metadata(key, value) VALUES (?, ?)
571
+ ON CONFLICT(key) DO UPDATE SET value = excluded.value
572
+ `).run(key, value);
573
+ }
574
+ #validateMetadata(readOnly) {
575
+ const schemaVersion = this.#metadata("schema_version");
576
+ if (readOnly && !schemaVersion) throw new IncompatibleIndexError("Target file is not a slopdex index.");
577
+ if (schemaVersion && schemaVersion !== SCHEMA_VERSION) {
578
+ throw new IncompatibleIndexError(`Unsupported index schema version ${schemaVersion}.`);
579
+ }
580
+ const storedRoot = this.#metadata("root_dir");
581
+ if (readOnly && !storedRoot) throw new IncompatibleIndexError("Target index has no repository identity.");
582
+ if (storedRoot && path5.resolve(storedRoot) !== this.#rootDir) {
583
+ throw new IncompatibleIndexError(`Index belongs to ${storedRoot}, not ${this.#rootDir}.`);
584
+ }
585
+ const profileJson = JSON.stringify(this.#profile);
586
+ const storedProfile = this.#metadata("embedding_profile");
587
+ if (readOnly && !storedProfile) throw new IncompatibleIndexError("Target index has no embedding profile.");
588
+ if (storedProfile && storedProfile !== profileJson) {
589
+ throw new IncompatibleIndexError("Embedding provider, model, dimensions, or strategy differs from this index.");
590
+ }
591
+ if (readOnly) return;
592
+ this.#transaction(() => {
593
+ this.#setMetadata("schema_version", SCHEMA_VERSION);
594
+ this.#setMetadata("root_dir", this.#rootDir);
595
+ this.#setMetadata("embedding_profile", profileJson);
596
+ if (!this.#metadata("generation")) this.#setMetadata("generation", "0");
597
+ });
598
+ }
599
+ #transaction(operation) {
600
+ this.#db.exec("BEGIN IMMEDIATE");
601
+ try {
602
+ const result = operation();
603
+ this.#db.exec("COMMIT");
604
+ return result;
605
+ } catch (error) {
606
+ this.#db.exec("ROLLBACK");
607
+ throw error;
608
+ }
609
+ }
610
+ getCheckpoint() {
611
+ return this.#metadata("git_checkpoint");
612
+ }
613
+ getGeneration() {
614
+ return Number(this.#metadata("generation") ?? 0);
615
+ }
616
+ getWorkingTreeFiles() {
617
+ return this.#db.prepare(`
618
+ SELECT path, previous_path AS previousPath
619
+ FROM files WHERE source_mode = 'working-tree' ORDER BY path
620
+ `).all();
621
+ }
622
+ getEmbeddingKeys(keys) {
623
+ const found = /* @__PURE__ */ new Set();
624
+ const statement = this.#db.prepare("SELECT 1 FROM embeddings WHERE embedding_key = ?");
625
+ for (const key of keys) {
626
+ if (statement.get(key)) found.add(key);
627
+ }
628
+ return found;
629
+ }
630
+ applyUpdate(options) {
631
+ return this.#transaction(() => {
632
+ if (options.expectedCheckpoint !== void 0 && this.getCheckpoint() !== options.expectedCheckpoint) {
633
+ throw new CodeIndexError("Git checkpoint changed while the update was being prepared.");
634
+ }
635
+ const stats = {
636
+ filesUpdated: options.files.length,
637
+ filesDeleted: 0,
638
+ functionsAdded: 0,
639
+ functionsUpdated: 0,
640
+ functionsDeleted: 0,
641
+ embeddingsCreated: 0,
642
+ checkpoint: options.checkpoint ?? this.getCheckpoint()
643
+ };
644
+ const deleteFile = this.#db.prepare("DELETE FROM files WHERE path = ?");
645
+ const countFunctions = this.#db.prepare("SELECT COUNT(*) AS count FROM functions WHERE path = ?");
646
+ const uniqueDeletes = new Set(options.deletePaths);
647
+ for (const file of options.files) {
648
+ uniqueDeletes.delete(file.path);
649
+ if (file.previousPath) uniqueDeletes.delete(file.previousPath);
650
+ if (file.replacePath) uniqueDeletes.delete(file.replacePath);
651
+ }
652
+ for (const filePath of uniqueDeletes) {
653
+ const count = Number(countFunctions.get(filePath).count);
654
+ const result = deleteFile.run(filePath);
655
+ if (result.changes > 0) {
656
+ stats.filesDeleted += 1;
657
+ stats.functionsDeleted += count;
658
+ }
659
+ }
660
+ for (const file of options.files) this.#replaceFile(file, stats);
661
+ this.#db.exec("DELETE FROM embeddings WHERE id NOT IN (SELECT DISTINCT embedding_id FROM functions)");
662
+ const generation = this.getGeneration() + 1;
663
+ this.#setMetadata("generation", String(generation));
664
+ if (options.checkpoint) this.#setMetadata("git_checkpoint", options.checkpoint);
665
+ return stats;
666
+ });
667
+ }
668
+ #replaceFile(file, stats) {
669
+ const sourcePath = file.replacePath ?? file.previousPath ?? file.path;
670
+ const oldRows = this.#rowsForPath(sourcePath);
671
+ const existingFile = this.#db.prepare("SELECT previous_path FROM files WHERE path = ?").get(sourcePath);
672
+ const oldMatches = reconcileFunctions(file.callables, oldRows);
673
+ if (sourcePath !== file.path) {
674
+ this.#db.prepare("DELETE FROM files WHERE path = ?").run(sourcePath);
675
+ } else {
676
+ this.#db.prepare("DELETE FROM files WHERE path = ?").run(file.path);
677
+ }
678
+ this.#db.prepare(`
679
+ INSERT INTO files(path, content_hash, blob_oid, source_mode, indexed_commit, previous_path, language, byte_size)
680
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?)
681
+ `).run(
682
+ file.path,
683
+ file.contentHash,
684
+ file.blobOid,
685
+ file.sourceMode,
686
+ file.indexedCommit,
687
+ file.sourceMode === "working-tree" ? file.previousPath ?? existingFile?.previous_path ?? null : null,
688
+ file.language,
689
+ file.byteSize
690
+ );
691
+ const usedIds = /* @__PURE__ */ new Set();
692
+ const orderedCallables = [
693
+ ...file.callables.filter((callable) => oldMatches.has(callable)),
694
+ ...file.callables.filter((callable) => !oldMatches.has(callable))
695
+ ];
696
+ for (const callable of orderedCallables) {
697
+ if (callable.vector) {
698
+ const result = this.#db.prepare("INSERT OR IGNORE INTO embeddings(embedding_key, vector) VALUES (?, ?)").run(callable.embeddingKey, vectorBuffer(callable.vector));
699
+ stats.embeddingsCreated += Number(result.changes);
700
+ }
701
+ const embedding = this.#db.prepare("SELECT id FROM embeddings WHERE embedding_key = ?").get(callable.embeddingKey);
702
+ if (!embedding) throw new CodeIndexError(`Missing embedding for ${callable.qualifiedName}.`);
703
+ const old = oldMatches.get(callable);
704
+ if (old) usedIds.add(old.id);
705
+ const firstSeenCommit = old?.first_seen_commit ?? file.indexedCommit;
706
+ this.#db.prepare(`
707
+ INSERT INTO functions(
708
+ id, path, language, kind, name, qualified_name, signature, identity_key,
709
+ start_line, start_column, end_line, end_column, source, source_hash,
710
+ embedding_input, first_seen_commit, last_seen_commit, source_mode, embedding_id
711
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
712
+ `).run(
713
+ old?.id ?? null,
714
+ file.path,
715
+ callable.language,
716
+ callable.kind,
717
+ callable.name,
718
+ callable.qualifiedName,
719
+ callable.signature,
720
+ callable.identityKey,
721
+ callable.startLine,
722
+ callable.startColumn,
723
+ callable.endLine,
724
+ callable.endColumn,
725
+ callable.source,
726
+ callable.sourceHash,
727
+ callable.embeddingInput,
728
+ firstSeenCommit,
729
+ file.indexedCommit,
730
+ file.sourceMode,
731
+ embedding.id
732
+ );
733
+ if (old) stats.functionsUpdated += 1;
734
+ else stats.functionsAdded += 1;
735
+ }
736
+ stats.functionsDeleted += oldRows.length - usedIds.size;
737
+ }
738
+ #rowsForPath(filePath) {
739
+ return this.#db.prepare("SELECT * FROM functions WHERE path = ? ORDER BY start_line, start_column, id").all(filePath);
740
+ }
741
+ allFunctions() {
742
+ return this.#db.prepare("SELECT * FROM functions ORDER BY path, start_line, start_column, id").all().map(toIndexedFunction);
743
+ }
744
+ allFilePaths() {
745
+ return this.#db.prepare("SELECT path FROM files ORDER BY path").all().map((row) => row.path);
746
+ }
747
+ previousPath(filePath) {
748
+ const row = this.#db.prepare("SELECT previous_path FROM files WHERE path = ?").get(filePath);
749
+ return row?.previous_path ?? null;
750
+ }
751
+ functionsForPaths(paths) {
752
+ const results = [];
753
+ for (const filePath of paths) results.push(...this.#rowsForPath(filePath).map(toIndexedFunction));
754
+ return results;
755
+ }
756
+ searchVector(vector, options) {
757
+ const rows = this.#db.prepare(`
758
+ SELECT f.*, 1.0 - vec_distance_cosine(e.vector, ?) AS similarity
759
+ FROM functions f
760
+ JOIN embeddings e ON e.id = f.embedding_id
761
+ WHERE (? IS NULL OR f.id != ?)
762
+ AND (1.0 - vec_distance_cosine(e.vector, ?)) >= ?
763
+ AND (? IS NULL OR (1.0 - vec_distance_cosine(e.vector, ?)) <= ?)
764
+ ORDER BY similarity DESC, f.id ASC
765
+ LIMIT ?
766
+ `).all(
767
+ vectorBuffer(vector),
768
+ options.excludeId ?? null,
769
+ options.excludeId ?? null,
770
+ vectorBuffer(vector),
771
+ options.minSimilarity,
772
+ options.maxSimilarity ?? null,
773
+ vectorBuffer(vector),
774
+ options.maxSimilarity ?? null,
775
+ options.limit
776
+ );
777
+ return rows.map((row) => ({ function: toIndexedFunction(row), similarity: row.similarity }));
778
+ }
779
+ vectorForFunction(id) {
780
+ const row = this.#db.prepare(`
781
+ SELECT e.vector FROM functions f JOIN embeddings e ON e.id = f.embedding_id WHERE f.id = ?
782
+ `).get(id);
783
+ if (!row) throw new CodeIndexError(`Function ${id} does not exist.`);
784
+ return Array.from(new Float32Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength / 4));
785
+ }
786
+ status() {
787
+ const functionCount = Number(this.#db.prepare("SELECT COUNT(*) AS count FROM functions").get().count);
788
+ const fileCount = Number(this.#db.prepare("SELECT COUNT(*) AS count FROM files").get().count);
789
+ return {
790
+ rootDir: this.#rootDir,
791
+ indexPath: this.#indexPath,
792
+ functionCount,
793
+ fileCount,
794
+ generation: this.getGeneration(),
795
+ gitCheckpoint: this.getCheckpoint(),
796
+ embeddingProfile: this.#profile
797
+ };
798
+ }
799
+ };
800
+ function vectorBuffer(vector) {
801
+ const values = Float32Array.from(vector);
802
+ return new Uint8Array(values.buffer);
803
+ }
804
+ function toIndexedFunction(row) {
805
+ return {
806
+ id: row.id,
807
+ path: row.path,
808
+ language: row.language,
809
+ kind: row.kind,
810
+ name: row.name,
811
+ qualifiedName: row.qualified_name,
812
+ signature: row.signature,
813
+ identityKey: row.identity_key,
814
+ startLine: row.start_line,
815
+ startColumn: row.start_column,
816
+ endLine: row.end_line,
817
+ endColumn: row.end_column,
818
+ source: row.source,
819
+ sourceHash: row.source_hash,
820
+ embeddingInput: row.embedding_input,
821
+ firstSeenCommit: row.first_seen_commit,
822
+ lastSeenCommit: row.last_seen_commit,
823
+ sourceMode: row.source_mode,
824
+ embeddingId: row.embedding_id
825
+ };
826
+ }
827
+ function reconcileFunctions(current, previous) {
828
+ const matches = /* @__PURE__ */ new Map();
829
+ const keys = /* @__PURE__ */ new Set([
830
+ ...current.map((callable) => `${callable.qualifiedName}\0${callable.kind}`),
831
+ ...previous.map((callable) => `${callable.qualified_name}\0${callable.kind}`)
832
+ ]);
833
+ for (const key of keys) {
834
+ const currentGroup = current.filter((callable) => `${callable.qualifiedName}\0${callable.kind}` === key);
835
+ const previousGroup = previous.filter((callable) => `${callable.qualified_name}\0${callable.kind}` === key);
836
+ const unmatchedPrevious = new Set(previousGroup);
837
+ const unmatchedCurrent = new Set(currentGroup);
838
+ for (const callable of currentGroup) {
839
+ const exact = [...unmatchedPrevious].find((old) => old.source_hash === callable.sourceHash);
840
+ if (exact) {
841
+ matches.set(callable, exact);
842
+ unmatchedPrevious.delete(exact);
843
+ unmatchedCurrent.delete(callable);
844
+ }
845
+ }
846
+ for (const callable of unmatchedCurrent) {
847
+ const old = unmatchedPrevious.values().next().value;
848
+ if (!old) break;
849
+ matches.set(callable, old);
850
+ unmatchedPrevious.delete(old);
851
+ }
852
+ }
853
+ return matches;
854
+ }
855
+
856
+ // src/code-index.ts
857
+ var DEFAULT_MAX_FILE_SIZE = 1024 * 1024;
858
+ var DEFAULT_BATCH_SIZE = 32;
859
+ var CodeIndex = class {
860
+ rootDir;
861
+ indexPath;
862
+ provider;
863
+ #database;
864
+ #policy;
865
+ #maxFileSize;
866
+ #embeddingBatchSize;
867
+ #git;
868
+ #onWarning;
869
+ constructor(options) {
870
+ this.rootDir = path6.resolve(options.rootDir);
871
+ this.indexPath = path6.resolve(options.indexPath ?? path6.join(this.rootDir, ".slopdex", "index.sqlite"));
872
+ this.provider = options.provider;
873
+ const profile = normalizeProfile(options.provider.profile);
874
+ assertPositiveInteger(profile.dimensions, "embedding dimensions");
875
+ this.#database = new IndexDatabase(this.indexPath, this.rootDir, profile, options.readOnly ?? false);
876
+ this.#policy = new SourcePolicy(options.include, options.exclude);
877
+ this.#maxFileSize = options.maxFileSize ?? DEFAULT_MAX_FILE_SIZE;
878
+ this.#embeddingBatchSize = options.embeddingBatchSize ?? DEFAULT_BATCH_SIZE;
879
+ this.#onWarning = options.onWarning ?? console.warn;
880
+ assertPositiveInteger(this.#maxFileSize, "maxFileSize");
881
+ assertPositiveInteger(this.#embeddingBatchSize, "embeddingBatchSize");
882
+ this.#git = new GitRepository(this.rootDir);
883
+ }
884
+ close() {
885
+ this.#database.close();
886
+ }
887
+ status() {
888
+ return this.#database.status();
889
+ }
890
+ allFunctions() {
891
+ return this.#database.allFunctions();
892
+ }
893
+ async updateFiles(options) {
894
+ const normalizedRenames = (options.renames ?? []).map(({ from, to }) => ({
895
+ from: normalizeRelativePath(this.rootDir, from),
896
+ to: normalizeRelativePath(this.rootDir, to)
897
+ }));
898
+ validateRenames(normalizedRenames);
899
+ const upsertPaths = [...new Set((options.upsert ?? []).map((value) => normalizeRelativePath(this.rootDir, value)))];
900
+ const deletePaths = [...new Set((options.delete ?? []).map((value) => normalizeRelativePath(this.rootDir, value)))];
901
+ const renameMap = new Map(
902
+ normalizedRenames.map(({ from, to }) => [to, from])
903
+ );
904
+ for (const target of renameMap.keys()) {
905
+ if (!upsertPaths.includes(target)) upsertPaths.push(target);
906
+ }
907
+ const prepared = [];
908
+ for (const relativePath of upsertPaths) {
909
+ throwIfAborted(options.signal);
910
+ if (!this.#policy.includes(relativePath)) throw new CodeIndexError(`Unsupported or excluded source file: ${relativePath}`);
911
+ const absolutePath = path6.join(this.rootDir, relativePath);
912
+ const info = await lstat(absolutePath);
913
+ if (info.isSymbolicLink()) throw new CodeIndexError(`Symbolic links are not supported: ${relativePath}`);
914
+ if (!info.isFile()) throw new CodeIndexError(`Not a regular file: ${relativePath}`);
915
+ if (info.size > this.#maxFileSize) throw new CodeIndexError(`File exceeds maxFileSize: ${relativePath}`);
916
+ const content = await readFile(absolutePath);
917
+ const renameSource = renameMap.get(relativePath);
918
+ const originalPath = renameSource ? this.#database.previousPath(renameSource) ?? renameSource : void 0;
919
+ prepared.push(this.#prepareFile(relativePath, content, {
920
+ blobOid: null,
921
+ sourceMode: "working-tree",
922
+ indexedCommit: null,
923
+ ...originalPath ? { previousPath: originalPath } : {},
924
+ ...renameSource ? { replacePath: renameSource } : {}
925
+ }));
926
+ }
927
+ await this.#attachEmbeddings(prepared, options.signal);
928
+ return this.#database.applyUpdate({
929
+ files: prepared,
930
+ deletePaths: [...deletePaths, ...renameMap.values()]
931
+ });
932
+ }
933
+ async updateFromGit(options = {}) {
934
+ throwIfAborted(options.signal);
935
+ await this.#git.assertRepository();
936
+ const relativeIndexPath = path6.relative(this.rootDir, this.indexPath).replaceAll(path6.sep, "/");
937
+ const indexArtifacts = relativeIndexPath && !relativeIndexPath.startsWith("../") ? [relativeIndexPath, `${relativeIndexPath}-shm`, `${relativeIndexPath}-wal`, `${relativeIndexPath}-journal`] : [];
938
+ await this.#git.assertCleanWorkingTree(indexArtifacts);
939
+ const target = await this.#git.resolveCommit(options.target ?? "HEAD");
940
+ const checkpoint = this.#database.getCheckpoint();
941
+ let rebuild = checkpoint === null;
942
+ if (checkpoint && !await this.#git.isAncestor(checkpoint, target)) {
943
+ if (!options.rebuildOnDivergence) throw new GitDivergenceError(checkpoint, target);
944
+ rebuild = true;
945
+ }
946
+ const tree = await this.#git.listTree(target);
947
+ const workingFiles = this.#database.getWorkingTreeFiles();
948
+ const changes = rebuild || !checkpoint ? [] : await this.#git.diff(checkpoint, target);
949
+ const upserts = /* @__PURE__ */ new Map();
950
+ const deletes = /* @__PURE__ */ new Set();
951
+ if (rebuild) {
952
+ for (const existing of this.#database.allFilePaths()) deletes.add(existing);
953
+ for (const entry of tree.values()) {
954
+ if (this.#policy.includes(entry.path) && entry.size <= this.#maxFileSize) upserts.set(entry.path, { entry });
955
+ }
956
+ } else {
957
+ this.#classifyGitChanges(changes, tree, upserts, deletes);
958
+ for (const { path: dirtyPath } of workingFiles) {
959
+ const entry = tree.get(dirtyPath);
960
+ if (entry && this.#policy.includes(dirtyPath) && entry.size <= this.#maxFileSize) upserts.set(dirtyPath, { entry });
961
+ else deletes.add(dirtyPath);
962
+ }
963
+ const eligibleTargetPaths = new Set(
964
+ [...tree.values()].filter((entry) => this.#policy.includes(entry.path) && entry.size <= this.#maxFileSize).map((entry) => entry.path)
965
+ );
966
+ const indexedPaths = new Set(this.#database.allFilePaths());
967
+ for (const targetPath of eligibleTargetPaths) {
968
+ if (!indexedPaths.has(targetPath) && !upserts.has(targetPath)) {
969
+ upserts.set(targetPath, { entry: tree.get(targetPath) });
970
+ }
971
+ }
972
+ for (const indexedPath of indexedPaths) {
973
+ if (!eligibleTargetPaths.has(indexedPath)) deletes.add(indexedPath);
974
+ }
975
+ }
976
+ if (!rebuild && checkpoint === target && upserts.size === 0 && deletes.size === 0) return emptyStats(checkpoint);
977
+ const prepared = [];
978
+ for (const { entry, previousPath } of upserts.values()) {
979
+ throwIfAborted(options.signal);
980
+ const content = await this.#git.readBlob(entry.oid);
981
+ if (content.byteLength > this.#maxFileSize) {
982
+ deletes.add(entry.path);
983
+ continue;
984
+ }
985
+ prepared.push(this.#prepareFile(entry.path, content, {
986
+ blobOid: entry.oid,
987
+ sourceMode: "git",
988
+ indexedCommit: target,
989
+ ...previousPath ? { replacePath: previousPath } : {}
990
+ }));
991
+ }
992
+ await this.#attachEmbeddings(prepared, options.signal);
993
+ const resolvedAgain = await this.#git.resolveCommit(options.target ?? "HEAD");
994
+ if (resolvedAgain !== target) throw new CodeIndexError("Target Git ref changed while indexing; retry the update.");
995
+ return this.#database.applyUpdate({
996
+ files: prepared,
997
+ deletePaths: [...deletes],
998
+ checkpoint: target,
999
+ expectedCheckpoint: checkpoint
1000
+ });
1001
+ }
1002
+ #classifyGitChanges(changes, tree, upserts, deletes) {
1003
+ for (const change of changes) {
1004
+ if (change.status === "D") {
1005
+ deletes.add(change.path);
1006
+ continue;
1007
+ }
1008
+ if (change.status === "R") {
1009
+ deletes.add(change.oldPath);
1010
+ const entry2 = tree.get(change.path);
1011
+ if (entry2 && this.#policy.includes(change.path) && entry2.size <= this.#maxFileSize) {
1012
+ upserts.set(change.path, { entry: entry2, previousPath: change.oldPath });
1013
+ }
1014
+ continue;
1015
+ }
1016
+ const entry = tree.get(change.path);
1017
+ if (entry && this.#policy.includes(change.path) && entry.size <= this.#maxFileSize) upserts.set(change.path, { entry });
1018
+ else deletes.add(change.path);
1019
+ }
1020
+ }
1021
+ #prepareFile(relativePath, buffer, provenance) {
1022
+ const content = buffer.toString("utf8");
1023
+ const callables = parseCallables(relativePath, content, this.#onWarning);
1024
+ const language = callables[0]?.language ?? path6.extname(relativePath).slice(1);
1025
+ return {
1026
+ path: relativePath,
1027
+ contentHash: sha256(content),
1028
+ blobOid: provenance.blobOid,
1029
+ sourceMode: provenance.sourceMode,
1030
+ indexedCommit: provenance.indexedCommit,
1031
+ language,
1032
+ byteSize: buffer.byteLength,
1033
+ ...provenance.previousPath ? { previousPath: provenance.previousPath } : {},
1034
+ ...provenance.replacePath ? { replacePath: provenance.replacePath } : {},
1035
+ callables
1036
+ };
1037
+ }
1038
+ async #attachEmbeddings(files, signal) {
1039
+ const profile = JSON.stringify(normalizeProfile(this.provider.profile));
1040
+ const unique = /* @__PURE__ */ new Map();
1041
+ for (const file of files) {
1042
+ for (const callable of file.callables) {
1043
+ callable.embeddingKey = sha256(`${profile}\0${callable.embeddingInput}`);
1044
+ const values = unique.get(callable.embeddingKey) ?? [];
1045
+ values.push(callable);
1046
+ unique.set(callable.embeddingKey, values);
1047
+ }
1048
+ }
1049
+ const existing = this.#database.getEmbeddingKeys([...unique.keys()]);
1050
+ const missing = [...unique.entries()].filter(([key]) => !existing.has(key));
1051
+ for (const batch of chunk(missing, this.#embeddingBatchSize)) {
1052
+ throwIfAborted(signal);
1053
+ const vectors = await this.provider.embedDocuments(batch.map(([, callables]) => callables[0].embeddingInput), signal ? { signal } : void 0);
1054
+ if (vectors.length !== batch.length) throw new CodeIndexError("Embedding provider returned an unexpected number of vectors.");
1055
+ vectors.forEach((vector, index) => {
1056
+ const converted = normalizeEmbeddingVector(vector, this.provider.profile.dimensions);
1057
+ for (const callable of batch[index][1]) callable.vector = converted;
1058
+ });
1059
+ }
1060
+ }
1061
+ async similaritySearch(options) {
1062
+ if (!options.query.trim()) throw new CodeIndexError("query must not be empty.");
1063
+ const limit = options.limit ?? 10;
1064
+ assertPositiveInteger(limit, "limit");
1065
+ throwIfAborted(options.signal);
1066
+ const vector = await this.provider.embedQuery(options.query, options.signal ? { signal: options.signal } : void 0);
1067
+ return this.searchByVector(vector, {
1068
+ limit,
1069
+ minSimilarity: options.minSimilarity ?? -1,
1070
+ ...options.maxSimilarity !== void 0 ? { maxSimilarity: options.maxSimilarity } : {}
1071
+ });
1072
+ }
1073
+ similarToFunction(functionId, options) {
1074
+ const vector = this.#database.vectorForFunction(functionId);
1075
+ return this.#database.searchVector(vector, {
1076
+ limit: options.limit,
1077
+ minSimilarity: options.minSimilarity,
1078
+ ...options.maxSimilarity !== void 0 ? { maxSimilarity: options.maxSimilarity } : {},
1079
+ excludeId: functionId
1080
+ });
1081
+ }
1082
+ vectorForFunction(functionId) {
1083
+ return this.#database.vectorForFunction(functionId);
1084
+ }
1085
+ searchByVector(vector, options) {
1086
+ return this.#database.searchVector(normalizeEmbeddingVector(vector, this.provider.profile.dimensions), options);
1087
+ }
1088
+ async sourceFunctions(filter) {
1089
+ const functions = filter.type === "all" ? this.allFunctions() : await this.#functionsAddedSince(filter.commit);
1090
+ if (!filter.path) return functions;
1091
+ const sourcePath = normalizeRelativePath(this.rootDir, filter.path);
1092
+ return functions.filter((callable) => callable.path === sourcePath || callable.path.startsWith(`${sourcePath}/`));
1093
+ }
1094
+ async #functionsAddedSince(baseRef) {
1095
+ await this.#git.assertRepository();
1096
+ const checkpoint = this.#database.getCheckpoint();
1097
+ if (!checkpoint) throw new CodeIndexError("added-since requires an index with a Git checkpoint.");
1098
+ const base = await this.#git.resolveCommit(baseRef);
1099
+ if (!await this.#git.isAncestor(base, checkpoint)) {
1100
+ throw new GitDivergenceError(base, checkpoint);
1101
+ }
1102
+ const changes = await this.#git.diff(base, checkpoint);
1103
+ const baseTree = await this.#git.listTree(base);
1104
+ const comparisonPaths = /* @__PURE__ */ new Map();
1105
+ for (const change of changes) {
1106
+ if (change.status === "D") continue;
1107
+ if (change.status === "R") comparisonPaths.set(change.path, change.oldPath);
1108
+ else if (change.status === "A" || change.status === "C") comparisonPaths.set(change.path, null);
1109
+ else comparisonPaths.set(change.path, change.path);
1110
+ }
1111
+ for (const dirtyFile of this.#database.getWorkingTreeFiles()) {
1112
+ if (!comparisonPaths.has(dirtyFile.path)) {
1113
+ comparisonPaths.set(dirtyFile.path, dirtyFile.previousPath ?? dirtyFile.path);
1114
+ }
1115
+ }
1116
+ const current = this.#database.functionsForPaths([...comparisonPaths.keys()]);
1117
+ const baseFunctionsByCurrentPath = /* @__PURE__ */ new Map();
1118
+ for (const [currentPath, basePath] of comparisonPaths) {
1119
+ if (!basePath) {
1120
+ baseFunctionsByCurrentPath.set(currentPath, []);
1121
+ continue;
1122
+ }
1123
+ const entry = baseTree.get(basePath);
1124
+ if (!entry) {
1125
+ baseFunctionsByCurrentPath.set(currentPath, []);
1126
+ continue;
1127
+ }
1128
+ const content = (await this.#git.readBlob(entry.oid)).toString("utf8");
1129
+ baseFunctionsByCurrentPath.set(currentPath, parseCallables(basePath, content, this.#onWarning));
1130
+ }
1131
+ const currentByPath = groupBy(current, (callable) => callable.path);
1132
+ const addedIds = /* @__PURE__ */ new Set();
1133
+ for (const [currentPath, currentFunctions] of currentByPath) {
1134
+ for (const id of addedFunctionIds(currentFunctions, baseFunctionsByCurrentPath.get(currentPath) ?? [])) {
1135
+ addedIds.add(id);
1136
+ }
1137
+ }
1138
+ return current.filter((callable) => addedIds.has(callable.id));
1139
+ }
1140
+ };
1141
+ function addedFunctionIds(current, base) {
1142
+ const currentGroups = groupBy(current, (callable) => `${callable.qualifiedName}\0${callable.kind}`);
1143
+ const baseGroups = groupBy(base, (callable) => `${callable.qualifiedName}\0${callable.kind}`);
1144
+ const added = [];
1145
+ for (const [key, currentGroup] of currentGroups) {
1146
+ const baseGroup = baseGroups.get(key) ?? [];
1147
+ const unmatchedBase = new Set(baseGroup.map((_, index) => index));
1148
+ const unmatchedCurrent = new Set(currentGroup.map((_, index) => index));
1149
+ for (let currentIndex = 0; currentIndex < currentGroup.length; currentIndex += 1) {
1150
+ const exactBaseIndex = [...unmatchedBase].find(
1151
+ (baseIndex) => baseGroup[baseIndex].sourceHash === currentGroup[currentIndex].sourceHash
1152
+ );
1153
+ if (exactBaseIndex !== void 0) {
1154
+ unmatchedBase.delete(exactBaseIndex);
1155
+ unmatchedCurrent.delete(currentIndex);
1156
+ }
1157
+ }
1158
+ for (const currentIndex of unmatchedCurrent) {
1159
+ const baseIndex = unmatchedBase.values().next().value;
1160
+ if (baseIndex === void 0) added.push(currentGroup[currentIndex].id);
1161
+ else unmatchedBase.delete(baseIndex);
1162
+ }
1163
+ }
1164
+ return added;
1165
+ }
1166
+ function normalizeProfile(profile) {
1167
+ return {
1168
+ provider: profile.provider,
1169
+ model: profile.model,
1170
+ dimensions: profile.dimensions,
1171
+ strategyVersion: profile.strategyVersion ?? "callable-v1"
1172
+ };
1173
+ }
1174
+ function emptyStats(checkpoint) {
1175
+ return {
1176
+ filesUpdated: 0,
1177
+ filesDeleted: 0,
1178
+ functionsAdded: 0,
1179
+ functionsUpdated: 0,
1180
+ functionsDeleted: 0,
1181
+ embeddingsCreated: 0,
1182
+ checkpoint
1183
+ };
1184
+ }
1185
+ function validateRenames(renames) {
1186
+ const sources = new Set(renames.map((rename) => rename.from));
1187
+ const targets = new Set(renames.map((rename) => rename.to));
1188
+ if (sources.size !== renames.length || targets.size !== renames.length) {
1189
+ throw new CodeIndexError("Rename sources and targets must be unique.");
1190
+ }
1191
+ if ([...sources].some((source) => targets.has(source))) {
1192
+ throw new CodeIndexError("Chained or cyclic renames must be applied in separate updates.");
1193
+ }
1194
+ }
1195
+ function groupBy(values, keyFor) {
1196
+ const groups = /* @__PURE__ */ new Map();
1197
+ for (const value of values) {
1198
+ const key = keyFor(value);
1199
+ const group = groups.get(key) ?? [];
1200
+ group.push(value);
1201
+ groups.set(key, group);
1202
+ }
1203
+ return groups;
1204
+ }
1205
+
1206
+ // src/search/cross-search.ts
1207
+ async function* crossSearch(options) {
1208
+ const target = options.target ?? options.source;
1209
+ const sourceProfile = JSON.stringify(options.source.status().embeddingProfile);
1210
+ const targetProfile = JSON.stringify(target.status().embeddingProfile);
1211
+ if (sourceProfile !== targetProfile) {
1212
+ throw new IncompatibleIndexError("Cross-search requires identical embedding profiles.");
1213
+ }
1214
+ const limit = options.limitPerFunction ?? 5;
1215
+ assertPositiveInteger(limit, "limitPerFunction");
1216
+ const sourceFunctions = await options.source.sourceFunctions(options.sourceFilter ?? { type: "all" });
1217
+ const sameIndex = target.indexPath === options.source.indexPath;
1218
+ const seenPairs = /* @__PURE__ */ new Set();
1219
+ for (let index = 0; index < sourceFunctions.length; index += 1) {
1220
+ throwIfAborted(options.signal);
1221
+ const source = sourceFunctions[index];
1222
+ const vector = options.source.vectorForFunction(source.id);
1223
+ const candidates = sameIndex ? options.source.similarToFunction(source.id, {
1224
+ limit,
1225
+ minSimilarity: options.minSimilarity ?? -1,
1226
+ ...options.maxSimilarity !== void 0 ? { maxSimilarity: options.maxSimilarity } : {}
1227
+ }) : target.searchByVector(vector, {
1228
+ limit,
1229
+ minSimilarity: options.minSimilarity ?? -1,
1230
+ ...options.maxSimilarity !== void 0 ? { maxSimilarity: options.maxSimilarity } : {}
1231
+ });
1232
+ const matches = sameIndex && !options.includeSymmetricDuplicates ? candidates.filter((match) => {
1233
+ const pair = source.id < match.function.id ? `${source.id}:${match.function.id}` : `${match.function.id}:${source.id}`;
1234
+ if (seenPairs.has(pair)) return false;
1235
+ seenPairs.add(pair);
1236
+ return true;
1237
+ }) : candidates;
1238
+ if (matches.length > 0) yield { source, matches };
1239
+ options.onProgress?.({ completed: index + 1, total: sourceFunctions.length });
1240
+ }
1241
+ }
1242
+
1243
+ // src/embeddings/openai.ts
1244
+ import { Tiktoken } from "js-tiktoken/lite";
1245
+ import cl100kBase from "js-tiktoken/ranks/cl100k_base";
1246
+
1247
+ // src/embeddings/http.ts
1248
+ async function requestEmbeddings(options) {
1249
+ let lastError;
1250
+ for (let attempt = 0; attempt < 3; attempt += 1) {
1251
+ try {
1252
+ const response = await fetch(options.url, {
1253
+ method: "POST",
1254
+ headers: {
1255
+ authorization: `Bearer ${options.apiKey}`,
1256
+ "content-type": "application/json"
1257
+ },
1258
+ body: JSON.stringify(options.body),
1259
+ ...options.signal ? { signal: options.signal } : {}
1260
+ });
1261
+ if (!response.ok) {
1262
+ const detail = (await response.text()).slice(0, 500);
1263
+ const error = new CodeIndexError(`Embedding request failed (${response.status}): ${detail}`);
1264
+ if (response.status !== 429 && response.status < 500) throw error;
1265
+ lastError = error;
1266
+ } else {
1267
+ const payload = await response.json();
1268
+ if (!Array.isArray(payload.data) || payload.data.length !== options.expectedCount) {
1269
+ throw new CodeIndexError("Embedding provider returned an unexpected number of vectors.");
1270
+ }
1271
+ const indexes = payload.data.map((item) => item.index);
1272
+ if (indexes.some((index) => !Number.isInteger(index)) || new Set(indexes).size !== options.expectedCount || indexes.some((index) => index < 0 || index >= options.expectedCount)) {
1273
+ throw new CodeIndexError("Embedding provider returned invalid vector indexes.");
1274
+ }
1275
+ const ordered = new Array(options.expectedCount);
1276
+ for (const item of payload.data) ordered[item.index] = item;
1277
+ try {
1278
+ return ordered.map((item) => normalizeEmbeddingVector(item.embedding, options.dimensions));
1279
+ } catch {
1280
+ throw new CodeIndexError("Embedding provider returned a malformed vector.");
1281
+ }
1282
+ }
1283
+ } catch (error) {
1284
+ if (options.signal?.aborted) throw error;
1285
+ lastError = error;
1286
+ if (error instanceof CodeIndexError && !/\((429|5\d\d)\)/.test(error.message)) throw error;
1287
+ }
1288
+ await new Promise((resolve) => setTimeout(resolve, 250 * 2 ** attempt));
1289
+ }
1290
+ throw new CodeIndexError("Embedding request failed after three attempts.", { cause: lastError });
1291
+ }
1292
+
1293
+ // src/embeddings/openai.ts
1294
+ var MAX_INPUT_TOKENS = 8192;
1295
+ var tokenizer;
1296
+ function truncateInput(input) {
1297
+ tokenizer ??= new Tiktoken(cl100kBase);
1298
+ const tokens = tokenizer.encode(input);
1299
+ return tokens.length <= MAX_INPUT_TOKENS ? input : tokenizer.decode(tokens.slice(0, MAX_INPUT_TOKENS));
1300
+ }
1301
+ var OpenAIEmbeddingProvider = class {
1302
+ profile;
1303
+ #apiKey;
1304
+ #url;
1305
+ constructor(options = {}) {
1306
+ this.#apiKey = options.apiKey ?? process.env.OPENAI_API_KEY ?? "";
1307
+ if (!this.#apiKey) throw new Error("OPENAI_API_KEY is required.");
1308
+ const model = options.model ?? "text-embedding-3-small";
1309
+ const dimensions = options.dimensions ?? 1536;
1310
+ if (!Number.isInteger(dimensions) || dimensions < 1) throw new Error("dimensions must be a positive integer.");
1311
+ this.profile = { provider: "openai", model, dimensions, strategyVersion: "callable-v1" };
1312
+ this.#url = `${(options.baseUrl ?? "https://api.openai.com/v1").replace(/\/$/, "")}/embeddings`;
1313
+ }
1314
+ embedDocuments(inputs, options) {
1315
+ if (inputs.length === 0) return Promise.resolve([]);
1316
+ return this.#embed(inputs, options?.signal);
1317
+ }
1318
+ async embedQuery(input, options) {
1319
+ return (await this.#embed([input], options?.signal))[0];
1320
+ }
1321
+ #embed(inputs, signal) {
1322
+ return requestEmbeddings({
1323
+ url: this.#url,
1324
+ apiKey: this.#apiKey,
1325
+ body: {
1326
+ model: this.profile.model,
1327
+ dimensions: this.profile.dimensions,
1328
+ encoding_format: "float",
1329
+ input: inputs.map(truncateInput)
1330
+ },
1331
+ expectedCount: inputs.length,
1332
+ dimensions: this.profile.dimensions,
1333
+ ...signal ? { signal } : {}
1334
+ });
1335
+ }
1336
+ };
1337
+
1338
+ // src/embeddings/jina.ts
1339
+ var JinaEmbeddingProvider = class {
1340
+ profile;
1341
+ #apiKey;
1342
+ #url;
1343
+ constructor(options = {}) {
1344
+ this.#apiKey = options.apiKey ?? process.env.JINA_API_KEY ?? "";
1345
+ if (!this.#apiKey) throw new Error("JINA_API_KEY is required.");
1346
+ const model = options.model ?? "jina-embeddings-v4";
1347
+ const dimensions = options.dimensions ?? 1024;
1348
+ if (!Number.isInteger(dimensions) || dimensions < 1) throw new Error("dimensions must be a positive integer.");
1349
+ this.profile = {
1350
+ provider: "jina",
1351
+ model,
1352
+ dimensions,
1353
+ strategyVersion: "callable-v1:code-query-passage"
1354
+ };
1355
+ this.#url = options.baseUrl ?? "https://api.jina.ai/v1/embeddings";
1356
+ }
1357
+ embedDocuments(inputs, options) {
1358
+ if (inputs.length === 0) return Promise.resolve([]);
1359
+ return this.#embed(inputs, "code.passage", options?.signal);
1360
+ }
1361
+ async embedQuery(input, options) {
1362
+ return (await this.#embed([input], "code.query", options?.signal))[0];
1363
+ }
1364
+ #embed(inputs, task, signal) {
1365
+ return requestEmbeddings({
1366
+ url: this.#url,
1367
+ apiKey: this.#apiKey,
1368
+ body: {
1369
+ model: this.profile.model,
1370
+ dimensions: this.profile.dimensions,
1371
+ embedding_type: "float",
1372
+ truncate: true,
1373
+ task,
1374
+ input: inputs
1375
+ },
1376
+ expectedCount: inputs.length,
1377
+ dimensions: this.profile.dimensions,
1378
+ ...signal ? { signal } : {}
1379
+ });
1380
+ }
1381
+ };
1382
+
1383
+ // src/index.ts
1384
+ function openCodeIndex(options) {
1385
+ return new CodeIndex(options);
1386
+ }
1387
+ function updateFiles(index, options) {
1388
+ return index.updateFiles(options);
1389
+ }
1390
+ function updateFromGit(index, options) {
1391
+ return index.updateFromGit(options);
1392
+ }
1393
+ function similaritySearch(index, options) {
1394
+ return index.similaritySearch(options);
1395
+ }
1396
+ function crossSearchFunctions(options) {
1397
+ return crossSearch(options);
1398
+ }
1399
+ export {
1400
+ CodeIndex,
1401
+ CodeIndexError,
1402
+ GitDivergenceError,
1403
+ IncompatibleIndexError,
1404
+ JinaEmbeddingProvider,
1405
+ OpenAIEmbeddingProvider,
1406
+ crossSearch,
1407
+ crossSearchFunctions,
1408
+ openCodeIndex,
1409
+ similaritySearch,
1410
+ updateFiles,
1411
+ updateFromGit
1412
+ };
1413
+ //# sourceMappingURL=index.js.map