@ninjaxtools/slopdex 0.15.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/slopdex/SKILL.md +12 -10
- package/README.md +91 -224
- package/dist/chunk-6LHZCHEB.js +1112 -0
- package/dist/chunk-6LHZCHEB.js.map +1 -0
- package/dist/chunk-BXWO2KMC.js +20 -0
- package/dist/chunk-BXWO2KMC.js.map +1 -0
- package/dist/chunk-DKRB2XT5.js +33 -0
- package/dist/chunk-DKRB2XT5.js.map +1 -0
- package/dist/chunk-DTI7SXGL.js +109 -0
- package/dist/chunk-DTI7SXGL.js.map +1 -0
- package/dist/chunk-IQU3YVZ3.js +85 -0
- package/dist/chunk-IQU3YVZ3.js.map +1 -0
- package/dist/chunk-JAVPHCZH.js +24 -0
- package/dist/chunk-JAVPHCZH.js.map +1 -0
- package/dist/chunk-KQRS5P4U.js +10 -0
- package/dist/chunk-KQRS5P4U.js.map +1 -0
- package/dist/chunk-KUV6PKKL.js +425 -0
- package/dist/chunk-KUV6PKKL.js.map +1 -0
- package/dist/chunk-MKUTTXZB.js +74 -0
- package/dist/chunk-MKUTTXZB.js.map +1 -0
- package/dist/chunk-N6D66ASM.js +2217 -0
- package/dist/chunk-N6D66ASM.js.map +1 -0
- package/dist/chunk-OIKBO3NJ.js +133 -0
- package/dist/chunk-OIKBO3NJ.js.map +1 -0
- package/dist/chunk-S225GYCL.js +79 -0
- package/dist/chunk-S225GYCL.js.map +1 -0
- package/dist/chunk-TSURHRFF.js +240 -0
- package/dist/chunk-TSURHRFF.js.map +1 -0
- package/dist/chunk-VH5VGRCI.js +103 -0
- package/dist/chunk-VH5VGRCI.js.map +1 -0
- package/dist/cli.js +218 -3590
- package/dist/cli.js.map +1 -1
- package/dist/code-index-YEFTNG7E.js +14 -0
- package/dist/code-index-YEFTNG7E.js.map +1 -0
- package/dist/cross-search-FVKDMGP7.js +10 -0
- package/dist/cross-search-FVKDMGP7.js.map +1 -0
- package/dist/database-KAPV2YMP.js +14 -0
- package/dist/database-KAPV2YMP.js.map +1 -0
- package/dist/hosted-GHIUHKOU.js +13 -0
- package/dist/hosted-GHIUHKOU.js.map +1 -0
- package/dist/index.d.ts +102 -2
- package/dist/index.js +38 -3615
- package/dist/index.js.map +1 -1
- package/dist/jina-43RL7C6M.js +12 -0
- package/dist/jina-43RL7C6M.js.map +1 -0
- package/dist/openai-CYUPNBGB.js +12 -0
- package/dist/openai-CYUPNBGB.js.map +1 -0
- package/dist/openai-EC6THXKV.js +21 -0
- package/dist/openai-EC6THXKV.js.map +1 -0
- package/dist/openai-TCRXM66O.js +11 -0
- package/dist/openai-TCRXM66O.js.map +1 -0
- package/docs/implementation.md +10 -19
- package/docs/reference.md +187 -0
- package/package.json +4 -3
|
@@ -0,0 +1,2217 @@
|
|
|
1
|
+
import {
|
|
2
|
+
IndexDatabase
|
|
3
|
+
} from "./chunk-6LHZCHEB.js";
|
|
4
|
+
import {
|
|
5
|
+
OpenAIDescriptionProvider,
|
|
6
|
+
isDescriptionProviderName
|
|
7
|
+
} from "./chunk-TSURHRFF.js";
|
|
8
|
+
import {
|
|
9
|
+
analysisSimilarity
|
|
10
|
+
} from "./chunk-KQRS5P4U.js";
|
|
11
|
+
import {
|
|
12
|
+
DEFAULT_PARALLELISM,
|
|
13
|
+
assertPositiveInteger,
|
|
14
|
+
chunk,
|
|
15
|
+
compileNameRegex,
|
|
16
|
+
forEachConcurrent,
|
|
17
|
+
normalizeEmbeddingVector,
|
|
18
|
+
normalizeRelativePath,
|
|
19
|
+
sha256,
|
|
20
|
+
throwIfAborted
|
|
21
|
+
} from "./chunk-VH5VGRCI.js";
|
|
22
|
+
import {
|
|
23
|
+
CodeIndexError,
|
|
24
|
+
GitDivergenceError,
|
|
25
|
+
GitUnavailableError
|
|
26
|
+
} from "./chunk-DKRB2XT5.js";
|
|
27
|
+
|
|
28
|
+
// src/code-index.ts
|
|
29
|
+
import { lstat as lstat3, readFile as readFile2, readdir } from "node:fs/promises";
|
|
30
|
+
import { lstatSync, readFileSync } from "node:fs";
|
|
31
|
+
import path5 from "node:path";
|
|
32
|
+
|
|
33
|
+
// src/git/repository.ts
|
|
34
|
+
import { spawn } from "node:child_process";
|
|
35
|
+
import { lstat } from "node:fs/promises";
|
|
36
|
+
import path from "node:path";
|
|
37
|
+
var GitRepository = class {
|
|
38
|
+
constructor(rootDir) {
|
|
39
|
+
this.rootDir = rootDir;
|
|
40
|
+
}
|
|
41
|
+
rootDir;
|
|
42
|
+
async #run(args, allowExitOne = false) {
|
|
43
|
+
return await new Promise((resolve, reject) => {
|
|
44
|
+
const child = spawn("git", ["-C", this.rootDir, ...args], { stdio: ["ignore", "pipe", "pipe"] });
|
|
45
|
+
const stdout = [];
|
|
46
|
+
const stderr = [];
|
|
47
|
+
child.stdout.on("data", (value) => stdout.push(value));
|
|
48
|
+
child.stderr.on("data", (value) => stderr.push(value));
|
|
49
|
+
child.on("error", (error) => {
|
|
50
|
+
if (error.code === "ENOENT") {
|
|
51
|
+
reject(new GitUnavailableError("Git executable is not available.", { cause: error }));
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
reject(error);
|
|
55
|
+
});
|
|
56
|
+
child.on("close", (code) => {
|
|
57
|
+
const exitCode = code ?? -1;
|
|
58
|
+
if (exitCode === 0 || allowExitOne && exitCode === 1) {
|
|
59
|
+
resolve({ stdout: Buffer.concat(stdout), code: exitCode });
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
reject(new CodeIndexError(`Git command failed: ${Buffer.concat(stderr).toString("utf8").trim()}`));
|
|
63
|
+
});
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
async assertRepository() {
|
|
67
|
+
let result;
|
|
68
|
+
try {
|
|
69
|
+
result = await this.#run(["rev-parse", "--is-inside-work-tree"]);
|
|
70
|
+
} catch (error) {
|
|
71
|
+
if (error instanceof GitUnavailableError) throw error;
|
|
72
|
+
try {
|
|
73
|
+
await lstat(path.join(this.rootDir, ".git"));
|
|
74
|
+
} catch (markerError) {
|
|
75
|
+
if (markerError.code === "ENOENT") {
|
|
76
|
+
throw new GitUnavailableError(`${this.rootDir} is not a Git worktree.`, { cause: error });
|
|
77
|
+
}
|
|
78
|
+
throw markerError;
|
|
79
|
+
}
|
|
80
|
+
throw error;
|
|
81
|
+
}
|
|
82
|
+
if (result.stdout.toString("utf8").trim() !== "true") {
|
|
83
|
+
throw new GitUnavailableError(`${this.rootDir} is not a Git worktree.`);
|
|
84
|
+
}
|
|
85
|
+
const topLevel = (await this.#run(["rev-parse", "--show-toplevel"])).stdout.toString("utf8").trim();
|
|
86
|
+
if (path.resolve(topLevel) !== path.resolve(this.rootDir)) {
|
|
87
|
+
throw new CodeIndexError(`rootDir must be the Git worktree root: ${topLevel}`);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
async workTreeChanges(base, excludePaths = []) {
|
|
91
|
+
const pathspecs = [".", ...excludePaths.map((filePath) => `:(top,literal,exclude)${filePath}`)];
|
|
92
|
+
const result = await this.#run([
|
|
93
|
+
"diff",
|
|
94
|
+
"--name-status",
|
|
95
|
+
"-z",
|
|
96
|
+
"--find-renames",
|
|
97
|
+
base,
|
|
98
|
+
"--",
|
|
99
|
+
...pathspecs
|
|
100
|
+
]);
|
|
101
|
+
const changes = parseChanges(splitNull(result.stdout));
|
|
102
|
+
const untracked = await this.#run([
|
|
103
|
+
"ls-files",
|
|
104
|
+
"--others",
|
|
105
|
+
"--exclude-standard",
|
|
106
|
+
"-z",
|
|
107
|
+
"--",
|
|
108
|
+
...pathspecs
|
|
109
|
+
]);
|
|
110
|
+
for (const filePath of splitNull(untracked.stdout)) changes.push({ status: "A", path: filePath });
|
|
111
|
+
return changes;
|
|
112
|
+
}
|
|
113
|
+
async resolveCommit(ref) {
|
|
114
|
+
const result = await this.#run(["rev-parse", "--verify", "--end-of-options", `${ref}^{commit}`]);
|
|
115
|
+
const oid = result.stdout.toString("ascii").trim();
|
|
116
|
+
if (!/^[0-9a-f]{40,64}$/.test(oid)) throw new CodeIndexError(`Could not resolve Git commit: ${ref}`);
|
|
117
|
+
return oid;
|
|
118
|
+
}
|
|
119
|
+
async isAncestor(ancestor, descendant) {
|
|
120
|
+
const result = await this.#run(["merge-base", "--is-ancestor", ancestor, descendant], true);
|
|
121
|
+
return result.code === 0;
|
|
122
|
+
}
|
|
123
|
+
async listTree(commit) {
|
|
124
|
+
const result = await this.#run(["ls-tree", "-r", "-z", "-l", "--full-tree", commit, "--"]);
|
|
125
|
+
const entries = /* @__PURE__ */ new Map();
|
|
126
|
+
for (const record of splitNull(result.stdout)) {
|
|
127
|
+
const tab = record.indexOf(" ");
|
|
128
|
+
if (tab < 0) continue;
|
|
129
|
+
const [mode, type, oid, sizeText] = record.slice(0, tab).trim().split(/\s+/);
|
|
130
|
+
const filePath = record.slice(tab + 1);
|
|
131
|
+
const size = Number(sizeText);
|
|
132
|
+
if ((mode === "100644" || mode === "100755") && type === "blob" && oid && Number.isSafeInteger(size)) {
|
|
133
|
+
entries.set(filePath, { path: filePath, oid, mode, size });
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
return entries;
|
|
137
|
+
}
|
|
138
|
+
async readBlob(oid) {
|
|
139
|
+
return (await this.#run(["cat-file", "blob", oid])).stdout;
|
|
140
|
+
}
|
|
141
|
+
async diff(base, target) {
|
|
142
|
+
const result = await this.#run(["diff", "--name-status", "-z", "--find-renames", `${base}..${target}`, "--"]);
|
|
143
|
+
return parseChanges(splitNull(result.stdout));
|
|
144
|
+
}
|
|
145
|
+
};
|
|
146
|
+
function parseChanges(values) {
|
|
147
|
+
const changes = [];
|
|
148
|
+
for (let index = 0; index < values.length; ) {
|
|
149
|
+
const rawStatus = values[index++];
|
|
150
|
+
if (!rawStatus) continue;
|
|
151
|
+
const status = rawStatus[0];
|
|
152
|
+
if (status === "R" || status === "C") {
|
|
153
|
+
const oldPath = values[index++];
|
|
154
|
+
const filePath = values[index++];
|
|
155
|
+
if (oldPath && filePath) changes.push({ status, oldPath, path: filePath });
|
|
156
|
+
} else if (status === "A" || status === "M" || status === "D" || status === "T") {
|
|
157
|
+
const filePath = values[index++];
|
|
158
|
+
if (filePath) changes.push({ status, path: filePath });
|
|
159
|
+
} else {
|
|
160
|
+
throw new CodeIndexError(`Unsupported Git diff status: ${rawStatus}`);
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
return changes;
|
|
164
|
+
}
|
|
165
|
+
function splitNull(buffer) {
|
|
166
|
+
const value = buffer.toString("utf8");
|
|
167
|
+
const parts = value.split("\0");
|
|
168
|
+
if (parts.at(-1) === "") parts.pop();
|
|
169
|
+
return parts;
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
// src/gitignore.ts
|
|
173
|
+
import { lstat as lstat2, readFile } from "node:fs/promises";
|
|
174
|
+
import path2 from "node:path";
|
|
175
|
+
import ignore from "ignore";
|
|
176
|
+
var GitignoreRules = class _GitignoreRules {
|
|
177
|
+
constructor(readIgnore) {
|
|
178
|
+
this.readIgnore = readIgnore;
|
|
179
|
+
}
|
|
180
|
+
readIgnore;
|
|
181
|
+
#directories = /* @__PURE__ */ new Map();
|
|
182
|
+
#contents = /* @__PURE__ */ new Map();
|
|
183
|
+
static workingTree(rootDir) {
|
|
184
|
+
return new _GitignoreRules(async (relativePath) => {
|
|
185
|
+
const absolutePath = path2.join(rootDir, relativePath);
|
|
186
|
+
try {
|
|
187
|
+
const info = await lstat2(absolutePath);
|
|
188
|
+
if (!info.isFile() || info.isSymbolicLink()) return null;
|
|
189
|
+
return await readFile(absolutePath, "utf8");
|
|
190
|
+
} catch (error) {
|
|
191
|
+
if (["ENOENT", "ENOTDIR"].includes(error.code ?? "")) return null;
|
|
192
|
+
throw error;
|
|
193
|
+
}
|
|
194
|
+
});
|
|
195
|
+
}
|
|
196
|
+
async ignores(relativePath, directory = false) {
|
|
197
|
+
const parent = path2.posix.dirname(relativePath);
|
|
198
|
+
const rules = await this.#directory(parent === "." ? "" : parent);
|
|
199
|
+
return rules === null || rules.ignores(directory ? `${relativePath}/` : relativePath);
|
|
200
|
+
}
|
|
201
|
+
async assertUnchanged(signal) {
|
|
202
|
+
for (const [filePath, content] of this.#contents) {
|
|
203
|
+
throwIfAborted(signal);
|
|
204
|
+
if (await this.readIgnore(filePath) !== content) {
|
|
205
|
+
throw new CodeIndexError(`Gitignore rules changed while indexing: ${filePath}; retry the update.`);
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
#directory(directory) {
|
|
210
|
+
let result = this.#directories.get(directory);
|
|
211
|
+
if (!result) {
|
|
212
|
+
result = this.#loadDirectory(directory);
|
|
213
|
+
this.#directories.set(directory, result);
|
|
214
|
+
}
|
|
215
|
+
return result;
|
|
216
|
+
}
|
|
217
|
+
async #loadDirectory(directory) {
|
|
218
|
+
let parentRules;
|
|
219
|
+
if (directory) {
|
|
220
|
+
const parent = path2.posix.dirname(directory);
|
|
221
|
+
const rules2 = await this.#directory(parent === "." ? "" : parent);
|
|
222
|
+
if (rules2 === null || rules2.ignores(`${directory}/`)) return null;
|
|
223
|
+
parentRules = rules2;
|
|
224
|
+
}
|
|
225
|
+
const filePath = directory ? `${directory}/.gitignore` : ".gitignore";
|
|
226
|
+
const content = await this.readIgnore(filePath);
|
|
227
|
+
this.#contents.set(filePath, content);
|
|
228
|
+
if (content === null && parentRules) return parentRules;
|
|
229
|
+
const rules = ignore({ ignorecase: false });
|
|
230
|
+
if (parentRules) rules.add(parentRules);
|
|
231
|
+
if (content !== null) rules.add(directory ? scopedPatterns(content, directory) : content);
|
|
232
|
+
return rules;
|
|
233
|
+
}
|
|
234
|
+
};
|
|
235
|
+
function scopedPatterns(content, directory) {
|
|
236
|
+
const prefix = directory.replace(/[\\*?\[\]#!]/g, "\\$&");
|
|
237
|
+
return content.split(/\r?\n/).map((line) => {
|
|
238
|
+
const pattern = line.replace(/^\uFEFF/, "");
|
|
239
|
+
if (/^ *$/.test(pattern) || pattern.startsWith("#")) return pattern;
|
|
240
|
+
const negative = pattern.startsWith("!");
|
|
241
|
+
const body = negative ? pattern.slice(1) : pattern;
|
|
242
|
+
if (/^\/? *$/.test(body)) return "";
|
|
243
|
+
const anchored = body.startsWith("/") || body.replace(/\/ *$/, "").includes("/");
|
|
244
|
+
return `${negative ? "!" : ""}/${prefix}/${anchored ? "" : "**/"}${body.replace(/^\//, "")}`;
|
|
245
|
+
});
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
// src/parser/callable-parser.ts
|
|
249
|
+
import path3 from "node:path";
|
|
250
|
+
import Parser from "tree-sitter";
|
|
251
|
+
import C from "tree-sitter-c";
|
|
252
|
+
import Go from "tree-sitter-go";
|
|
253
|
+
import Java from "tree-sitter-java";
|
|
254
|
+
import JavaScript from "tree-sitter-javascript";
|
|
255
|
+
import Python from "tree-sitter-python";
|
|
256
|
+
import Rust from "tree-sitter-rust";
|
|
257
|
+
import TypeScript from "tree-sitter-typescript";
|
|
258
|
+
|
|
259
|
+
// src/parser/native-callables.ts
|
|
260
|
+
function cFunctionName(node) {
|
|
261
|
+
if (!node) return null;
|
|
262
|
+
if (node.type === "identifier") return node.text;
|
|
263
|
+
return cFunctionName(node.childForFieldName("declarator") ?? (node.type === "parenthesized_declarator" ? node.namedChildren[0] ?? null : null));
|
|
264
|
+
}
|
|
265
|
+
function containsYield(node) {
|
|
266
|
+
if (node.type === "yield") return true;
|
|
267
|
+
if (["function_definition", "class_definition", "lambda"].includes(node.type)) return false;
|
|
268
|
+
return node.namedChildren.some(containsYield);
|
|
269
|
+
}
|
|
270
|
+
function unwrap(node) {
|
|
271
|
+
while (node?.type === "parenthesized_expression" && node.namedChildCount === 1) {
|
|
272
|
+
node = node.namedChildren[0];
|
|
273
|
+
}
|
|
274
|
+
return node;
|
|
275
|
+
}
|
|
276
|
+
function pythonIsMethod(node) {
|
|
277
|
+
const definition = node.parent?.type === "decorated_definition" ? node.parent : node;
|
|
278
|
+
return definition.parent?.parent?.type === "class_definition";
|
|
279
|
+
}
|
|
280
|
+
function pythonDocstring(body, content) {
|
|
281
|
+
const statement = body?.namedChildren[0];
|
|
282
|
+
if (statement?.type !== "expression_statement") return null;
|
|
283
|
+
const value = statement.namedChildren[0];
|
|
284
|
+
if (value?.type !== "string" && value?.type !== "concatenated_string") return null;
|
|
285
|
+
return content.slice(value.startIndex, value.endIndex);
|
|
286
|
+
}
|
|
287
|
+
function goReceiver(node) {
|
|
288
|
+
let type = node.childForFieldName("receiver")?.namedChildren[0]?.childForFieldName("type");
|
|
289
|
+
while (type?.type === "pointer_type") type = type.namedChildren[0] ?? null;
|
|
290
|
+
return type?.text ?? null;
|
|
291
|
+
}
|
|
292
|
+
function collectNativeCallables(root, content, language) {
|
|
293
|
+
const candidates = [];
|
|
294
|
+
const text = (node) => content.slice(node.startIndex, node.endIndex);
|
|
295
|
+
const add = (node, name, kind, scope, bound = false) => {
|
|
296
|
+
const body = node.childForFieldName("body");
|
|
297
|
+
if ((!body || body.isMissing) && !node.hasError) return false;
|
|
298
|
+
const sourceNode = language === "python" && node.parent?.type === "decorated_definition" ? node.parent : node;
|
|
299
|
+
const header = content.slice(node.startIndex, body?.startIndex ?? node.endIndex).trimEnd();
|
|
300
|
+
const documentation = language === "python" && body ? pythonDocstring(body, content) : null;
|
|
301
|
+
candidates.push({
|
|
302
|
+
node: sourceNode,
|
|
303
|
+
name,
|
|
304
|
+
kind,
|
|
305
|
+
scope,
|
|
306
|
+
signature: bound ? `${name} = ${header}` : header,
|
|
307
|
+
...documentation ? { documentation } : {}
|
|
308
|
+
});
|
|
309
|
+
if (body) walk2(body, [...scope, name]);
|
|
310
|
+
return true;
|
|
311
|
+
};
|
|
312
|
+
const addBound = (value, name, scope) => {
|
|
313
|
+
const callable = unwrap(value);
|
|
314
|
+
if (!callable || !name || !["identifier", "attribute", "selector_expression"].includes(name.type)) return false;
|
|
315
|
+
const closureType = { python: "lambda", rust: "closure_expression", go: "func_literal", java: "lambda_expression" };
|
|
316
|
+
if (!(language in closureType) || callable.type !== closureType[language]) return false;
|
|
317
|
+
return add(callable, text(name), "function", scope, true);
|
|
318
|
+
};
|
|
319
|
+
const walk2 = (node, scope) => {
|
|
320
|
+
if (language === "python") {
|
|
321
|
+
if (node.type === "class_definition") {
|
|
322
|
+
const name = node.childForFieldName("name");
|
|
323
|
+
const body = node.childForFieldName("body");
|
|
324
|
+
if (body) walk2(body, name ? [...scope, text(name)] : scope);
|
|
325
|
+
return;
|
|
326
|
+
}
|
|
327
|
+
if (node.type === "function_definition") {
|
|
328
|
+
const name = node.childForFieldName("name");
|
|
329
|
+
const body = node.childForFieldName("body");
|
|
330
|
+
const method = pythonIsMethod(node);
|
|
331
|
+
const kind = method && ["__init__", "__new__"].includes(name?.text ?? "") ? "constructor" : body && containsYield(body) ? "generator" : method ? "method" : "function";
|
|
332
|
+
if (name) add(node, text(name), kind, scope);
|
|
333
|
+
return;
|
|
334
|
+
}
|
|
335
|
+
if (node.type === "assignment" || node.type === "named_expression") {
|
|
336
|
+
if (addBound(
|
|
337
|
+
node.childForFieldName("right") ?? node.childForFieldName("value"),
|
|
338
|
+
node.childForFieldName("left") ?? node.childForFieldName("name"),
|
|
339
|
+
scope
|
|
340
|
+
)) return;
|
|
341
|
+
}
|
|
342
|
+
} else if (language === "rust") {
|
|
343
|
+
if (["mod_item", "trait_item", "impl_item"].includes(node.type)) {
|
|
344
|
+
const body = node.childForFieldName("body");
|
|
345
|
+
const type = node.childForFieldName("type");
|
|
346
|
+
const trait = node.childForFieldName("trait");
|
|
347
|
+
const name = node.type === "impl_item" ? type && (trait ? `<${text(type)} as ${text(trait)}>` : text(type)) : node.childForFieldName("name")?.text;
|
|
348
|
+
if (body) walk2(body, name ? [...scope, name] : scope);
|
|
349
|
+
return;
|
|
350
|
+
}
|
|
351
|
+
if (node.type === "function_item") {
|
|
352
|
+
const name = node.childForFieldName("name");
|
|
353
|
+
const owner = node.parent?.parent?.type;
|
|
354
|
+
if (name) add(node, text(name), owner === "impl_item" || owner === "trait_item" ? "method" : "function", scope);
|
|
355
|
+
return;
|
|
356
|
+
}
|
|
357
|
+
if (node.type === "let_declaration" && addBound(node.childForFieldName("value"), node.childForFieldName("pattern"), scope)) return;
|
|
358
|
+
} else if (language === "go") {
|
|
359
|
+
if (node.type === "function_declaration" || node.type === "method_declaration") {
|
|
360
|
+
const name = node.childForFieldName("name");
|
|
361
|
+
const receiver = goReceiver(node);
|
|
362
|
+
if (name) add(node, text(name), receiver ? "method" : "function", receiver ? [...scope, receiver] : scope);
|
|
363
|
+
return;
|
|
364
|
+
}
|
|
365
|
+
if (["short_var_declaration", "assignment_statement", "var_spec"].includes(node.type)) {
|
|
366
|
+
const names = node.type === "var_spec" ? node.childrenForFieldName("name") : node.childForFieldName("left")?.namedChildren ?? [];
|
|
367
|
+
const values = (node.childForFieldName("right") ?? node.childForFieldName("value"))?.namedChildren ?? [];
|
|
368
|
+
if (names.length === values.length) {
|
|
369
|
+
for (let i = 0; i < names.length; i += 1) {
|
|
370
|
+
if (!addBound(values[i], names[i], scope)) walk2(values[i], scope);
|
|
371
|
+
}
|
|
372
|
+
return;
|
|
373
|
+
}
|
|
374
|
+
}
|
|
375
|
+
} else if (language === "java") {
|
|
376
|
+
if (["class_declaration", "interface_declaration", "enum_declaration", "record_declaration", "annotation_type_declaration", "enum_constant"].includes(node.type)) {
|
|
377
|
+
const name = node.childForFieldName("name");
|
|
378
|
+
for (const child of node.namedChildren) walk2(child, name ? [...scope, text(name)] : scope);
|
|
379
|
+
return;
|
|
380
|
+
}
|
|
381
|
+
if (node.type === "class_body" && node.parent?.type === "object_creation_expression") {
|
|
382
|
+
const anonymousScope = [...scope, `<anonymous@${node.startPosition.row + 1}:${node.startPosition.column + 1}>`];
|
|
383
|
+
for (const child of node.namedChildren) walk2(child, anonymousScope);
|
|
384
|
+
return;
|
|
385
|
+
}
|
|
386
|
+
if (["method_declaration", "constructor_declaration", "compact_constructor_declaration"].includes(node.type)) {
|
|
387
|
+
const name = node.childForFieldName("name");
|
|
388
|
+
if (name) add(node, text(name), node.type === "method_declaration" ? "method" : "constructor", scope);
|
|
389
|
+
return;
|
|
390
|
+
}
|
|
391
|
+
if (node.type === "variable_declarator" && addBound(node.childForFieldName("value"), node.childForFieldName("name"), scope)) return;
|
|
392
|
+
} else if (language === "c" && node.type === "function_definition") {
|
|
393
|
+
const name = cFunctionName(node.childForFieldName("declarator"));
|
|
394
|
+
if (name) add(node, name, "function", scope);
|
|
395
|
+
return;
|
|
396
|
+
}
|
|
397
|
+
if (["lambda", "closure_expression", "func_literal", "lambda_expression"].includes(node.type)) return;
|
|
398
|
+
for (const child of node.namedChildren) walk2(child, scope);
|
|
399
|
+
};
|
|
400
|
+
walk2(root, []);
|
|
401
|
+
return candidates;
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
// src/parser/diagnostics.ts
|
|
405
|
+
function parseDiagnostics(root, content, filePath, language, candidates) {
|
|
406
|
+
if (!root.hasError) return [];
|
|
407
|
+
const errors = [];
|
|
408
|
+
const brokenNodes = /* @__PURE__ */ new Set();
|
|
409
|
+
const issue = (node, qualifiedName, message) => ({
|
|
410
|
+
path: filePath,
|
|
411
|
+
language,
|
|
412
|
+
scope: qualifiedName === null ? "file" : "function",
|
|
413
|
+
code: "parse-error",
|
|
414
|
+
message,
|
|
415
|
+
qualifiedName,
|
|
416
|
+
startLine: node.startPosition.row + 1,
|
|
417
|
+
startColumn: node.startPosition.column + 1,
|
|
418
|
+
endLine: node.endPosition.row + 1,
|
|
419
|
+
endColumn: node.endPosition.column + 1,
|
|
420
|
+
source: content.slice(node.startIndex, node.endIndex)
|
|
421
|
+
});
|
|
422
|
+
for (const candidate of candidates) {
|
|
423
|
+
if (!candidate.node.hasError) continue;
|
|
424
|
+
const name = [...candidate.scope, candidate.name].join(".");
|
|
425
|
+
errors.push(issue(candidate.node, name, `Cannot fully parse callable ${name}; omitted from the searchable index.`));
|
|
426
|
+
brokenNodes.add(candidate.node.id);
|
|
427
|
+
}
|
|
428
|
+
const walk2 = (node, insideBrokenCallable) => {
|
|
429
|
+
const inside = insideBrokenCallable || brokenNodes.has(node.id);
|
|
430
|
+
if (node.isError || node.isMissing) {
|
|
431
|
+
if (!inside) errors.push(issue(node, null, node.isMissing ? `Tree-sitter expected ${node.type}.` : "Tree-sitter could not parse this source region."));
|
|
432
|
+
const children = node.children;
|
|
433
|
+
for (let index = 0; index < children.length; index += 1) {
|
|
434
|
+
if (!["function", "def", "fn", "func"].includes(children[index].type)) continue;
|
|
435
|
+
let next = index + 1;
|
|
436
|
+
if (children[next]?.type === "*") next += 1;
|
|
437
|
+
const name = children[next];
|
|
438
|
+
if (name && ["identifier", "field_identifier"].includes(name.type)) {
|
|
439
|
+
errors.push(issue(node, name.text, `Cannot parse declaration for ${name.text}; name recovered from an error node.`));
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
return;
|
|
443
|
+
}
|
|
444
|
+
for (const child of node.children) if (child.hasError || child.isError || child.isMissing) walk2(child, inside);
|
|
445
|
+
};
|
|
446
|
+
walk2(root, false);
|
|
447
|
+
return errors;
|
|
448
|
+
}
|
|
449
|
+
|
|
450
|
+
// src/parser/callable-parser.ts
|
|
451
|
+
var parsers = /* @__PURE__ */ new Map();
|
|
452
|
+
var CALLABLE_PARSER_CACHE_VERSION = "callable-parser-v1";
|
|
453
|
+
function languageForPath(filePath) {
|
|
454
|
+
switch (path3.extname(filePath).toLowerCase()) {
|
|
455
|
+
case ".ts":
|
|
456
|
+
case ".mts":
|
|
457
|
+
case ".cts":
|
|
458
|
+
return "typescript";
|
|
459
|
+
case ".tsx":
|
|
460
|
+
return "tsx";
|
|
461
|
+
case ".js":
|
|
462
|
+
case ".mjs":
|
|
463
|
+
case ".cjs":
|
|
464
|
+
return "javascript";
|
|
465
|
+
case ".jsx":
|
|
466
|
+
return "jsx";
|
|
467
|
+
case ".py":
|
|
468
|
+
case ".pyw":
|
|
469
|
+
return "python";
|
|
470
|
+
case ".rs":
|
|
471
|
+
return "rust";
|
|
472
|
+
case ".go":
|
|
473
|
+
return "go";
|
|
474
|
+
case ".java":
|
|
475
|
+
return "java";
|
|
476
|
+
case ".c":
|
|
477
|
+
case ".h":
|
|
478
|
+
return "c";
|
|
479
|
+
default:
|
|
480
|
+
return null;
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
function getParser(language) {
|
|
484
|
+
const existing = parsers.get(language);
|
|
485
|
+
if (existing) return existing;
|
|
486
|
+
const parser = new Parser();
|
|
487
|
+
if (language === "typescript") parser.setLanguage(TypeScript.typescript);
|
|
488
|
+
else if (language === "tsx") parser.setLanguage(TypeScript.tsx);
|
|
489
|
+
else if (language === "python") parser.setLanguage(Python);
|
|
490
|
+
else if (language === "rust") parser.setLanguage(Rust);
|
|
491
|
+
else if (language === "go") parser.setLanguage(Go);
|
|
492
|
+
else if (language === "java") parser.setLanguage(Java);
|
|
493
|
+
else if (language === "c") parser.setLanguage(C);
|
|
494
|
+
else parser.setLanguage(JavaScript);
|
|
495
|
+
parsers.set(language, parser);
|
|
496
|
+
return parser;
|
|
497
|
+
}
|
|
498
|
+
function parseTree(parser, content, relativePath) {
|
|
499
|
+
try {
|
|
500
|
+
return parser.parse(content, void 0, { bufferSize: Buffer.byteLength(content) + 1 });
|
|
501
|
+
} catch (error) {
|
|
502
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
503
|
+
throw new CodeIndexError(`Cannot index ${relativePath}: tree-sitter failed: ${message}`, { cause: error });
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
function maskSyntax(value, replacement) {
|
|
507
|
+
return replacement + value.slice(replacement.length).replace(/[^\r\n]/g, " ");
|
|
508
|
+
}
|
|
509
|
+
function recoverUnsupportedTypeScriptSyntax(parser, tree, content, relativePath) {
|
|
510
|
+
const errors = [];
|
|
511
|
+
const collectErrors = (node) => {
|
|
512
|
+
if (node.isError || node.isMissing) errors.push(node);
|
|
513
|
+
for (const child of node.children) collectErrors(child);
|
|
514
|
+
};
|
|
515
|
+
collectErrors(tree.rootNode);
|
|
516
|
+
if (errors.length === 0) return null;
|
|
517
|
+
let recoveredContent = content;
|
|
518
|
+
let changed = false;
|
|
519
|
+
const taggedTemplateTypeArguments = /* @__PURE__ */ new Map();
|
|
520
|
+
for (const error of errors) {
|
|
521
|
+
const parent = error.parent;
|
|
522
|
+
if (error.isMissing && error.type === "!" && parent?.type === "type_arguments" && /^\s*`/.test(content.slice(parent.endIndex))) {
|
|
523
|
+
taggedTemplateTypeArguments.set(parent.startIndex, parent);
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
for (const node of [...taggedTemplateTypeArguments.values()].toSorted((left, right) => right.startIndex - left.startIndex)) {
|
|
527
|
+
const value = recoveredContent.slice(node.startIndex, node.endIndex);
|
|
528
|
+
recoveredContent = `${recoveredContent.slice(0, node.startIndex)}${value.replace(/[^\r\n]/g, " ")}${recoveredContent.slice(node.endIndex)}`;
|
|
529
|
+
changed = true;
|
|
530
|
+
}
|
|
531
|
+
for (const node of errors.filter((candidate) => {
|
|
532
|
+
const parent = candidate.parent;
|
|
533
|
+
return !candidate.isMissing && candidate.type === "ERROR" && candidate.text === "type" && parent?.type === "export_statement" && content.slice(parent.startIndex, candidate.startIndex).trim() === "export" && /^\s*\*/.test(content.slice(candidate.endIndex, parent.endIndex));
|
|
534
|
+
}).toSorted((left, right) => right.startIndex - left.startIndex)) {
|
|
535
|
+
recoveredContent = `${recoveredContent.slice(0, node.startIndex)} ${recoveredContent.slice(node.endIndex)}`;
|
|
536
|
+
changed = true;
|
|
537
|
+
}
|
|
538
|
+
recoveredContent = recoveredContent.replace(
|
|
539
|
+
/\bimport\s*\(\s*(["'])(?:\\.|(?!\1)[^\\])*\1\s*\)/g,
|
|
540
|
+
(value) => {
|
|
541
|
+
changed = true;
|
|
542
|
+
return maskSyntax(value, "X");
|
|
543
|
+
}
|
|
544
|
+
);
|
|
545
|
+
if (!changed) return null;
|
|
546
|
+
const recoveredTree = parseTree(parser, recoveredContent, relativePath);
|
|
547
|
+
return recoveredTree.rootNode.hasError ? null : recoveredTree;
|
|
548
|
+
}
|
|
549
|
+
function textFor(content, node) {
|
|
550
|
+
return content.slice(node.startIndex, node.endIndex);
|
|
551
|
+
}
|
|
552
|
+
function fieldText(content, node, field) {
|
|
553
|
+
const child = node.childForFieldName(field);
|
|
554
|
+
return child ? textFor(content, child) : null;
|
|
555
|
+
}
|
|
556
|
+
function callableKind(node, name) {
|
|
557
|
+
if (node.type === "method_definition" && name === "constructor") return "constructor";
|
|
558
|
+
if (node.type.includes("generator") || /^\s*(?:async\s+)?\*/.test(node.text)) return "generator";
|
|
559
|
+
return node.type === "method_definition" ? "method" : "function";
|
|
560
|
+
}
|
|
561
|
+
function bindingName(content, node) {
|
|
562
|
+
const nameNode = node.childForFieldName("name");
|
|
563
|
+
if (!nameNode || !["identifier", "property_identifier", "private_property_identifier"].includes(nameNode.type)) {
|
|
564
|
+
return null;
|
|
565
|
+
}
|
|
566
|
+
return textFor(content, nameNode).replace(/^#/, "");
|
|
567
|
+
}
|
|
568
|
+
function assignmentName(content, node) {
|
|
569
|
+
const left = node.childForFieldName("left");
|
|
570
|
+
if (!left) return null;
|
|
571
|
+
const value = textFor(content, left).trim();
|
|
572
|
+
const match = value.match(/(?:^|[.#])([A-Za-z_$][\w$]*)$/);
|
|
573
|
+
return match?.[1] ?? null;
|
|
574
|
+
}
|
|
575
|
+
function objectScopeName(content, node) {
|
|
576
|
+
const parent = node.parent;
|
|
577
|
+
if (!parent) return null;
|
|
578
|
+
if (parent.type === "variable_declarator" && parent.childForFieldName("value")?.id === node.id) {
|
|
579
|
+
return bindingName(content, parent);
|
|
580
|
+
}
|
|
581
|
+
if (parent.type === "pair" && parent.childForFieldName("value")?.id === node.id) {
|
|
582
|
+
return fieldText(content, parent, "key")?.replace(/["']/g, "") ?? null;
|
|
583
|
+
}
|
|
584
|
+
if (parent.type === "assignment_expression" && parent.childForFieldName("right")?.id === node.id) {
|
|
585
|
+
return assignmentName(content, parent);
|
|
586
|
+
}
|
|
587
|
+
return null;
|
|
588
|
+
}
|
|
589
|
+
function walk(node, content, scope, candidates) {
|
|
590
|
+
if (["class_declaration", "abstract_class_declaration", "class"].includes(node.type)) {
|
|
591
|
+
const name = fieldText(content, node, "name") ?? objectScopeName(content, node);
|
|
592
|
+
const nextScope = name ? [...scope, name] : scope;
|
|
593
|
+
for (const child of node.namedChildren) walk(child, content, nextScope, candidates);
|
|
594
|
+
return;
|
|
595
|
+
}
|
|
596
|
+
if (node.type === "internal_module" || node.type === "module") {
|
|
597
|
+
const name = fieldText(content, node, "name")?.replace(/["']/g, "");
|
|
598
|
+
const nextScope = name ? [...scope, name] : scope;
|
|
599
|
+
for (const child of node.namedChildren) walk(child, content, nextScope, candidates);
|
|
600
|
+
return;
|
|
601
|
+
}
|
|
602
|
+
if (node.type === "object") {
|
|
603
|
+
const name = objectScopeName(content, node);
|
|
604
|
+
const nextScope = name ? [...scope, name] : scope;
|
|
605
|
+
for (const child of node.namedChildren) walk(child, content, nextScope, candidates);
|
|
606
|
+
return;
|
|
607
|
+
}
|
|
608
|
+
if (["function_declaration", "generator_function_declaration"].includes(node.type)) {
|
|
609
|
+
const name = fieldText(content, node, "name");
|
|
610
|
+
const body = node.childForFieldName("body");
|
|
611
|
+
if (name && (body || node.hasError)) {
|
|
612
|
+
candidates.push({ node, name, kind: callableKind(node, name), scope });
|
|
613
|
+
if (body) walk(body, content, [...scope, name], candidates);
|
|
614
|
+
}
|
|
615
|
+
return;
|
|
616
|
+
}
|
|
617
|
+
if (node.type === "method_definition") {
|
|
618
|
+
const name = fieldText(content, node, "name")?.replace(/^#/, "");
|
|
619
|
+
const body = node.childForFieldName("body");
|
|
620
|
+
if (name && (body || node.hasError)) {
|
|
621
|
+
candidates.push({ node, name, kind: callableKind(node, name), scope });
|
|
622
|
+
if (body) walk(body, content, [...scope, name], candidates);
|
|
623
|
+
}
|
|
624
|
+
return;
|
|
625
|
+
}
|
|
626
|
+
if (["public_field_definition", "property_definition", "field_definition"].includes(node.type)) {
|
|
627
|
+
const name = (fieldText(content, node, "name") ?? fieldText(content, node, "property"))?.replace(/^#/, "");
|
|
628
|
+
const value = node.childForFieldName("value");
|
|
629
|
+
if (name && value && ["arrow_function", "function_expression", "generator_function"].includes(value.type)) {
|
|
630
|
+
const kind = callableKind(value, name) === "generator" ? "generator" : "method";
|
|
631
|
+
candidates.push({ node: value, name, kind, scope });
|
|
632
|
+
const body = value.childForFieldName("body");
|
|
633
|
+
if (body) walk(body, content, [...scope, name], candidates);
|
|
634
|
+
return;
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
if (node.type === "variable_declarator") {
|
|
638
|
+
const name = bindingName(content, node);
|
|
639
|
+
const value = node.childForFieldName("value");
|
|
640
|
+
if (name && value && ["arrow_function", "function_expression", "generator_function"].includes(value.type)) {
|
|
641
|
+
candidates.push({ node: value, name, kind: callableKind(value, name), scope });
|
|
642
|
+
const body = value.childForFieldName("body");
|
|
643
|
+
if (body) walk(body, content, [...scope, name], candidates);
|
|
644
|
+
return;
|
|
645
|
+
}
|
|
646
|
+
}
|
|
647
|
+
if (node.type === "pair") {
|
|
648
|
+
const name = fieldText(content, node, "key")?.replace(/["']/g, "");
|
|
649
|
+
const value = node.childForFieldName("value");
|
|
650
|
+
if (name && value && ["arrow_function", "function_expression", "generator_function"].includes(value.type)) {
|
|
651
|
+
candidates.push({ node: value, name, kind: callableKind(value, name), scope });
|
|
652
|
+
const body = value.childForFieldName("body");
|
|
653
|
+
if (body) walk(body, content, [...scope, name], candidates);
|
|
654
|
+
return;
|
|
655
|
+
}
|
|
656
|
+
}
|
|
657
|
+
if (node.type === "assignment_expression") {
|
|
658
|
+
const name = assignmentName(content, node);
|
|
659
|
+
const value = node.childForFieldName("right");
|
|
660
|
+
if (name && value && ["arrow_function", "function_expression", "generator_function"].includes(value.type)) {
|
|
661
|
+
candidates.push({ node: value, name, kind: callableKind(value, name), scope });
|
|
662
|
+
const body = value.childForFieldName("body");
|
|
663
|
+
if (body) walk(body, content, [...scope, name], candidates);
|
|
664
|
+
return;
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
if (["function_expression", "generator_function"].includes(node.type)) {
|
|
668
|
+
const name = fieldText(content, node, "name");
|
|
669
|
+
const body = node.childForFieldName("body");
|
|
670
|
+
if (name && body) {
|
|
671
|
+
candidates.push({ node, name, kind: callableKind(node, name), scope });
|
|
672
|
+
walk(body, content, [...scope, name], candidates);
|
|
673
|
+
return;
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
for (const child of node.namedChildren) walk(child, content, scope, candidates);
|
|
677
|
+
}
|
|
678
|
+
function signatureFor(content, node, name) {
|
|
679
|
+
const parameters = node.childForFieldName("parameters");
|
|
680
|
+
const parameter = node.childForFieldName("parameter");
|
|
681
|
+
const typeParameters = node.childForFieldName("type_parameters");
|
|
682
|
+
const returnType = node.childForFieldName("return_type");
|
|
683
|
+
const async = node.children.some((child) => child.type === "async") ? "async " : "";
|
|
684
|
+
const generator = node.children.some((child) => child.type === "*") ? "*" : "";
|
|
685
|
+
const parameterText = parameters ? textFor(content, parameters) : parameter ? `(${textFor(content, parameter)})` : "";
|
|
686
|
+
return `${async}${generator}${name}${typeParameters ? textFor(content, typeParameters) : ""}${parameterText}${returnType ? textFor(content, returnType) : ""}`;
|
|
687
|
+
}
|
|
688
|
+
function parseCallables(relativePath, content, onWarning = console.warn) {
|
|
689
|
+
return parseFileCallables(relativePath, content, onWarning).callables;
|
|
690
|
+
}
|
|
691
|
+
function parseFileCallables(relativePath, content, onWarning = console.warn) {
|
|
692
|
+
let result;
|
|
693
|
+
try {
|
|
694
|
+
result = extractCallables(relativePath, content);
|
|
695
|
+
} catch (error) {
|
|
696
|
+
const message = `Cannot index ${relativePath}: ${error instanceof Error ? error.message : String(error)}`;
|
|
697
|
+
onWarning(message);
|
|
698
|
+
return { callables: [], errors: [{
|
|
699
|
+
path: relativePath,
|
|
700
|
+
language: languageForPath(relativePath),
|
|
701
|
+
scope: "file",
|
|
702
|
+
code: "parse-failed",
|
|
703
|
+
message,
|
|
704
|
+
qualifiedName: null,
|
|
705
|
+
startLine: null,
|
|
706
|
+
startColumn: null,
|
|
707
|
+
endLine: null,
|
|
708
|
+
endColumn: null,
|
|
709
|
+
source: content
|
|
710
|
+
}] };
|
|
711
|
+
}
|
|
712
|
+
if (result.errors.length > 0) {
|
|
713
|
+
onWarning(`Cannot fully parse ${relativePath}: tree-sitter reported syntax errors; indexing recoverable callables only.`);
|
|
714
|
+
}
|
|
715
|
+
return result;
|
|
716
|
+
}
|
|
717
|
+
function extractCallables(relativePath, content) {
|
|
718
|
+
const language = languageForPath(relativePath);
|
|
719
|
+
if (!language) return { callables: [], errors: [] };
|
|
720
|
+
const parser = getParser(language);
|
|
721
|
+
const parsedTree = parseTree(parser, content, relativePath);
|
|
722
|
+
const tree = parsedTree.rootNode.hasError && (language === "typescript" || language === "tsx") ? recoverUnsupportedTypeScriptSyntax(parser, parsedTree, content, relativePath) ?? parsedTree : parsedTree;
|
|
723
|
+
const candidates = [];
|
|
724
|
+
if (["typescript", "tsx", "javascript", "jsx"].includes(language)) {
|
|
725
|
+
walk(tree.rootNode, content, [], candidates);
|
|
726
|
+
} else {
|
|
727
|
+
candidates.push(...collectNativeCallables(tree.rootNode, content, language));
|
|
728
|
+
}
|
|
729
|
+
candidates.sort((left, right) => left.node.startIndex - right.node.startIndex);
|
|
730
|
+
const errors = parseDiagnostics(tree.rootNode, content, relativePath, language, candidates);
|
|
731
|
+
const occurrences = /* @__PURE__ */ new Map();
|
|
732
|
+
const callables = [];
|
|
733
|
+
for (const { node, name, kind, scope, signature: candidateSignature, documentation } of candidates) {
|
|
734
|
+
const qualifiedName = [...scope, name].join(".");
|
|
735
|
+
const baseIdentity = `${relativePath}\0${qualifiedName}\0${kind}`;
|
|
736
|
+
const occurrence = occurrences.get(baseIdentity) ?? 0;
|
|
737
|
+
occurrences.set(baseIdentity, occurrence + 1);
|
|
738
|
+
if (node.hasError) continue;
|
|
739
|
+
try {
|
|
740
|
+
const identityKey = sha256(`${baseIdentity}\0${occurrence}`);
|
|
741
|
+
const source = textFor(content, node);
|
|
742
|
+
const signature = candidateSignature ?? signatureFor(content, node, name);
|
|
743
|
+
const embeddingInput = [
|
|
744
|
+
`language: ${language}`,
|
|
745
|
+
`kind: ${kind}`,
|
|
746
|
+
`symbol: ${qualifiedName}`,
|
|
747
|
+
signature ? `signature: ${signature}` : null,
|
|
748
|
+
documentation ? `documentation:
|
|
749
|
+
${documentation}` : null,
|
|
750
|
+
"source:",
|
|
751
|
+
source
|
|
752
|
+
].filter((value) => value !== null).join("\n");
|
|
753
|
+
callables.push({
|
|
754
|
+
path: relativePath,
|
|
755
|
+
language,
|
|
756
|
+
kind,
|
|
757
|
+
name,
|
|
758
|
+
qualifiedName,
|
|
759
|
+
signature,
|
|
760
|
+
identityKey,
|
|
761
|
+
startLine: node.startPosition.row + 1,
|
|
762
|
+
startColumn: node.startPosition.column + 1,
|
|
763
|
+
endLine: node.endPosition.row + 1,
|
|
764
|
+
endColumn: node.endPosition.column + 1,
|
|
765
|
+
lineCount: node.endPosition.row - node.startPosition.row + 1,
|
|
766
|
+
source,
|
|
767
|
+
sourceHash: sha256(source),
|
|
768
|
+
embeddingInput
|
|
769
|
+
});
|
|
770
|
+
} catch (error) {
|
|
771
|
+
errors.push({
|
|
772
|
+
path: relativePath,
|
|
773
|
+
language,
|
|
774
|
+
scope: "function",
|
|
775
|
+
code: "extraction-error",
|
|
776
|
+
qualifiedName,
|
|
777
|
+
message: `Cannot extract ${qualifiedName}: ${error instanceof Error ? error.message : String(error)}`,
|
|
778
|
+
startLine: node.startPosition.row + 1,
|
|
779
|
+
startColumn: node.startPosition.column + 1,
|
|
780
|
+
endLine: node.endPosition.row + 1,
|
|
781
|
+
endColumn: node.endPosition.column + 1,
|
|
782
|
+
source: textFor(content, node)
|
|
783
|
+
});
|
|
784
|
+
}
|
|
785
|
+
}
|
|
786
|
+
return { callables, errors };
|
|
787
|
+
}
|
|
788
|
+
|
|
789
|
+
// src/source-policy.ts
|
|
790
|
+
import path4 from "node:path";
|
|
791
|
+
var DEFAULT_EXCLUDED_SEGMENTS = /* @__PURE__ */ new Set([
|
|
792
|
+
".git",
|
|
793
|
+
".slopdex",
|
|
794
|
+
"node_modules",
|
|
795
|
+
"dist",
|
|
796
|
+
"build",
|
|
797
|
+
"coverage",
|
|
798
|
+
"vendor",
|
|
799
|
+
"generated",
|
|
800
|
+
".venv",
|
|
801
|
+
"venv",
|
|
802
|
+
"__pycache__",
|
|
803
|
+
".tox",
|
|
804
|
+
".mypy_cache",
|
|
805
|
+
".pytest_cache",
|
|
806
|
+
"target"
|
|
807
|
+
]);
|
|
808
|
+
var SourcePolicy = class {
|
|
809
|
+
#include;
|
|
810
|
+
#exclude;
|
|
811
|
+
constructor(include = [], exclude = []) {
|
|
812
|
+
this.#include = include;
|
|
813
|
+
this.#exclude = exclude;
|
|
814
|
+
}
|
|
815
|
+
includes(relativePath) {
|
|
816
|
+
const normalized = relativePath.replaceAll("\\", "/");
|
|
817
|
+
if (!languageForPath(normalized)) return false;
|
|
818
|
+
if (hasExcludedSegment(normalized)) return false;
|
|
819
|
+
if (this.#exclude.some((pattern) => path4.matchesGlob(normalized, pattern))) return false;
|
|
820
|
+
return this.#include.length === 0 || this.#include.some((pattern) => path4.matchesGlob(normalized, pattern));
|
|
821
|
+
}
|
|
822
|
+
traversesDirectory(relativePath) {
|
|
823
|
+
return !hasExcludedSegment(relativePath.replaceAll("\\", "/"));
|
|
824
|
+
}
|
|
825
|
+
};
|
|
826
|
+
function hasExcludedSegment(relativePath) {
|
|
827
|
+
return relativePath.split("/").some((segment) => DEFAULT_EXCLUDED_SEGMENTS.has(segment));
|
|
828
|
+
}
|
|
829
|
+
|
|
830
|
+
// src/code-index.ts
|
|
831
|
+
var DEFAULT_MAX_FILE_SIZE = 1024 * 1024;
|
|
832
|
+
var DEFAULT_BATCH_SIZE = 32;
|
|
833
|
+
var RERANK_CANDIDATE_MULTIPLIER = 5;
|
|
834
|
+
var DEFAULT_SIMILARITY_CACHE_WIDTH = 50;
|
|
835
|
+
var MAX_SIMILARITY_CACHE_WIDTH = 200;
|
|
836
|
+
var CodeIndex = class {
|
|
837
|
+
rootDir;
|
|
838
|
+
indexPath;
|
|
839
|
+
provider;
|
|
840
|
+
reranker;
|
|
841
|
+
descriptionProvider;
|
|
842
|
+
#database;
|
|
843
|
+
#policy;
|
|
844
|
+
#maxFileSize;
|
|
845
|
+
#embeddingBatchSize;
|
|
846
|
+
#parallelism;
|
|
847
|
+
#onProgress;
|
|
848
|
+
#git;
|
|
849
|
+
#onWarning;
|
|
850
|
+
constructor(options) {
|
|
851
|
+
this.rootDir = path5.resolve(options.rootDir);
|
|
852
|
+
this.indexPath = path5.resolve(options.indexPath ?? path5.join(this.rootDir, ".slopdex", "index.sqlite"));
|
|
853
|
+
this.provider = options.provider;
|
|
854
|
+
this.reranker = options.reranker;
|
|
855
|
+
if (this.reranker?.candidateCount !== void 0) {
|
|
856
|
+
assertPositiveInteger(this.reranker.candidateCount, "reranker candidate count");
|
|
857
|
+
}
|
|
858
|
+
if (this.reranker?.maximumCandidateCount !== void 0) {
|
|
859
|
+
assertPositiveInteger(this.reranker.maximumCandidateCount, "reranker maximum candidate count");
|
|
860
|
+
if (this.reranker.candidateCount !== void 0 && this.reranker.candidateCount > this.reranker.maximumCandidateCount) {
|
|
861
|
+
throw new CodeIndexError("reranker candidate count must not exceed its maximum candidate count.");
|
|
862
|
+
}
|
|
863
|
+
}
|
|
864
|
+
const profile = normalizeProfile(options.provider.profile);
|
|
865
|
+
assertPositiveInteger(profile.dimensions, "embedding dimensions");
|
|
866
|
+
this.#database = new IndexDatabase(this.indexPath, this.rootDir, profile, options.readOnly ?? false);
|
|
867
|
+
this.#parallelism = options.parallelism ?? DEFAULT_PARALLELISM;
|
|
868
|
+
assertPositiveInteger(this.#parallelism, "parallelism");
|
|
869
|
+
this.#onProgress = options.onProgress;
|
|
870
|
+
const storedDescriptionProfile = this.#database.descriptionProfile();
|
|
871
|
+
this.descriptionProvider = options.descriptionProvider ?? new OpenAIDescriptionProvider({
|
|
872
|
+
...storedDescriptionProfile && isDescriptionProviderName(storedDescriptionProfile.provider) ? { provider: storedDescriptionProfile.provider, model: storedDescriptionProfile.model } : {},
|
|
873
|
+
parallelism: this.#parallelism,
|
|
874
|
+
...options.verbose ? { verbose: true } : {}
|
|
875
|
+
});
|
|
876
|
+
this.#policy = new SourcePolicy(options.include, options.exclude);
|
|
877
|
+
this.#maxFileSize = options.maxFileSize ?? DEFAULT_MAX_FILE_SIZE;
|
|
878
|
+
this.#embeddingBatchSize = options.embeddingBatchSize ?? DEFAULT_BATCH_SIZE;
|
|
879
|
+
this.#onWarning = options.onWarning ?? console.warn;
|
|
880
|
+
assertPositiveInteger(this.#maxFileSize, "maxFileSize");
|
|
881
|
+
assertPositiveInteger(this.#embeddingBatchSize, "embeddingBatchSize");
|
|
882
|
+
this.#git = new GitRepository(this.rootDir);
|
|
883
|
+
}
|
|
884
|
+
#progress(phase, completed, total) {
|
|
885
|
+
this.#onProgress?.({ phase, completed, total });
|
|
886
|
+
}
|
|
887
|
+
close() {
|
|
888
|
+
this.#database.close();
|
|
889
|
+
}
|
|
890
|
+
status() {
|
|
891
|
+
return this.#database.status();
|
|
892
|
+
}
|
|
893
|
+
allFunctions() {
|
|
894
|
+
return this.#database.allFunctions();
|
|
895
|
+
}
|
|
896
|
+
indexErrors() {
|
|
897
|
+
return this.#database.indexErrors();
|
|
898
|
+
}
|
|
899
|
+
async updateFiles(options) {
|
|
900
|
+
throwIfAborted(options.signal);
|
|
901
|
+
const gitignore = GitignoreRules.workingTree(this.rootDir);
|
|
902
|
+
const generation = this.#database.getGeneration();
|
|
903
|
+
const normalizedRenames = (options.renames ?? []).map(({ from, to }) => ({
|
|
904
|
+
from: normalizeRelativePath(this.rootDir, from),
|
|
905
|
+
to: normalizeRelativePath(this.rootDir, to)
|
|
906
|
+
}));
|
|
907
|
+
validateRenames(normalizedRenames);
|
|
908
|
+
const upsertPaths = [...new Set((options.upsert ?? []).map((value) => normalizeRelativePath(this.rootDir, value)))];
|
|
909
|
+
const deletePaths = [...new Set((options.delete ?? []).map((value) => normalizeRelativePath(this.rootDir, value)))];
|
|
910
|
+
const renameMap = new Map(
|
|
911
|
+
normalizedRenames.map(({ from, to }) => [to, from])
|
|
912
|
+
);
|
|
913
|
+
for (const target of renameMap.keys()) {
|
|
914
|
+
if (!upsertPaths.includes(target)) upsertPaths.push(target);
|
|
915
|
+
}
|
|
916
|
+
const prepared = [];
|
|
917
|
+
for (const relativePath of upsertPaths) {
|
|
918
|
+
throwIfAborted(options.signal);
|
|
919
|
+
if (!this.#policy.includes(relativePath) || await gitignore.ignores(relativePath)) {
|
|
920
|
+
throw new CodeIndexError(`Unsupported or excluded source file: ${relativePath}`);
|
|
921
|
+
}
|
|
922
|
+
const renameSource = renameMap.get(relativePath);
|
|
923
|
+
const originalPath = renameSource ? this.#database.previousPath(renameSource) ?? renameSource : void 0;
|
|
924
|
+
const file = await this.#prepareWorkingFile(relativePath, {
|
|
925
|
+
blobOid: null,
|
|
926
|
+
sourceMode: "working-tree",
|
|
927
|
+
indexedCommit: null,
|
|
928
|
+
...originalPath ? { previousPath: originalPath } : {},
|
|
929
|
+
...renameSource ? { replacePath: renameSource } : {}
|
|
930
|
+
}, true);
|
|
931
|
+
if (file) prepared.push(file);
|
|
932
|
+
}
|
|
933
|
+
let embeddingsCreated = await this.#attachEmbeddings(prepared, options.signal);
|
|
934
|
+
for (let attempt = 0; ; attempt += 1) {
|
|
935
|
+
await gitignore.assertUnchanged(options.signal);
|
|
936
|
+
const changed = [];
|
|
937
|
+
for (let index = 0; index < prepared.length; index += 1) {
|
|
938
|
+
const file = prepared[index];
|
|
939
|
+
if (!this.#workingFileChangedSynchronously(file)) continue;
|
|
940
|
+
if (attempt >= 2) throw new CodeIndexError(`Source changed repeatedly while indexing: ${file.path}; retry the update.`);
|
|
941
|
+
const refreshed = await this.#prepareWorkingFile(file.path, {
|
|
942
|
+
blobOid: null,
|
|
943
|
+
sourceMode: "working-tree",
|
|
944
|
+
indexedCommit: null,
|
|
945
|
+
...file.previousPath ? { previousPath: file.previousPath } : {},
|
|
946
|
+
...file.replacePath ? { replacePath: file.replacePath } : {}
|
|
947
|
+
}, true);
|
|
948
|
+
if (!refreshed) throw new CodeIndexError(`Source changed while indexing: ${file.path}; retry the update.`);
|
|
949
|
+
prepared[index] = refreshed;
|
|
950
|
+
changed.push(refreshed);
|
|
951
|
+
}
|
|
952
|
+
if (changed.length === 0) break;
|
|
953
|
+
embeddingsCreated += await this.#attachEmbeddings(changed, options.signal);
|
|
954
|
+
}
|
|
955
|
+
return this.#database.applyUpdate({
|
|
956
|
+
files: prepared,
|
|
957
|
+
deletePaths: [...deletePaths, ...renameMap.values()],
|
|
958
|
+
expectedGeneration: generation,
|
|
959
|
+
embeddingsCreated
|
|
960
|
+
});
|
|
961
|
+
}
|
|
962
|
+
async updateFromWorkingTree(options = {}) {
|
|
963
|
+
throwIfAborted(options.signal);
|
|
964
|
+
const generation = this.#database.getGeneration();
|
|
965
|
+
const indexedFiles = this.#database.getFileStates();
|
|
966
|
+
const gitignore = GitignoreRules.workingTree(this.rootDir);
|
|
967
|
+
let sourcePaths = await this.#workingTreeSourcePaths(gitignore, options.signal);
|
|
968
|
+
let prepared = await this.#prepareWorkingPaths(sourcePaths, options.signal);
|
|
969
|
+
let embeddingsCreated = await this.#attachEmbeddings(prepared, options.signal);
|
|
970
|
+
while (true) {
|
|
971
|
+
await gitignore.assertUnchanged(options.signal);
|
|
972
|
+
const pathsAgain = await this.#workingTreeSourcePaths(gitignore, options.signal);
|
|
973
|
+
if (JSON.stringify(pathsAgain) !== JSON.stringify(sourcePaths)) {
|
|
974
|
+
sourcePaths = pathsAgain;
|
|
975
|
+
prepared = await this.#prepareWorkingPaths(sourcePaths, options.signal);
|
|
976
|
+
embeddingsCreated += await this.#attachEmbeddings(prepared, options.signal);
|
|
977
|
+
continue;
|
|
978
|
+
}
|
|
979
|
+
const changed = [];
|
|
980
|
+
for (let index = 0; index < prepared.length; index += 1) {
|
|
981
|
+
const file = prepared[index];
|
|
982
|
+
if (!await this.#workingFileChanged(file)) continue;
|
|
983
|
+
const refreshed = await this.#prepareWorkingFile(file.path, {
|
|
984
|
+
blobOid: null,
|
|
985
|
+
sourceMode: "working-tree",
|
|
986
|
+
indexedCommit: null
|
|
987
|
+
});
|
|
988
|
+
if (!refreshed) {
|
|
989
|
+
sourcePaths = await this.#workingTreeSourcePaths(gitignore, options.signal);
|
|
990
|
+
prepared = await this.#prepareWorkingPaths(sourcePaths, options.signal);
|
|
991
|
+
changed.push(...prepared);
|
|
992
|
+
break;
|
|
993
|
+
}
|
|
994
|
+
prepared[index] = refreshed;
|
|
995
|
+
changed.push(refreshed);
|
|
996
|
+
}
|
|
997
|
+
if (changed.length === 0) break;
|
|
998
|
+
embeddingsCreated += await this.#attachEmbeddings(changed, options.signal);
|
|
999
|
+
}
|
|
1000
|
+
await gitignore.assertUnchanged(options.signal);
|
|
1001
|
+
if (this.#isWorkingTreeClean(indexedFiles, prepared)) {
|
|
1002
|
+
return emptyStats(this.#database.getCheckpoint());
|
|
1003
|
+
}
|
|
1004
|
+
return this.#database.applyUpdate({
|
|
1005
|
+
files: prepared,
|
|
1006
|
+
deletePaths: indexedFiles.map((file) => file.path),
|
|
1007
|
+
checkpoint: null,
|
|
1008
|
+
expectedGeneration: generation,
|
|
1009
|
+
completeDiagnosticsScan: true,
|
|
1010
|
+
embeddingsCreated
|
|
1011
|
+
});
|
|
1012
|
+
}
|
|
1013
|
+
/**
|
|
1014
|
+
* Whether a working-tree refresh changed nothing: same file set, same
|
|
1015
|
+
* content, same language, still working-tree rows, no checkout to clear,
|
|
1016
|
+
* and no recorded errors to retry. Skipping the write avoids a generation
|
|
1017
|
+
* bump and, via foreign-key cascades, needlessly invalidating the
|
|
1018
|
+
* similarity cache for unchanged files. Description state must already be
|
|
1019
|
+
* quiescent: a refresh is what lazily clears stored descriptions after
|
|
1020
|
+
* disableDescriptions, so a stale description count must fall through to
|
|
1021
|
+
* the rewrite, and enabled indexes keep the legacy behavior.
|
|
1022
|
+
*/
|
|
1023
|
+
#isWorkingTreeClean(indexedFiles, prepared) {
|
|
1024
|
+
if (this.#database.getCheckpoint() !== null) return false;
|
|
1025
|
+
if (this.#database.needsDiagnosticsScan()) return false;
|
|
1026
|
+
if (this.#database.filesWithErrors().length > 0) return false;
|
|
1027
|
+
if (this.#database.descriptionsEnabled()) return false;
|
|
1028
|
+
if (indexedFiles.length !== prepared.length) return false;
|
|
1029
|
+
const status = this.status();
|
|
1030
|
+
if (status.descriptionCount > 0 || status.fileDescriptionCount > 0) return false;
|
|
1031
|
+
const indexedByPath = new Map(indexedFiles.map((file) => [file.path, file]));
|
|
1032
|
+
return prepared.every((file) => {
|
|
1033
|
+
if (file.unavailable || file.errors.length > 0) return false;
|
|
1034
|
+
const indexed = indexedByPath.get(file.path);
|
|
1035
|
+
return !!indexed && indexed.sourceMode === "working-tree" && indexed.blobOid === file.blobOid && indexed.language === file.language && indexed.contentHash === file.contentHash;
|
|
1036
|
+
});
|
|
1037
|
+
}
|
|
1038
|
+
async updateFromGit(options = {}) {
|
|
1039
|
+
throwIfAborted(options.signal);
|
|
1040
|
+
await this.#git.assertRepository();
|
|
1041
|
+
const relativeIndexPath = path5.relative(this.rootDir, this.indexPath).replaceAll(path5.sep, "/");
|
|
1042
|
+
const indexArtifacts = relativeIndexPath && !relativeIndexPath.startsWith("../") ? [relativeIndexPath, `${relativeIndexPath}-shm`, `${relativeIndexPath}-wal`, `${relativeIndexPath}-journal`] : [];
|
|
1043
|
+
const target = await this.#git.resolveCommit(options.target ?? "HEAD");
|
|
1044
|
+
const head = await this.#git.resolveCommit("HEAD");
|
|
1045
|
+
const overlayWorkingTree = target === head && options.includeWorkingTree !== false;
|
|
1046
|
+
const checkpoint = this.#database.getCheckpoint();
|
|
1047
|
+
const generation = this.#database.getGeneration();
|
|
1048
|
+
const diagnosticsScan = this.#database.needsDiagnosticsScan();
|
|
1049
|
+
const retryPaths = new Set(this.#database.filesWithErrors());
|
|
1050
|
+
let reconcileAll = checkpoint === null;
|
|
1051
|
+
if (checkpoint && !await this.#git.isAncestor(checkpoint, target)) {
|
|
1052
|
+
if (!options.rebuildOnDivergence) throw new GitDivergenceError(checkpoint, target);
|
|
1053
|
+
reconcileAll = true;
|
|
1054
|
+
}
|
|
1055
|
+
const tree = await this.#git.listTree(target);
|
|
1056
|
+
const indexedFiles = this.#database.getFileStates();
|
|
1057
|
+
const indexedByPath = new Map(indexedFiles.map((file) => [file.path, file]));
|
|
1058
|
+
const workingFiles = indexedFiles.filter((file) => file.sourceMode === "working-tree");
|
|
1059
|
+
const workingChanges = overlayWorkingTree ? await this.#git.workTreeChanges(target, indexArtifacts) : [];
|
|
1060
|
+
const gitignore = overlayWorkingTree ? GitignoreRules.workingTree(this.rootDir) : new GitignoreRules(async (filePath) => {
|
|
1061
|
+
const entry = tree.get(filePath);
|
|
1062
|
+
return entry ? (await this.#git.readBlob(entry.oid)).toString("utf8") : null;
|
|
1063
|
+
});
|
|
1064
|
+
const allowedPaths = /* @__PURE__ */ new Set();
|
|
1065
|
+
const candidatePaths = /* @__PURE__ */ new Set([
|
|
1066
|
+
...tree.keys(),
|
|
1067
|
+
...workingChanges.map((change) => change.path),
|
|
1068
|
+
...indexedFiles.map((file) => file.path)
|
|
1069
|
+
]);
|
|
1070
|
+
for (const filePath of candidatePaths) {
|
|
1071
|
+
throwIfAborted(options.signal);
|
|
1072
|
+
if (this.#policy.includes(filePath) && !await gitignore.ignores(filePath)) allowedPaths.add(filePath);
|
|
1073
|
+
}
|
|
1074
|
+
const visibleWorkingPaths = new Set(
|
|
1075
|
+
workingChanges.filter((change) => change.status !== "D").map((change) => change.path)
|
|
1076
|
+
);
|
|
1077
|
+
const changes = reconcileAll || !checkpoint ? [] : await this.#git.diff(checkpoint, target);
|
|
1078
|
+
const upserts = /* @__PURE__ */ new Map();
|
|
1079
|
+
const deletes = /* @__PURE__ */ new Set();
|
|
1080
|
+
const eligibleEntries = [...tree.values()].filter((entry) => allowedPaths.has(entry.path));
|
|
1081
|
+
const eligibleTargetPaths = new Set(eligibleEntries.map((entry) => entry.path));
|
|
1082
|
+
if (reconcileAll) {
|
|
1083
|
+
const renameCandidates = indexedFiles.filter((file) => file.sourceMode === "git" && file.blobOid && !eligibleTargetPaths.has(file.path));
|
|
1084
|
+
const claimedRenameSources = /* @__PURE__ */ new Set();
|
|
1085
|
+
for (const entry of eligibleEntries) {
|
|
1086
|
+
const indexed = indexedByPath.get(entry.path);
|
|
1087
|
+
if (indexed?.sourceMode === "git" && indexed.blobOid === entry.oid && entry.size <= this.#maxFileSize && !diagnosticsScan && !retryPaths.has(entry.path)) continue;
|
|
1088
|
+
const matches = renameCandidates.filter((candidate) => candidate.blobOid === entry.oid && !claimedRenameSources.has(candidate.path));
|
|
1089
|
+
const previousPath = matches.length === 1 ? matches[0].path : void 0;
|
|
1090
|
+
if (previousPath) claimedRenameSources.add(previousPath);
|
|
1091
|
+
upserts.set(entry.path, { entry, ...previousPath ? { previousPath } : {} });
|
|
1092
|
+
}
|
|
1093
|
+
for (const indexed of indexedFiles) {
|
|
1094
|
+
if (!eligibleTargetPaths.has(indexed.path)) deletes.add(indexed.path);
|
|
1095
|
+
}
|
|
1096
|
+
} else {
|
|
1097
|
+
this.#classifyGitChanges(changes, tree, upserts, deletes, allowedPaths);
|
|
1098
|
+
const indexedPaths = new Set(indexedByPath.keys());
|
|
1099
|
+
for (const targetPath of eligibleTargetPaths) {
|
|
1100
|
+
const indexed = indexedByPath.get(targetPath);
|
|
1101
|
+
const entry = tree.get(targetPath);
|
|
1102
|
+
if ((diagnosticsScan || retryPaths.has(targetPath) || entry.size > this.#maxFileSize || !indexed || indexed.sourceMode === "git" && indexed.blobOid !== entry.oid) && !upserts.has(targetPath)) {
|
|
1103
|
+
upserts.set(targetPath, { entry });
|
|
1104
|
+
}
|
|
1105
|
+
}
|
|
1106
|
+
for (const indexedPath of indexedPaths) {
|
|
1107
|
+
if (!eligibleTargetPaths.has(indexedPath)) deletes.add(indexedPath);
|
|
1108
|
+
}
|
|
1109
|
+
}
|
|
1110
|
+
for (const dirtyFile of workingFiles) {
|
|
1111
|
+
const entry = tree.get(dirtyFile.path);
|
|
1112
|
+
const previousEntry = dirtyFile.previousPath ? tree.get(dirtyFile.previousPath) : void 0;
|
|
1113
|
+
if (previousEntry && allowedPaths.has(previousEntry.path) && (entry !== void 0 || !visibleWorkingPaths.has(dirtyFile.path))) {
|
|
1114
|
+
const currentPlan = entry ? upserts.get(dirtyFile.path) : void 0;
|
|
1115
|
+
upserts.delete(dirtyFile.path);
|
|
1116
|
+
upserts.delete(previousEntry.path);
|
|
1117
|
+
upserts.set(previousEntry.path, { entry: previousEntry, previousPath: dirtyFile.path });
|
|
1118
|
+
if (entry && allowedPaths.has(entry.path)) {
|
|
1119
|
+
upserts.set(dirtyFile.path, currentPlan ?? { entry });
|
|
1120
|
+
}
|
|
1121
|
+
continue;
|
|
1122
|
+
}
|
|
1123
|
+
if (entry && allowedPaths.has(dirtyFile.path)) {
|
|
1124
|
+
const planned = upserts.get(dirtyFile.path);
|
|
1125
|
+
if (planned?.previousPath && !indexedByPath.has(planned.previousPath)) {
|
|
1126
|
+
upserts.set(dirtyFile.path, { entry, previousPath: dirtyFile.path });
|
|
1127
|
+
} else if (!planned) {
|
|
1128
|
+
upserts.set(dirtyFile.path, { entry });
|
|
1129
|
+
}
|
|
1130
|
+
continue;
|
|
1131
|
+
}
|
|
1132
|
+
if (previousEntry && allowedPaths.has(previousEntry.path)) {
|
|
1133
|
+
upserts.set(previousEntry.path, { entry: previousEntry, previousPath: dirtyFile.path });
|
|
1134
|
+
} else {
|
|
1135
|
+
deletes.add(dirtyFile.path);
|
|
1136
|
+
}
|
|
1137
|
+
}
|
|
1138
|
+
for (const [filePath, { entry, previousPath }] of upserts) {
|
|
1139
|
+
const indexed = indexedByPath.get(filePath);
|
|
1140
|
+
if (!previousPath && indexed?.sourceMode === "git" && indexed.blobOid === entry.oid && entry.size <= this.#maxFileSize && !diagnosticsScan && !retryPaths.has(filePath)) upserts.delete(filePath);
|
|
1141
|
+
}
|
|
1142
|
+
const workingPrepared = [];
|
|
1143
|
+
const workingDeletes = /* @__PURE__ */ new Set();
|
|
1144
|
+
const skippedWorkingPaths = /* @__PURE__ */ new Set();
|
|
1145
|
+
for (const change of workingChanges) {
|
|
1146
|
+
throwIfAborted(options.signal);
|
|
1147
|
+
if (change.status === "D") {
|
|
1148
|
+
upserts.delete(change.path);
|
|
1149
|
+
workingDeletes.add(change.path);
|
|
1150
|
+
continue;
|
|
1151
|
+
}
|
|
1152
|
+
let previousPath;
|
|
1153
|
+
let replacePath;
|
|
1154
|
+
if (change.status === "R") {
|
|
1155
|
+
const plannedBase = upserts.get(change.oldPath);
|
|
1156
|
+
const existingAtTarget = indexedByPath.get(change.oldPath);
|
|
1157
|
+
const existingAtFinal = indexedByPath.get(change.path);
|
|
1158
|
+
const continuingRename = existingAtFinal?.sourceMode === "working-tree" && existingAtFinal.previousPath === change.oldPath;
|
|
1159
|
+
replacePath = continuingRename ? change.path : existingAtTarget ? change.oldPath : existingAtFinal ? change.path : plannedBase?.previousPath && indexedByPath.has(plannedBase.previousPath) ? plannedBase.previousPath : void 0;
|
|
1160
|
+
previousPath = existingAtFinal?.previousPath ?? change.oldPath;
|
|
1161
|
+
upserts.delete(change.oldPath);
|
|
1162
|
+
upserts.delete(change.path);
|
|
1163
|
+
if (replacePath) deletes.delete(replacePath);
|
|
1164
|
+
workingDeletes.add(change.oldPath);
|
|
1165
|
+
} else {
|
|
1166
|
+
const planned = upserts.get(change.path);
|
|
1167
|
+
const existing = indexedByPath.get(change.path);
|
|
1168
|
+
replacePath = existing ? change.path : planned?.previousPath && indexedByPath.has(planned.previousPath) ? planned.previousPath : void 0;
|
|
1169
|
+
previousPath = existing?.previousPath ?? planned?.previousPath;
|
|
1170
|
+
upserts.delete(change.path);
|
|
1171
|
+
if (replacePath) deletes.delete(replacePath);
|
|
1172
|
+
}
|
|
1173
|
+
const relativePath = change.path;
|
|
1174
|
+
if (!allowedPaths.has(relativePath)) {
|
|
1175
|
+
if (replacePath) workingDeletes.add(replacePath);
|
|
1176
|
+
workingDeletes.add(relativePath);
|
|
1177
|
+
continue;
|
|
1178
|
+
}
|
|
1179
|
+
const file = await this.#prepareWorkingFile(relativePath, {
|
|
1180
|
+
blobOid: null,
|
|
1181
|
+
sourceMode: "working-tree",
|
|
1182
|
+
indexedCommit: null
|
|
1183
|
+
});
|
|
1184
|
+
if (!file) {
|
|
1185
|
+
if (replacePath) workingDeletes.add(replacePath);
|
|
1186
|
+
workingDeletes.add(relativePath);
|
|
1187
|
+
skippedWorkingPaths.add(relativePath);
|
|
1188
|
+
continue;
|
|
1189
|
+
}
|
|
1190
|
+
if (!file.unavailable && !replacePath && change.status === "A") {
|
|
1191
|
+
const contentHash = file.contentHash;
|
|
1192
|
+
const renameCandidates = workingFiles.filter((file2) => file2.path !== relativePath && !visibleWorkingPaths.has(file2.path) && deletes.has(file2.path) && file2.contentHash === contentHash);
|
|
1193
|
+
const renamed = renameCandidates.length === 1 ? renameCandidates[0] : void 0;
|
|
1194
|
+
if (renamed) {
|
|
1195
|
+
replacePath = renamed.path;
|
|
1196
|
+
previousPath = renamed.previousPath ?? renamed.path;
|
|
1197
|
+
deletes.delete(renamed.path);
|
|
1198
|
+
}
|
|
1199
|
+
}
|
|
1200
|
+
workingPrepared.push({
|
|
1201
|
+
...file,
|
|
1202
|
+
...previousPath ? { previousPath } : {},
|
|
1203
|
+
...replacePath ? { replacePath } : {}
|
|
1204
|
+
});
|
|
1205
|
+
}
|
|
1206
|
+
const prepared = [];
|
|
1207
|
+
for (const { entry, previousPath } of upserts.values()) {
|
|
1208
|
+
throwIfAborted(options.signal);
|
|
1209
|
+
const provenance = {
|
|
1210
|
+
blobOid: entry.oid,
|
|
1211
|
+
sourceMode: "git",
|
|
1212
|
+
indexedCommit: target,
|
|
1213
|
+
...previousPath ? { replacePath: previousPath } : {}
|
|
1214
|
+
};
|
|
1215
|
+
if (entry.size > this.#maxFileSize) {
|
|
1216
|
+
prepared.push(this.#failedFile(entry.path, "file-too-large", `File exceeds maxFileSize (${this.#maxFileSize} bytes).`, provenance, entry.size));
|
|
1217
|
+
continue;
|
|
1218
|
+
}
|
|
1219
|
+
try {
|
|
1220
|
+
const content = await this.#git.readBlob(entry.oid);
|
|
1221
|
+
prepared.push(this.#prepareFile(entry.path, content, provenance));
|
|
1222
|
+
} catch (error) {
|
|
1223
|
+
prepared.push(this.#failedFile(entry.path, "read-error", error instanceof Error ? error.message : String(error), provenance, entry.size));
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
let embeddingsCreated = await this.#attachEmbeddings([...prepared, ...workingPrepared], options.signal);
|
|
1227
|
+
const noChanges = !reconcileAll && !diagnosticsScan && checkpoint === target && prepared.length === 0 && deletes.size === 0 && workingPrepared.length === 0 && workingDeletes.size === 0;
|
|
1228
|
+
while (true) {
|
|
1229
|
+
const resolvedAgain = await this.#git.resolveCommit(options.target ?? "HEAD");
|
|
1230
|
+
if (resolvedAgain !== target) throw new CodeIndexError("Target Git ref changed while indexing; retry the update.");
|
|
1231
|
+
if (!overlayWorkingTree) break;
|
|
1232
|
+
const headAgain = await this.#git.resolveCommit("HEAD");
|
|
1233
|
+
const workingChangesAgain = await this.#git.workTreeChanges(target, indexArtifacts);
|
|
1234
|
+
if (headAgain !== head) throw new CodeIndexError("Git HEAD changed while indexing; retry the update.");
|
|
1235
|
+
if (JSON.stringify(workingChangesAgain) !== JSON.stringify(workingChanges)) return await this.updateFromGit(options);
|
|
1236
|
+
const changed = [];
|
|
1237
|
+
for (let index = 0; index < workingPrepared.length; index += 1) {
|
|
1238
|
+
const file = workingPrepared[index];
|
|
1239
|
+
if (!await this.#workingFileChanged(file)) continue;
|
|
1240
|
+
const refreshed = await this.#prepareWorkingFile(file.path, {
|
|
1241
|
+
blobOid: null,
|
|
1242
|
+
sourceMode: "working-tree",
|
|
1243
|
+
indexedCommit: null,
|
|
1244
|
+
...file.previousPath ? { previousPath: file.previousPath } : {},
|
|
1245
|
+
...file.replacePath ? { replacePath: file.replacePath } : {}
|
|
1246
|
+
});
|
|
1247
|
+
if (!refreshed) return await this.updateFromGit(options);
|
|
1248
|
+
workingPrepared[index] = refreshed;
|
|
1249
|
+
changed.push(refreshed);
|
|
1250
|
+
}
|
|
1251
|
+
for (const relativePath of skippedWorkingPaths) {
|
|
1252
|
+
const refreshed = await this.#prepareWorkingFile(relativePath, {
|
|
1253
|
+
blobOid: null,
|
|
1254
|
+
sourceMode: "working-tree",
|
|
1255
|
+
indexedCommit: null
|
|
1256
|
+
});
|
|
1257
|
+
if (refreshed && !refreshed.unavailable) return await this.updateFromGit(options);
|
|
1258
|
+
}
|
|
1259
|
+
if (changed.length > 0) {
|
|
1260
|
+
embeddingsCreated += await this.#attachEmbeddings(changed, options.signal);
|
|
1261
|
+
continue;
|
|
1262
|
+
}
|
|
1263
|
+
await gitignore.assertUnchanged(options.signal);
|
|
1264
|
+
break;
|
|
1265
|
+
}
|
|
1266
|
+
if (noChanges) {
|
|
1267
|
+
if (this.#database.getCheckpoint() !== checkpoint || this.#database.getGeneration() !== generation) {
|
|
1268
|
+
throw new CodeIndexError("Index changed while the update was being prepared; retry the update.");
|
|
1269
|
+
}
|
|
1270
|
+
return emptyStats(checkpoint);
|
|
1271
|
+
}
|
|
1272
|
+
return this.#database.applyUpdate({
|
|
1273
|
+
files: prepared,
|
|
1274
|
+
deletePaths: [...deletes],
|
|
1275
|
+
workingTree: {
|
|
1276
|
+
files: workingPrepared,
|
|
1277
|
+
deletePaths: [...workingDeletes]
|
|
1278
|
+
},
|
|
1279
|
+
checkpoint: target,
|
|
1280
|
+
expectedCheckpoint: checkpoint,
|
|
1281
|
+
expectedGeneration: generation,
|
|
1282
|
+
completeDiagnosticsScan: true,
|
|
1283
|
+
embeddingsCreated
|
|
1284
|
+
});
|
|
1285
|
+
}
|
|
1286
|
+
#classifyGitChanges(changes, tree, upserts, deletes, allowedPaths) {
|
|
1287
|
+
for (const change of changes) {
|
|
1288
|
+
if (change.status === "D") {
|
|
1289
|
+
deletes.add(change.path);
|
|
1290
|
+
continue;
|
|
1291
|
+
}
|
|
1292
|
+
if (change.status === "R") {
|
|
1293
|
+
deletes.add(change.oldPath);
|
|
1294
|
+
const entry2 = tree.get(change.path);
|
|
1295
|
+
if (entry2 && allowedPaths.has(change.path)) {
|
|
1296
|
+
upserts.set(change.path, { entry: entry2, previousPath: change.oldPath });
|
|
1297
|
+
}
|
|
1298
|
+
continue;
|
|
1299
|
+
}
|
|
1300
|
+
const entry = tree.get(change.path);
|
|
1301
|
+
if (entry && allowedPaths.has(change.path)) upserts.set(change.path, { entry });
|
|
1302
|
+
else deletes.add(change.path);
|
|
1303
|
+
}
|
|
1304
|
+
}
|
|
1305
|
+
async #workingTreeSourcePaths(gitignore, signal) {
|
|
1306
|
+
const relativeIndexPath = path5.relative(this.rootDir, this.indexPath).replaceAll(path5.sep, "/");
|
|
1307
|
+
const indexArtifacts = new Set(relativeIndexPath && !relativeIndexPath.startsWith("../") ? [relativeIndexPath, `${relativeIndexPath}-shm`, `${relativeIndexPath}-wal`, `${relativeIndexPath}-journal`] : []);
|
|
1308
|
+
const directories = [""];
|
|
1309
|
+
const sourcePaths = [];
|
|
1310
|
+
while (directories.length > 0) {
|
|
1311
|
+
throwIfAborted(signal);
|
|
1312
|
+
const relativeDirectory = directories.shift();
|
|
1313
|
+
const entries = await readdir(path5.join(this.rootDir, relativeDirectory), { withFileTypes: true });
|
|
1314
|
+
entries.sort((left, right) => left.name.localeCompare(right.name));
|
|
1315
|
+
for (const entry of entries) {
|
|
1316
|
+
const relativePath = relativeDirectory ? `${relativeDirectory.replaceAll(path5.sep, "/")}/${entry.name}` : entry.name;
|
|
1317
|
+
if (entry.isDirectory()) {
|
|
1318
|
+
if (this.#policy.traversesDirectory(relativePath) && !await gitignore.ignores(relativePath, true)) directories.push(relativePath);
|
|
1319
|
+
} else if (entry.isFile() && !indexArtifacts.has(relativePath) && this.#policy.includes(relativePath) && !await gitignore.ignores(relativePath)) {
|
|
1320
|
+
sourcePaths.push(relativePath);
|
|
1321
|
+
}
|
|
1322
|
+
}
|
|
1323
|
+
}
|
|
1324
|
+
sourcePaths.sort();
|
|
1325
|
+
return sourcePaths;
|
|
1326
|
+
}
|
|
1327
|
+
async #prepareWorkingPaths(relativePaths, signal) {
|
|
1328
|
+
const prepared = [];
|
|
1329
|
+
for (const relativePath of relativePaths) {
|
|
1330
|
+
throwIfAborted(signal);
|
|
1331
|
+
const file = await this.#prepareWorkingFile(relativePath, {
|
|
1332
|
+
blobOid: null,
|
|
1333
|
+
sourceMode: "working-tree",
|
|
1334
|
+
indexedCommit: null
|
|
1335
|
+
});
|
|
1336
|
+
if (file) prepared.push(file);
|
|
1337
|
+
}
|
|
1338
|
+
return prepared;
|
|
1339
|
+
}
|
|
1340
|
+
async #workingFileChanged(file) {
|
|
1341
|
+
try {
|
|
1342
|
+
const info = await lstat3(path5.join(this.rootDir, file.path));
|
|
1343
|
+
if (!info.isFile() || info.isSymbolicLink() || info.size !== file.byteSize) return true;
|
|
1344
|
+
if (info.size > this.#maxFileSize) return !file.errors.some((error) => error.code === "file-too-large");
|
|
1345
|
+
const content = await readFile2(path5.join(this.rootDir, file.path));
|
|
1346
|
+
return file.unavailable === true || sha256(content.toString("utf8")) !== file.contentHash;
|
|
1347
|
+
} catch {
|
|
1348
|
+
return file.unavailable !== true;
|
|
1349
|
+
}
|
|
1350
|
+
}
|
|
1351
|
+
#workingFileChangedSynchronously(file) {
|
|
1352
|
+
try {
|
|
1353
|
+
const absolutePath = path5.join(this.rootDir, file.path);
|
|
1354
|
+
const info = lstatSync(absolutePath);
|
|
1355
|
+
if (!info.isFile() || info.isSymbolicLink() || info.size !== file.byteSize) return true;
|
|
1356
|
+
if (info.size > this.#maxFileSize) return !file.errors.some((error) => error.code === "file-too-large");
|
|
1357
|
+
const contentHash = sha256(readFileSync(absolutePath, "utf8"));
|
|
1358
|
+
return file.unavailable === true || contentHash !== file.contentHash;
|
|
1359
|
+
} catch {
|
|
1360
|
+
return file.unavailable !== true;
|
|
1361
|
+
}
|
|
1362
|
+
}
|
|
1363
|
+
#prepareFile(relativePath, buffer, provenance) {
|
|
1364
|
+
const content = buffer.toString("utf8");
|
|
1365
|
+
const contentHash = sha256(content);
|
|
1366
|
+
const parseKey = sha256(`${CALLABLE_PARSER_CACHE_VERSION}\0${relativePath}\0${contentHash}`);
|
|
1367
|
+
let parsed = this.#database.cachedParse(parseKey);
|
|
1368
|
+
if (!parsed) {
|
|
1369
|
+
parsed = parseFileCallables(relativePath, content, this.#onWarning);
|
|
1370
|
+
if (!parsed.errors.some((error) => error.code === "parse-failed" || error.code === "extraction-error")) {
|
|
1371
|
+
this.#database.storeParse(parseKey, parsed);
|
|
1372
|
+
}
|
|
1373
|
+
} else if (parsed.errors.length > 0) {
|
|
1374
|
+
this.#onWarning(`Cannot fully parse ${relativePath}: tree-sitter reported syntax errors; indexing recoverable callables only.`);
|
|
1375
|
+
}
|
|
1376
|
+
const language = languageForPath(relativePath) ?? path5.extname(relativePath).slice(1);
|
|
1377
|
+
return {
|
|
1378
|
+
path: relativePath,
|
|
1379
|
+
contentHash,
|
|
1380
|
+
blobOid: provenance.blobOid,
|
|
1381
|
+
sourceMode: provenance.sourceMode,
|
|
1382
|
+
indexedCommit: provenance.indexedCommit,
|
|
1383
|
+
language,
|
|
1384
|
+
byteSize: buffer.byteLength,
|
|
1385
|
+
source: content,
|
|
1386
|
+
...provenance.previousPath ? { previousPath: provenance.previousPath } : {},
|
|
1387
|
+
...provenance.replacePath ? { replacePath: provenance.replacePath } : {},
|
|
1388
|
+
callables: parsed.callables,
|
|
1389
|
+
errors: parsed.errors
|
|
1390
|
+
};
|
|
1391
|
+
}
|
|
1392
|
+
async #prepareWorkingFile(relativePath, provenance, strict = false) {
|
|
1393
|
+
let size = 0;
|
|
1394
|
+
try {
|
|
1395
|
+
const absolutePath = path5.join(this.rootDir, relativePath);
|
|
1396
|
+
const info = await lstat3(absolutePath);
|
|
1397
|
+
if (info.isSymbolicLink() || !info.isFile()) {
|
|
1398
|
+
if (strict) throw new CodeIndexError(`Not a regular file or symbolic links are not supported: ${relativePath}`);
|
|
1399
|
+
return null;
|
|
1400
|
+
}
|
|
1401
|
+
size = info.size;
|
|
1402
|
+
if (size > this.#maxFileSize) {
|
|
1403
|
+
return this.#failedFile(relativePath, "file-too-large", `File exceeds maxFileSize (${this.#maxFileSize} bytes).`, provenance, size);
|
|
1404
|
+
}
|
|
1405
|
+
return this.#prepareFile(relativePath, await readFile2(absolutePath), provenance);
|
|
1406
|
+
} catch (error) {
|
|
1407
|
+
if (error instanceof CodeIndexError) throw error;
|
|
1408
|
+
return this.#failedFile(relativePath, "read-error", error instanceof Error ? error.message : String(error), provenance, size);
|
|
1409
|
+
}
|
|
1410
|
+
}
|
|
1411
|
+
#failedFile(relativePath, code, message, provenance, byteSize) {
|
|
1412
|
+
this.#onWarning(`Cannot index ${relativePath}: ${message}`);
|
|
1413
|
+
return {
|
|
1414
|
+
...provenance,
|
|
1415
|
+
path: relativePath,
|
|
1416
|
+
contentHash: sha256(""),
|
|
1417
|
+
source: "",
|
|
1418
|
+
byteSize,
|
|
1419
|
+
language: languageForPath(relativePath) ?? "unknown",
|
|
1420
|
+
callables: [],
|
|
1421
|
+
unavailable: true,
|
|
1422
|
+
errors: [{
|
|
1423
|
+
path: relativePath,
|
|
1424
|
+
language: languageForPath(relativePath),
|
|
1425
|
+
scope: "file",
|
|
1426
|
+
code,
|
|
1427
|
+
message,
|
|
1428
|
+
qualifiedName: null,
|
|
1429
|
+
startLine: null,
|
|
1430
|
+
startColumn: null,
|
|
1431
|
+
endLine: null,
|
|
1432
|
+
endColumn: null,
|
|
1433
|
+
source: null
|
|
1434
|
+
}]
|
|
1435
|
+
};
|
|
1436
|
+
}
|
|
1437
|
+
async #attachEmbeddings(files, signal) {
|
|
1438
|
+
if (this.#database.descriptionsEnabled()) {
|
|
1439
|
+
await this.#attachDescriptions(files.filter((file) => !file.unavailable), signal);
|
|
1440
|
+
}
|
|
1441
|
+
throwIfAborted(signal);
|
|
1442
|
+
const profile = JSON.stringify(normalizeProfile(this.provider.profile));
|
|
1443
|
+
const unique = /* @__PURE__ */ new Map();
|
|
1444
|
+
for (const file of files) {
|
|
1445
|
+
for (const callable of file.callables) {
|
|
1446
|
+
callable.embeddingKey = embeddingKey(profile, "document", callable.embeddingInput);
|
|
1447
|
+
const values = unique.get(callable.embeddingKey) ?? [];
|
|
1448
|
+
values.push(callable);
|
|
1449
|
+
unique.set(callable.embeddingKey, values);
|
|
1450
|
+
}
|
|
1451
|
+
}
|
|
1452
|
+
const missing = [];
|
|
1453
|
+
for (const [key, callables] of unique) {
|
|
1454
|
+
const vector = this.#database.cachedEmbedding(key);
|
|
1455
|
+
if (vector) {
|
|
1456
|
+
for (const callable of callables) callable.vector = vector;
|
|
1457
|
+
} else {
|
|
1458
|
+
missing.push([key, callables]);
|
|
1459
|
+
}
|
|
1460
|
+
}
|
|
1461
|
+
if (missing.length === 0) return 0;
|
|
1462
|
+
let completed = 0;
|
|
1463
|
+
this.#progress("vectors", 0, missing.length);
|
|
1464
|
+
await forEachConcurrent(chunk(missing, this.#embeddingBatchSize), this.#parallelism, async (batch, _index, workerSignal) => {
|
|
1465
|
+
const vectors = await this.provider.embedDocuments(
|
|
1466
|
+
batch.map(([, callables]) => callables[0].embeddingInput),
|
|
1467
|
+
{ signal: workerSignal }
|
|
1468
|
+
);
|
|
1469
|
+
if (vectors.length !== batch.length) throw new CodeIndexError("Embedding provider returned an unexpected number of vectors.");
|
|
1470
|
+
vectors.forEach((vector, index) => {
|
|
1471
|
+
const converted = normalizeEmbeddingVector(vector, this.provider.profile.dimensions);
|
|
1472
|
+
this.#database.storeEmbedding(batch[index][0], converted);
|
|
1473
|
+
for (const callable of batch[index][1]) callable.vector = converted;
|
|
1474
|
+
});
|
|
1475
|
+
completed += batch.length;
|
|
1476
|
+
this.#progress("vectors", completed, missing.length);
|
|
1477
|
+
}, signal);
|
|
1478
|
+
throwIfAborted(signal);
|
|
1479
|
+
return missing.length;
|
|
1480
|
+
}
|
|
1481
|
+
async #attachDescriptions(files, signal, options = {}) {
|
|
1482
|
+
const profile = descriptionProfileJson(this.descriptionProvider.profile);
|
|
1483
|
+
const embeddingProfile = JSON.stringify(normalizeProfile(this.provider.profile));
|
|
1484
|
+
const prepared = /* @__PURE__ */ new Map();
|
|
1485
|
+
const repository = path5.basename(this.rootDir);
|
|
1486
|
+
let descriptionsCreated = 0;
|
|
1487
|
+
let fileDescriptionsCreated = 0;
|
|
1488
|
+
let pending = 0;
|
|
1489
|
+
const tasks = [];
|
|
1490
|
+
const strategy = this.descriptionProvider.profile.strategyVersion;
|
|
1491
|
+
const useLiveReuse = !options.skipCallables && !options.reindexCache && !options.forceCallableDescriptions && !options.ignoreLiveDescriptions;
|
|
1492
|
+
const liveCallableDescriptions = useLiveReuse ? this.#database.liveFunctionDescriptions(files.flatMap((file) => file.callables.map((callable) => callable.identityKey))) : /* @__PURE__ */ new Map();
|
|
1493
|
+
const liveFileHashes = useLiveReuse && files.length > 0 ? this.#database.liveFileContentHashes(files.map((file) => file.replacePath ?? file.previousPath ?? file.path)) : /* @__PURE__ */ new Map();
|
|
1494
|
+
for (const file of files) {
|
|
1495
|
+
const lookupPath = file.replacePath ?? file.previousPath ?? file.path;
|
|
1496
|
+
const storedFileDescription = this.#database.fileDescription(lookupPath);
|
|
1497
|
+
const fileDescriptionKey = sha256(`${profile}\0${repository}\0${file.path}\0${file.contentHash}\0file`);
|
|
1498
|
+
const fileAgnosticKey = sha256(`${strategy}\0${repository}\0${file.path}\0${file.contentHash}\0file`);
|
|
1499
|
+
const fileCacheKey = options.reindexCache ? sha256(`${fileDescriptionKey}\0reindex\0${storedFileDescription?.description ?? ""}`) : fileDescriptionKey;
|
|
1500
|
+
const refreshFileDescription = options.refreshFileDescriptions || !storedFileDescription;
|
|
1501
|
+
const fileDescription = refreshFileDescription ? this.#database.cachedDescription(fileCacheKey) ?? (!options.reindexCache ? this.#database.cachedDescription(fileAgnosticKey) : void 0) : storedFileDescription.description;
|
|
1502
|
+
const fileUnchanged = liveFileHashes.get(lookupPath) === file.contentHash;
|
|
1503
|
+
const descriptions = (options.skipCallables ? [] : file.callables).map((callable) => {
|
|
1504
|
+
const descriptionKey = sha256(`${profile}\0${repository}\0${file.path}\0${file.contentHash}\0${callable.identityKey}\0${callable.sourceHash}`);
|
|
1505
|
+
const agnosticKey = sha256(`${strategy}\0${repository}\0${file.path}\0${file.contentHash}\0${callable.identityKey}\0${callable.sourceHash}`);
|
|
1506
|
+
const cacheKey = options.reindexCache ? sha256(`${descriptionKey}\0reindex\0${this.#database.functionDescription(callable.identityKey) ?? ""}`) : descriptionKey;
|
|
1507
|
+
if (useLiveReuse && fileUnchanged) {
|
|
1508
|
+
const live = liveCallableDescriptions.get(callable.identityKey);
|
|
1509
|
+
if (live?.description && live.sourceHash === callable.sourceHash) {
|
|
1510
|
+
return {
|
|
1511
|
+
callable,
|
|
1512
|
+
descriptionKey,
|
|
1513
|
+
cacheKey,
|
|
1514
|
+
agnosticKey,
|
|
1515
|
+
description: live.description
|
|
1516
|
+
};
|
|
1517
|
+
}
|
|
1518
|
+
}
|
|
1519
|
+
const description = options.forceCallableDescriptions || options.reindexCache ? this.#database.cachedDescription(cacheKey) : this.#database.cachedDescription(descriptionKey) ?? this.#database.cachedDescription(agnosticKey);
|
|
1520
|
+
return {
|
|
1521
|
+
callable,
|
|
1522
|
+
descriptionKey,
|
|
1523
|
+
cacheKey,
|
|
1524
|
+
agnosticKey,
|
|
1525
|
+
...description ? { description } : {}
|
|
1526
|
+
};
|
|
1527
|
+
});
|
|
1528
|
+
pending += (refreshFileDescription && !fileDescription ? 1 : 0) + descriptions.filter(({ description }) => !description).length;
|
|
1529
|
+
tasks.push({
|
|
1530
|
+
file,
|
|
1531
|
+
fileCacheKey,
|
|
1532
|
+
fileAgnosticKey,
|
|
1533
|
+
fileDescriptionKey,
|
|
1534
|
+
refreshFileDescription,
|
|
1535
|
+
...fileDescription ? { fileDescription } : {},
|
|
1536
|
+
fileDescriptionPath: refreshFileDescription ? file.path : storedFileDescription.path,
|
|
1537
|
+
fileDescriptionContentHash: refreshFileDescription ? file.contentHash : storedFileDescription.contentHash,
|
|
1538
|
+
descriptions
|
|
1539
|
+
});
|
|
1540
|
+
}
|
|
1541
|
+
let completed = 0;
|
|
1542
|
+
if (pending > 0) this.#progress("descriptions", 0, pending);
|
|
1543
|
+
await forEachConcurrent(tasks, this.#parallelism, async (task, _index, workerSignal) => {
|
|
1544
|
+
const { file } = task;
|
|
1545
|
+
const session = task.refreshFileDescription || task.descriptions.some(({ description }) => !description) ? this.descriptionProvider.startFile?.({ repository, path: file.path, fileSource: file.source }) : void 0;
|
|
1546
|
+
let fileDescription = task.fileDescription;
|
|
1547
|
+
let fileDescriptionAddedToSession = false;
|
|
1548
|
+
if (task.refreshFileDescription && !fileDescription) {
|
|
1549
|
+
const generated = session ? await session.describeFile({ signal: workerSignal }) : await this.descriptionProvider.describeFile(
|
|
1550
|
+
{ repository, path: file.path, fileSource: file.source },
|
|
1551
|
+
{ signal: workerSignal }
|
|
1552
|
+
);
|
|
1553
|
+
if (typeof generated !== "string" || !generated.trim()) {
|
|
1554
|
+
throw new CodeIndexError("Description provider does not support file descriptions or returned an empty description.");
|
|
1555
|
+
}
|
|
1556
|
+
fileDescription = this.#database.storeDescription(task.fileCacheKey, generated.trim());
|
|
1557
|
+
this.#database.storeDescription(task.fileAgnosticKey, fileDescription);
|
|
1558
|
+
fileDescriptionsCreated += 1;
|
|
1559
|
+
fileDescriptionAddedToSession = session !== void 0;
|
|
1560
|
+
completed += 1;
|
|
1561
|
+
this.#progress("descriptions", completed, pending);
|
|
1562
|
+
}
|
|
1563
|
+
if (session && fileDescription && !fileDescriptionAddedToSession) session.replayFile(fileDescription);
|
|
1564
|
+
if (fileDescription) {
|
|
1565
|
+
const value = prepareDescription(prepared, embeddingProfile, fileDescription, this.#database);
|
|
1566
|
+
file.fileDescription = {
|
|
1567
|
+
path: task.fileDescriptionPath,
|
|
1568
|
+
contentHash: task.fileDescriptionContentHash,
|
|
1569
|
+
descriptionKey: task.fileDescriptionKey,
|
|
1570
|
+
value
|
|
1571
|
+
};
|
|
1572
|
+
}
|
|
1573
|
+
for (const entry of task.descriptions) {
|
|
1574
|
+
throwIfAborted(workerSignal);
|
|
1575
|
+
const { callable, descriptionKey, cacheKey, agnosticKey } = entry;
|
|
1576
|
+
let { description } = entry;
|
|
1577
|
+
if (!description) {
|
|
1578
|
+
const generated = session ? await session.describe(callable, { signal: workerSignal }) : await this.descriptionProvider.describe({ repository, callable, fileSource: file.source }, { signal: workerSignal });
|
|
1579
|
+
if (typeof generated !== "string" || !generated.trim()) throw new CodeIndexError("Description provider returned an empty description.");
|
|
1580
|
+
description = this.#database.storeDescription(cacheKey, generated.trim());
|
|
1581
|
+
this.#database.storeDescription(agnosticKey, description);
|
|
1582
|
+
descriptionsCreated += 1;
|
|
1583
|
+
completed += 1;
|
|
1584
|
+
this.#progress("descriptions", completed, pending);
|
|
1585
|
+
} else {
|
|
1586
|
+
session?.replay(callable, description);
|
|
1587
|
+
}
|
|
1588
|
+
callable.descriptionKey = descriptionKey;
|
|
1589
|
+
callable.description = prepareDescription(prepared, embeddingProfile, description, this.#database);
|
|
1590
|
+
}
|
|
1591
|
+
}, signal);
|
|
1592
|
+
const missing = [...prepared.values()].filter((description) => !description.vector);
|
|
1593
|
+
if (missing.length > 0) {
|
|
1594
|
+
let embedded = 0;
|
|
1595
|
+
this.#progress("description-vectors", 0, missing.length);
|
|
1596
|
+
await forEachConcurrent(chunk(missing, this.#embeddingBatchSize), this.#parallelism, async (batch, _index, workerSignal) => {
|
|
1597
|
+
const vectors = await this.provider.embedDocuments(batch.map((value) => value.description), { signal: workerSignal });
|
|
1598
|
+
if (vectors.length !== batch.length) throw new CodeIndexError("Embedding provider returned an unexpected number of description vectors.");
|
|
1599
|
+
vectors.forEach((vector, index) => {
|
|
1600
|
+
const converted = normalizeEmbeddingVector(vector, this.provider.profile.dimensions);
|
|
1601
|
+
this.#database.storeEmbedding(batch[index].key, converted);
|
|
1602
|
+
batch[index].vector = converted;
|
|
1603
|
+
});
|
|
1604
|
+
embedded += batch.length;
|
|
1605
|
+
this.#progress("description-vectors", embedded, missing.length);
|
|
1606
|
+
}, signal);
|
|
1607
|
+
}
|
|
1608
|
+
throwIfAborted(signal);
|
|
1609
|
+
return { descriptionsCreated, fileDescriptionsCreated };
|
|
1610
|
+
}
|
|
1611
|
+
async useDescriptions(options = {}) {
|
|
1612
|
+
throwIfAborted(options.signal);
|
|
1613
|
+
const generation = this.#database.getGeneration();
|
|
1614
|
+
const functions = this.allFunctions();
|
|
1615
|
+
const status = this.status();
|
|
1616
|
+
const storedProfile = this.#database.descriptionProfile();
|
|
1617
|
+
const strategyChanged = (storedProfile?.strategyVersion ?? this.descriptionProvider.profile.strategyVersion) !== this.descriptionProvider.profile.strategyVersion;
|
|
1618
|
+
const profileChanged = descriptionProfileJson(storedProfile ?? this.descriptionProvider.profile) !== descriptionProfileJson(this.descriptionProvider.profile);
|
|
1619
|
+
if (this.#database.descriptionsEnabled() && !strategyChanged && functions.every((callable) => callable.descriptionEmbeddingId !== null) && status.fileDescriptionCount === status.describableFileCount) {
|
|
1620
|
+
if (profileChanged) this.#database.updateDescriptionProfile(this.descriptionProvider.profile, generation);
|
|
1621
|
+
return { descriptionsCreated: 0, fileDescriptionsCreated: 0, descriptionsEnabled: true };
|
|
1622
|
+
}
|
|
1623
|
+
const files = await this.#descriptionFiles(this.#database.getFileStates().filter((file) => file.describable), functions, options.signal);
|
|
1624
|
+
const { descriptionsCreated, fileDescriptionsCreated } = await this.#attachDescriptions(files, options.signal, {
|
|
1625
|
+
refreshFileDescriptions: strategyChanged,
|
|
1626
|
+
...strategyChanged ? { ignoreLiveDescriptions: true } : {}
|
|
1627
|
+
});
|
|
1628
|
+
const ids = new Map(functions.map((callable) => [callable.identityKey, callable.id]));
|
|
1629
|
+
this.#database.enableDescriptions(files.flatMap((file) => file.callables.map((callable) => ({
|
|
1630
|
+
id: ids.get(callable.identityKey),
|
|
1631
|
+
description: callable.description
|
|
1632
|
+
}))), files, this.descriptionProvider.profile, generation);
|
|
1633
|
+
return { descriptionsCreated, fileDescriptionsCreated, descriptionsEnabled: true };
|
|
1634
|
+
}
|
|
1635
|
+
async reindexFiles(options = {}) {
|
|
1636
|
+
throwIfAborted(options.signal);
|
|
1637
|
+
if (!this.#database.descriptionsEnabled()) throw new CodeIndexError("Descriptions are not enabled; run descriptions enable first.");
|
|
1638
|
+
const generation = this.#database.getGeneration();
|
|
1639
|
+
const states = this.#database.getFileStates().filter((file) => file.describable && (file.fileDescriptionPath !== file.path || file.fileDescriptionContentHash !== file.contentHash));
|
|
1640
|
+
const files = await this.#descriptionFiles(states, this.allFunctions(), options.signal);
|
|
1641
|
+
const created = await this.#attachDescriptions(files, options.signal, {
|
|
1642
|
+
refreshFileDescriptions: true,
|
|
1643
|
+
...options.includeCallables !== void 0 ? { forceCallableDescriptions: options.includeCallables } : {},
|
|
1644
|
+
reindexCache: true,
|
|
1645
|
+
skipCallables: !options.includeCallables
|
|
1646
|
+
});
|
|
1647
|
+
this.#database.updateDescriptions(files, options.includeCallables ?? false, generation, this.descriptionProvider.profile);
|
|
1648
|
+
return { filesReindexed: files.length, ...created };
|
|
1649
|
+
}
|
|
1650
|
+
async #descriptionFiles(states, functions, signal) {
|
|
1651
|
+
const byPath = groupBy(functions, (callable) => callable.path);
|
|
1652
|
+
const files = [];
|
|
1653
|
+
for (const file of states) {
|
|
1654
|
+
throwIfAborted(signal);
|
|
1655
|
+
const buffer = file.sourceMode === "git" && file.blobOid ? await this.#git.readBlob(file.blobOid) : await readFile2(path5.join(this.rootDir, file.path));
|
|
1656
|
+
const source = buffer.toString("utf8");
|
|
1657
|
+
if (sha256(source) !== file.contentHash) {
|
|
1658
|
+
throw new CodeIndexError(`Source changed since indexing: ${file.path}; update the index first.`);
|
|
1659
|
+
}
|
|
1660
|
+
const callables = byPath.get(file.path) ?? [];
|
|
1661
|
+
files.push({
|
|
1662
|
+
path: file.path,
|
|
1663
|
+
contentHash: file.contentHash,
|
|
1664
|
+
blobOid: file.blobOid,
|
|
1665
|
+
sourceMode: file.sourceMode,
|
|
1666
|
+
indexedCommit: null,
|
|
1667
|
+
language: file.language,
|
|
1668
|
+
byteSize: buffer.byteLength,
|
|
1669
|
+
source,
|
|
1670
|
+
errors: [],
|
|
1671
|
+
callables: callables.map(({ description: _description, descriptionEmbeddingId: _descriptionEmbeddingId, ...callable }) => ({
|
|
1672
|
+
...callable,
|
|
1673
|
+
embeddingKey: ""
|
|
1674
|
+
}))
|
|
1675
|
+
});
|
|
1676
|
+
}
|
|
1677
|
+
return files;
|
|
1678
|
+
}
|
|
1679
|
+
disableDescriptions() {
|
|
1680
|
+
this.#database.disableDescriptions();
|
|
1681
|
+
return { descriptionsCreated: 0, fileDescriptionsCreated: 0, descriptionsEnabled: false };
|
|
1682
|
+
}
|
|
1683
|
+
async searchDescription(options) {
|
|
1684
|
+
if (!this.#database.descriptionsEnabled()) throw new CodeIndexError("Descriptions are not enabled; run descriptions enable first.");
|
|
1685
|
+
if (!options.query.trim()) throw new CodeIndexError("query must not be empty.");
|
|
1686
|
+
const limit = options.limit;
|
|
1687
|
+
if (limit !== void 0) assertPositiveInteger(limit, "limit");
|
|
1688
|
+
const candidateLimit = this.#candidateLimit(limit);
|
|
1689
|
+
compileNameRegex(options.nameRegex);
|
|
1690
|
+
throwIfAborted(options.signal);
|
|
1691
|
+
const vector = await this.#queryEmbedding(options.query, options.signal);
|
|
1692
|
+
throwIfAborted(options.signal);
|
|
1693
|
+
const includeFileDescriptions = this.#descriptionScoringAvailable();
|
|
1694
|
+
const results = this.#database.searchVector(vector, {
|
|
1695
|
+
descriptions: true,
|
|
1696
|
+
...candidateLimit === void 0 ? {} : { limit: candidateLimit },
|
|
1697
|
+
minSimilarity: options.minSimilarity ?? -1,
|
|
1698
|
+
...includeFileDescriptions ? { fileDescriptionVector: vector } : {},
|
|
1699
|
+
...options.nameRegex !== void 0 ? { nameRegex: options.nameRegex } : {},
|
|
1700
|
+
...options.maxSimilarity !== void 0 ? { maxSimilarity: options.maxSimilarity } : {}
|
|
1701
|
+
});
|
|
1702
|
+
return await this.#rerank(options.query, results, limit, options.signal);
|
|
1703
|
+
}
|
|
1704
|
+
async similaritySearch(options) {
|
|
1705
|
+
if (!options.query.trim()) throw new CodeIndexError("query must not be empty.");
|
|
1706
|
+
const limit = options.limit;
|
|
1707
|
+
if (limit !== void 0) assertPositiveInteger(limit, "limit");
|
|
1708
|
+
const candidateLimit = this.#candidateLimit(limit);
|
|
1709
|
+
compileNameRegex(options.nameRegex);
|
|
1710
|
+
throwIfAborted(options.signal);
|
|
1711
|
+
const vector = await this.#queryEmbedding(options.query, options.signal);
|
|
1712
|
+
const includeDescriptions = this.#descriptionScoringAvailable();
|
|
1713
|
+
const results = this.searchByVector(vector, {
|
|
1714
|
+
...includeDescriptions ? { descriptionVector: vector, fileDescriptionVector: vector } : {},
|
|
1715
|
+
...candidateLimit === void 0 ? {} : { limit: candidateLimit },
|
|
1716
|
+
...options.nameRegex !== void 0 ? { nameRegex: options.nameRegex } : {},
|
|
1717
|
+
minSimilarity: options.minSimilarity ?? -1,
|
|
1718
|
+
...options.maxSimilarity !== void 0 ? { maxSimilarity: options.maxSimilarity } : {}
|
|
1719
|
+
});
|
|
1720
|
+
return await this.#rerank(options.query, results, limit, options.signal);
|
|
1721
|
+
}
|
|
1722
|
+
#candidateLimit(limit) {
|
|
1723
|
+
if (limit === void 0) return void 0;
|
|
1724
|
+
if (!this.reranker) return limit;
|
|
1725
|
+
if (this.reranker.maximumCandidateCount !== void 0 && limit > this.reranker.maximumCandidateCount) {
|
|
1726
|
+
throw new CodeIndexError(`${this.reranker.profile.provider} reranker supports at most ${this.reranker.maximumCandidateCount} results.`);
|
|
1727
|
+
}
|
|
1728
|
+
const preferred = this.reranker.candidateCount === void 0 ? limit * RERANK_CANDIDATE_MULTIPLIER : Math.max(limit, this.reranker.candidateCount);
|
|
1729
|
+
return this.reranker.maximumCandidateCount === void 0 ? preferred : Math.min(preferred, this.reranker.maximumCandidateCount);
|
|
1730
|
+
}
|
|
1731
|
+
async #rerank(query, candidates, limit, signal) {
|
|
1732
|
+
if (!this.reranker || candidates.length === 0) return limit === void 0 ? candidates : candidates.slice(0, limit);
|
|
1733
|
+
const documents = candidates.map(({ function: callable }) => [
|
|
1734
|
+
`path: ${callable.path}`,
|
|
1735
|
+
callable.description ? `description:
|
|
1736
|
+
${callable.description}` : null,
|
|
1737
|
+
callable.embeddingInput
|
|
1738
|
+
].filter((value) => value !== null).join("\n"));
|
|
1739
|
+
const rerankLimit = limit === void 0 ? candidates.length : Math.min(limit, candidates.length);
|
|
1740
|
+
const rankings = await this.reranker.rerank(query, documents, signal ? { limit: rerankLimit, signal } : { limit: rerankLimit });
|
|
1741
|
+
throwIfAborted(signal);
|
|
1742
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1743
|
+
if (rankings.length !== rerankLimit) {
|
|
1744
|
+
throw new CodeIndexError(`${this.reranker.profile.provider} returned invalid reranking results.`);
|
|
1745
|
+
}
|
|
1746
|
+
for (const { index, score } of rankings) {
|
|
1747
|
+
if (!Number.isInteger(index) || index < 0 || index >= candidates.length || typeof score !== "number" || !Number.isFinite(score) || seen.has(index)) {
|
|
1748
|
+
throw new CodeIndexError(`${this.reranker.profile.provider} returned invalid reranking results.`);
|
|
1749
|
+
}
|
|
1750
|
+
seen.add(index);
|
|
1751
|
+
}
|
|
1752
|
+
return rankings.map(({ index, score }) => ({ ...candidates[index], rerankScore: score }));
|
|
1753
|
+
}
|
|
1754
|
+
async #queryEmbedding(query, signal) {
|
|
1755
|
+
const profile = JSON.stringify(normalizeProfile(this.provider.profile));
|
|
1756
|
+
const key = embeddingKey(profile, "query", query);
|
|
1757
|
+
const cached = this.#database.cachedEmbedding(key);
|
|
1758
|
+
if (cached) return cached;
|
|
1759
|
+
const vector = normalizeEmbeddingVector(
|
|
1760
|
+
await this.provider.embedQuery(query, signal ? { signal } : void 0),
|
|
1761
|
+
this.provider.profile.dimensions
|
|
1762
|
+
);
|
|
1763
|
+
this.#database.storeEmbedding(key, vector);
|
|
1764
|
+
return vector;
|
|
1765
|
+
}
|
|
1766
|
+
similarToFunction(functionId, options) {
|
|
1767
|
+
const vector = this.#database.vectorForFunction(functionId);
|
|
1768
|
+
return this.#database.searchVector(vector, {
|
|
1769
|
+
...options.includeDescriptions ? { descriptionVector: this.#database.vectorForFunction(functionId, "description") } : {},
|
|
1770
|
+
...options.includeDescriptions ? { fileDescriptionVector: this.#database.fileVectorForFunction(functionId) } : {},
|
|
1771
|
+
limit: options.limit,
|
|
1772
|
+
minSimilarity: options.minSimilarity,
|
|
1773
|
+
...options.maxSimilarity !== void 0 ? { maxSimilarity: options.maxSimilarity } : {},
|
|
1774
|
+
excludeId: functionId,
|
|
1775
|
+
...options.excludePaths !== void 0 ? { excludePaths: options.excludePaths } : {},
|
|
1776
|
+
...options.minLines !== void 0 ? { minLines: options.minLines } : {},
|
|
1777
|
+
...options.nameRegex !== void 0 ? { nameRegex: options.nameRegex } : {}
|
|
1778
|
+
});
|
|
1779
|
+
}
|
|
1780
|
+
vectorForFunction(functionId, kind = "code") {
|
|
1781
|
+
return this.#database.vectorForFunction(functionId, kind);
|
|
1782
|
+
}
|
|
1783
|
+
get readOnly() {
|
|
1784
|
+
return this.#database.isReadOnly;
|
|
1785
|
+
}
|
|
1786
|
+
similarityCacheInfo() {
|
|
1787
|
+
return this.#database.similarityCacheInfo();
|
|
1788
|
+
}
|
|
1789
|
+
/**
|
|
1790
|
+
* Incrementally refresh the persisted pairwise-similarity cache for same-index
|
|
1791
|
+
* analysis. Only pairs scoring at or above `minSimilarity` (the caller's
|
|
1792
|
+
* effective threshold) are stored; queries below the cached floor fall back to
|
|
1793
|
+
* live vector search. Only functions whose embeddings changed (plus a merge
|
|
1794
|
+
* pass over the remaining functions when the changed set is small) are
|
|
1795
|
+
* re-queried; unchanged pairs are reused. Lowering the floor below what is
|
|
1796
|
+
* cached re-queries every function, since the missing band cannot be rebuilt
|
|
1797
|
+
* incrementally; raising it reuses the cache for free. A per-function state
|
|
1798
|
+
* row records the embedding triple, floor, row count, and a completeness flag,
|
|
1799
|
+
* so a sparse row set is never mistaken for an unfinished computation. Call
|
|
1800
|
+
* before cross-search/cohesion, then read through
|
|
1801
|
+
* {@link cachedSimilarToFunction}. Best-effort under concurrent writers: a
|
|
1802
|
+
* later refresh repairs any rows raced by an overlapping update.
|
|
1803
|
+
*/
|
|
1804
|
+
async refreshSimilarityCache(options = {}) {
|
|
1805
|
+
const width = Math.min(
|
|
1806
|
+
MAX_SIMILARITY_CACHE_WIDTH,
|
|
1807
|
+
Math.max(1, Math.floor(options.width ?? DEFAULT_SIMILARITY_CACHE_WIDTH))
|
|
1808
|
+
);
|
|
1809
|
+
const floor = options.minSimilarity ?? -1;
|
|
1810
|
+
const mode = analysisSimilarity(this.status()).similarityMode;
|
|
1811
|
+
const includeDescriptions = mode !== "code";
|
|
1812
|
+
if (this.#database.isReadOnly) {
|
|
1813
|
+
return { similarityMode: mode, width, minSimilarity: floor, sourcesRefreshed: 0, pairsStored: 0, skipped: true };
|
|
1814
|
+
}
|
|
1815
|
+
throwIfAborted(options.signal);
|
|
1816
|
+
const generation = this.#database.getGeneration();
|
|
1817
|
+
const triples = this.#database.similarityCacheTriples();
|
|
1818
|
+
if (triples.length <= 1) {
|
|
1819
|
+
return { similarityMode: mode, width, minSimilarity: floor, sourcesRefreshed: 0, pairsStored: 0, skipped: false };
|
|
1820
|
+
}
|
|
1821
|
+
const states = this.#database.similarityCacheStates(mode);
|
|
1822
|
+
const tripleById = new Map(triples.map((triple) => [triple.functionId, triple]));
|
|
1823
|
+
const dirty = /* @__PURE__ */ new Set();
|
|
1824
|
+
let expanding = false;
|
|
1825
|
+
for (const triple of triples) {
|
|
1826
|
+
throwIfAborted(options.signal);
|
|
1827
|
+
const state = states.get(triple.functionId);
|
|
1828
|
+
if (!state || state.codeEmbeddingId !== triple.codeEmbeddingId || state.descriptionEmbeddingId !== triple.descriptionEmbeddingId || state.fileDescriptionEmbeddingId !== triple.fileDescriptionEmbeddingId || state.cachedWidth < width && !state.complete) {
|
|
1829
|
+
dirty.add(triple.functionId);
|
|
1830
|
+
} else if (state.floor > floor) {
|
|
1831
|
+
expanding = true;
|
|
1832
|
+
dirty.add(triple.functionId);
|
|
1833
|
+
}
|
|
1834
|
+
}
|
|
1835
|
+
if (dirty.size === 0 && !expanding) {
|
|
1836
|
+
const counts = this.#database.similarityCacheCounts(mode);
|
|
1837
|
+
for (const triple of triples) {
|
|
1838
|
+
if ((counts.get(triple.functionId) ?? 0) < (states.get(triple.functionId)?.storedCount ?? 0)) {
|
|
1839
|
+
dirty.add(triple.functionId);
|
|
1840
|
+
}
|
|
1841
|
+
}
|
|
1842
|
+
}
|
|
1843
|
+
let sourcesRefreshed = 0;
|
|
1844
|
+
let pairsStored = 0;
|
|
1845
|
+
if (dirty.size === 0) {
|
|
1846
|
+
return { similarityMode: mode, width, minSimilarity: floor, sourcesRefreshed, pairsStored, skipped: false };
|
|
1847
|
+
}
|
|
1848
|
+
const report = options.onProgress ?? this.#onProgress;
|
|
1849
|
+
const total = triples.length;
|
|
1850
|
+
let completed = 0;
|
|
1851
|
+
report?.({ phase: "similarity-cache", completed, total });
|
|
1852
|
+
const fullRefresh = states.size === 0 || expanding || dirty.size > Math.max(8, Math.ceil(triples.length * 0.25));
|
|
1853
|
+
if (fullRefresh) {
|
|
1854
|
+
for (const triple of triples) {
|
|
1855
|
+
throwIfAborted(options.signal);
|
|
1856
|
+
const scanned = this.similarToFunction(triple.functionId, {
|
|
1857
|
+
...includeDescriptions ? { includeDescriptions: true } : {},
|
|
1858
|
+
limit: Math.min(width + 1, triples.length - 1),
|
|
1859
|
+
minSimilarity: floor
|
|
1860
|
+
});
|
|
1861
|
+
const neighbors = scanned.slice(0, width);
|
|
1862
|
+
this.#database.storeSimilarityNeighbors(triple.functionId, mode, neighbors, {
|
|
1863
|
+
codeEmbeddingId: triple.codeEmbeddingId,
|
|
1864
|
+
descriptionEmbeddingId: triple.descriptionEmbeddingId,
|
|
1865
|
+
fileDescriptionEmbeddingId: triple.fileDescriptionEmbeddingId,
|
|
1866
|
+
cachedWidth: width,
|
|
1867
|
+
generation,
|
|
1868
|
+
floor,
|
|
1869
|
+
complete: scanned.length <= width
|
|
1870
|
+
});
|
|
1871
|
+
sourcesRefreshed += 1;
|
|
1872
|
+
pairsStored += neighbors.length;
|
|
1873
|
+
completed += 1;
|
|
1874
|
+
report?.({ phase: "similarity-cache", completed, total });
|
|
1875
|
+
}
|
|
1876
|
+
return { similarityMode: mode, width, minSimilarity: floor, sourcesRefreshed, pairsStored, skipped: false };
|
|
1877
|
+
}
|
|
1878
|
+
const functionsById = new Map(this.allFunctions().map((callable) => [callable.id, callable]));
|
|
1879
|
+
const lean = /* @__PURE__ */ new Map();
|
|
1880
|
+
for (const dirtyId of dirty) {
|
|
1881
|
+
throwIfAborted(options.signal);
|
|
1882
|
+
const full = this.similarToFunction(dirtyId, {
|
|
1883
|
+
...includeDescriptions ? { includeDescriptions: true } : {},
|
|
1884
|
+
limit: triples.length - 1,
|
|
1885
|
+
minSimilarity: floor
|
|
1886
|
+
});
|
|
1887
|
+
const byTarget = /* @__PURE__ */ new Map();
|
|
1888
|
+
for (const match of full) byTarget.set(match.function.id, match);
|
|
1889
|
+
lean.set(dirtyId, byTarget);
|
|
1890
|
+
const triple = tripleById.get(dirtyId);
|
|
1891
|
+
const top = full.slice(0, width);
|
|
1892
|
+
this.#database.storeSimilarityNeighbors(dirtyId, mode, top, {
|
|
1893
|
+
codeEmbeddingId: triple.codeEmbeddingId,
|
|
1894
|
+
descriptionEmbeddingId: triple.descriptionEmbeddingId,
|
|
1895
|
+
fileDescriptionEmbeddingId: triple.fileDescriptionEmbeddingId,
|
|
1896
|
+
cachedWidth: width,
|
|
1897
|
+
generation,
|
|
1898
|
+
floor,
|
|
1899
|
+
complete: full.length <= width
|
|
1900
|
+
});
|
|
1901
|
+
sourcesRefreshed += 1;
|
|
1902
|
+
pairsStored += top.length;
|
|
1903
|
+
completed += 1;
|
|
1904
|
+
report?.({ phase: "similarity-cache", completed, total });
|
|
1905
|
+
}
|
|
1906
|
+
for (const triple of triples) {
|
|
1907
|
+
throwIfAborted(options.signal);
|
|
1908
|
+
if (dirty.has(triple.functionId)) continue;
|
|
1909
|
+
const cached = this.#database.cachedSimilarityNeighbors(triple.functionId, mode);
|
|
1910
|
+
const kept = cached.filter((match) => match.similarity >= floor && !dirty.has(match.function.id) && functionsById.has(match.function.id));
|
|
1911
|
+
const added = [];
|
|
1912
|
+
for (const [dirtyId, byTarget] of lean) {
|
|
1913
|
+
if (dirtyId === triple.functionId) continue;
|
|
1914
|
+
const match = byTarget.get(triple.functionId);
|
|
1915
|
+
const dirtyFunction = functionsById.get(dirtyId);
|
|
1916
|
+
if (match && dirtyFunction) {
|
|
1917
|
+
const { function: _function, ...scores } = match;
|
|
1918
|
+
added.push({ ...scores, function: dirtyFunction });
|
|
1919
|
+
}
|
|
1920
|
+
}
|
|
1921
|
+
const plusOne = [...kept, ...added].sort((left, right) => right.similarity - left.similarity || left.function.id - right.function.id).slice(0, width + 1);
|
|
1922
|
+
const merged = plusOne.slice(0, width);
|
|
1923
|
+
const contributorsComplete = triples.every((candidate) => dirty.has(candidate.functionId) || states.get(candidate.functionId)?.complete === true);
|
|
1924
|
+
const complete = plusOne.length <= width && contributorsComplete;
|
|
1925
|
+
const unchanged = merged.length === cached.length && merged.every((match, index) => match.function.id === cached[index].function.id && match.similarity === cached[index].similarity);
|
|
1926
|
+
if (unchanged) {
|
|
1927
|
+
const narrowed = cached.some((match) => match.similarity < floor);
|
|
1928
|
+
this.#database.touchSimilarityCacheState(triple.functionId, mode, {
|
|
1929
|
+
codeEmbeddingId: triple.codeEmbeddingId,
|
|
1930
|
+
descriptionEmbeddingId: triple.descriptionEmbeddingId,
|
|
1931
|
+
fileDescriptionEmbeddingId: triple.fileDescriptionEmbeddingId,
|
|
1932
|
+
cachedWidth: width,
|
|
1933
|
+
generation,
|
|
1934
|
+
floor: narrowed ? floor : states.get(triple.functionId)?.floor ?? floor,
|
|
1935
|
+
storedCount: merged.length,
|
|
1936
|
+
complete
|
|
1937
|
+
});
|
|
1938
|
+
} else {
|
|
1939
|
+
this.#database.storeSimilarityNeighbors(triple.functionId, mode, merged, {
|
|
1940
|
+
codeEmbeddingId: triple.codeEmbeddingId,
|
|
1941
|
+
descriptionEmbeddingId: triple.descriptionEmbeddingId,
|
|
1942
|
+
fileDescriptionEmbeddingId: triple.fileDescriptionEmbeddingId,
|
|
1943
|
+
cachedWidth: width,
|
|
1944
|
+
generation,
|
|
1945
|
+
floor,
|
|
1946
|
+
complete
|
|
1947
|
+
});
|
|
1948
|
+
pairsStored += merged.length;
|
|
1949
|
+
}
|
|
1950
|
+
sourcesRefreshed += 1;
|
|
1951
|
+
completed += 1;
|
|
1952
|
+
report?.({ phase: "similarity-cache", completed, total });
|
|
1953
|
+
}
|
|
1954
|
+
return { similarityMode: mode, width, minSimilarity: floor, sourcesRefreshed, pairsStored, skipped: false };
|
|
1955
|
+
}
|
|
1956
|
+
/**
|
|
1957
|
+
* Same-index neighbor lookup backed by {@link refreshSimilarityCache}.
|
|
1958
|
+
* Returns cache rows filtered exactly like {@link similarToFunction} and falls
|
|
1959
|
+
* back to a live vector query whenever the cache entry is missing, stale,
|
|
1960
|
+
* computed at a higher floor than requested, or too truncated to satisfy the
|
|
1961
|
+
* filters. A cache entry flagged complete holds every pair above its floor,
|
|
1962
|
+
* so short results from it are exact and need no live query.
|
|
1963
|
+
*
|
|
1964
|
+
* Prefer {@link cachedSimilarityReader} inside per-function loops: it loads
|
|
1965
|
+
* the validity snapshot once instead of re-reading it for every function.
|
|
1966
|
+
*/
|
|
1967
|
+
cachedSimilarToFunction(functionId, options) {
|
|
1968
|
+
return this.cachedSimilarityReader({
|
|
1969
|
+
...options.includeDescriptions ? { includeDescriptions: true } : {}
|
|
1970
|
+
}).similarToFunction(functionId, options);
|
|
1971
|
+
}
|
|
1972
|
+
/**
|
|
1973
|
+
* Snapshot the similarity-cache validity state (per-mode states, embedding
|
|
1974
|
+
* triples, generation) once for a run of neighbor lookups. The returned
|
|
1975
|
+
* lookup behaves exactly like {@link cachedSimilarToFunction} but avoids
|
|
1976
|
+
* re-reading whole tables per function; refresh the cache first and create
|
|
1977
|
+
* one reader per analysis run. Best-effort under concurrent writers, like the
|
|
1978
|
+
* refresh itself: a concurrent mutation mid-run is picked up by the next run.
|
|
1979
|
+
*
|
|
1980
|
+
* A fallback scan doubles as a read-repair: when the entry is fresh but
|
|
1981
|
+
* unusable (missing, recorded at a higher floor, or incomplete), the scan is
|
|
1982
|
+
* widened to the full cache width and written back, so later lookups hit.
|
|
1983
|
+
* Stale entries are left for the refresh, dense-at-max-width entries cannot
|
|
1984
|
+
* be improved, and read-only indexes never write.
|
|
1985
|
+
*/
|
|
1986
|
+
cachedSimilarityReader(options = {}) {
|
|
1987
|
+
const mode = options.includeDescriptions ? "code-description-file-average" : "code";
|
|
1988
|
+
const states = this.#database.similarityCacheStates(mode);
|
|
1989
|
+
const triples = this.#database.similarityCacheTriples();
|
|
1990
|
+
const tripleById = new Map(triples.map((triple) => [triple.functionId, triple]));
|
|
1991
|
+
const generation = this.#database.getGeneration();
|
|
1992
|
+
const readFiltered = (functionId, query) => this.#database.cachedSimilarityNeighbors(functionId, mode, {
|
|
1993
|
+
limit: query.limit,
|
|
1994
|
+
minSimilarity: query.minSimilarity,
|
|
1995
|
+
...query.maxSimilarity !== void 0 ? { maxSimilarity: query.maxSimilarity } : {},
|
|
1996
|
+
...query.minLines !== void 0 ? { minLines: query.minLines } : {},
|
|
1997
|
+
...query.nameRegex !== void 0 ? { nameRegex: query.nameRegex } : {},
|
|
1998
|
+
...query.excludePaths !== void 0 ? { excludePaths: query.excludePaths } : {}
|
|
1999
|
+
});
|
|
2000
|
+
return {
|
|
2001
|
+
similarToFunction: (functionId, query) => {
|
|
2002
|
+
const live = () => this.similarToFunction(functionId, query);
|
|
2003
|
+
if (query.includeDescriptions === true !== (mode !== "code")) {
|
|
2004
|
+
return live();
|
|
2005
|
+
}
|
|
2006
|
+
compileNameRegex(query.nameRegex);
|
|
2007
|
+
const triple = tripleById.get(functionId);
|
|
2008
|
+
if (!triple) {
|
|
2009
|
+
return live();
|
|
2010
|
+
}
|
|
2011
|
+
const state = states.get(functionId);
|
|
2012
|
+
const tripleMatches = !!state && triple.codeEmbeddingId === state.codeEmbeddingId && triple.descriptionEmbeddingId === state.descriptionEmbeddingId && triple.fileDescriptionEmbeddingId === state.fileDescriptionEmbeddingId;
|
|
2013
|
+
if (state && (!tripleMatches || state.generation !== generation)) {
|
|
2014
|
+
return live();
|
|
2015
|
+
}
|
|
2016
|
+
if (state && state.floor <= query.minSimilarity) {
|
|
2017
|
+
const cached2 = readFiltered(functionId, query);
|
|
2018
|
+
if (cached2.length >= query.limit) return cached2;
|
|
2019
|
+
if (state.complete) return cached2;
|
|
2020
|
+
if (state.storedCount >= MAX_SIMILARITY_CACHE_WIDTH) return live();
|
|
2021
|
+
} else if (this.#database.isReadOnly || this.#database.getGeneration() !== generation) {
|
|
2022
|
+
return live();
|
|
2023
|
+
}
|
|
2024
|
+
const repairFloor = Math.min(query.minSimilarity, state?.floor ?? query.minSimilarity);
|
|
2025
|
+
const width = MAX_SIMILARITY_CACHE_WIDTH;
|
|
2026
|
+
const scanned = this.similarToFunction(functionId, {
|
|
2027
|
+
...mode !== "code" ? { includeDescriptions: true } : {},
|
|
2028
|
+
limit: Math.min(width + 1, tripleById.size - 1),
|
|
2029
|
+
minSimilarity: repairFloor
|
|
2030
|
+
});
|
|
2031
|
+
const neighbors = scanned.slice(0, width);
|
|
2032
|
+
const complete = scanned.length <= width;
|
|
2033
|
+
this.#database.storeSimilarityNeighbors(functionId, mode, neighbors, {
|
|
2034
|
+
codeEmbeddingId: triple.codeEmbeddingId,
|
|
2035
|
+
descriptionEmbeddingId: triple.descriptionEmbeddingId,
|
|
2036
|
+
fileDescriptionEmbeddingId: triple.fileDescriptionEmbeddingId,
|
|
2037
|
+
cachedWidth: width,
|
|
2038
|
+
generation,
|
|
2039
|
+
floor: repairFloor,
|
|
2040
|
+
complete
|
|
2041
|
+
});
|
|
2042
|
+
states.set(functionId, {
|
|
2043
|
+
codeEmbeddingId: triple.codeEmbeddingId,
|
|
2044
|
+
descriptionEmbeddingId: triple.descriptionEmbeddingId,
|
|
2045
|
+
fileDescriptionEmbeddingId: triple.fileDescriptionEmbeddingId,
|
|
2046
|
+
cachedWidth: width,
|
|
2047
|
+
generation,
|
|
2048
|
+
floor: repairFloor,
|
|
2049
|
+
storedCount: neighbors.length,
|
|
2050
|
+
complete
|
|
2051
|
+
});
|
|
2052
|
+
const cached = readFiltered(functionId, query);
|
|
2053
|
+
if (cached.length >= query.limit) return cached;
|
|
2054
|
+
if (complete) return cached;
|
|
2055
|
+
return live();
|
|
2056
|
+
}
|
|
2057
|
+
};
|
|
2058
|
+
}
|
|
2059
|
+
vectorForFile(filePath) {
|
|
2060
|
+
return this.#database.vectorForFile(filePath);
|
|
2061
|
+
}
|
|
2062
|
+
searchByVector(vector, options) {
|
|
2063
|
+
return this.#database.searchVector(normalizeEmbeddingVector(vector, this.provider.profile.dimensions), {
|
|
2064
|
+
...options,
|
|
2065
|
+
...options.descriptionVector !== void 0 ? { descriptionVector: normalizeEmbeddingVector(options.descriptionVector, this.provider.profile.dimensions) } : {},
|
|
2066
|
+
...options.fileDescriptionVector !== void 0 ? { fileDescriptionVector: normalizeEmbeddingVector(options.fileDescriptionVector, this.provider.profile.dimensions) } : {}
|
|
2067
|
+
});
|
|
2068
|
+
}
|
|
2069
|
+
#descriptionScoringAvailable() {
|
|
2070
|
+
const status = this.status();
|
|
2071
|
+
return status.descriptionsEnabled && status.descriptionCount === status.functionCount && status.fileDescriptionCount === status.describableFileCount;
|
|
2072
|
+
}
|
|
2073
|
+
async sourceFunctions(filter) {
|
|
2074
|
+
const nameRegex = compileNameRegex(filter.nameRegex);
|
|
2075
|
+
const sourcePath = filter.path ? normalizeRelativePath(this.rootDir, filter.path) : void 0;
|
|
2076
|
+
const uncommitted = filter.type === "uncommitted" || filter.type === "changed-since" && filter.uncommitted;
|
|
2077
|
+
const functions = filter.type === "changed-since" ? await this.#functionsChangedSince(filter.commit) : this.allFunctions();
|
|
2078
|
+
return functions.filter((callable) => (!uncommitted || callable.sourceMode === "working-tree") && (!sourcePath || callable.path === sourcePath || callable.path.startsWith(`${sourcePath}/`)) && (!nameRegex || nameRegex.test(callable.qualifiedName)));
|
|
2079
|
+
}
|
|
2080
|
+
async #functionsChangedSince(baseRef) {
|
|
2081
|
+
await this.#git.assertRepository();
|
|
2082
|
+
const checkpoint = this.#database.getCheckpoint();
|
|
2083
|
+
if (!checkpoint) throw new CodeIndexError("changed-since requires an index with a Git checkpoint.");
|
|
2084
|
+
const base = await this.#git.resolveCommit(baseRef);
|
|
2085
|
+
if (!await this.#git.isAncestor(base, checkpoint)) {
|
|
2086
|
+
throw new GitDivergenceError(base, checkpoint);
|
|
2087
|
+
}
|
|
2088
|
+
const changes = await this.#git.diff(base, checkpoint);
|
|
2089
|
+
const baseTree = await this.#git.listTree(base);
|
|
2090
|
+
const comparisonPaths = /* @__PURE__ */ new Map();
|
|
2091
|
+
for (const change of changes) {
|
|
2092
|
+
if (change.status === "D") continue;
|
|
2093
|
+
if (change.status === "R") comparisonPaths.set(change.path, { basePath: change.oldPath, pathChanged: true });
|
|
2094
|
+
else if (change.status === "A" || change.status === "C") comparisonPaths.set(change.path, { basePath: null, pathChanged: false });
|
|
2095
|
+
else comparisonPaths.set(change.path, { basePath: change.path, pathChanged: false });
|
|
2096
|
+
}
|
|
2097
|
+
for (const dirtyFile of this.#database.getWorkingTreeFiles()) {
|
|
2098
|
+
const basePath = dirtyFile.previousPath ?? comparisonPaths.get(dirtyFile.path)?.basePath ?? dirtyFile.path;
|
|
2099
|
+
comparisonPaths.set(dirtyFile.path, { basePath, pathChanged: basePath !== dirtyFile.path });
|
|
2100
|
+
}
|
|
2101
|
+
const current = this.#database.functionsForPaths([...comparisonPaths.keys()]);
|
|
2102
|
+
const baseFunctionsByCurrentPath = /* @__PURE__ */ new Map();
|
|
2103
|
+
for (const [currentPath, { basePath }] of comparisonPaths) {
|
|
2104
|
+
if (!basePath) {
|
|
2105
|
+
baseFunctionsByCurrentPath.set(currentPath, []);
|
|
2106
|
+
continue;
|
|
2107
|
+
}
|
|
2108
|
+
const entry = baseTree.get(basePath);
|
|
2109
|
+
if (!entry) {
|
|
2110
|
+
baseFunctionsByCurrentPath.set(currentPath, []);
|
|
2111
|
+
continue;
|
|
2112
|
+
}
|
|
2113
|
+
const content = (await this.#git.readBlob(entry.oid)).toString("utf8");
|
|
2114
|
+
baseFunctionsByCurrentPath.set(currentPath, parseCallables(basePath, content, this.#onWarning));
|
|
2115
|
+
}
|
|
2116
|
+
const currentByPath = groupBy(current, (callable) => callable.path);
|
|
2117
|
+
const changedIds = /* @__PURE__ */ new Set();
|
|
2118
|
+
for (const [currentPath, currentFunctions] of currentByPath) {
|
|
2119
|
+
const comparison = comparisonPaths.get(currentPath);
|
|
2120
|
+
for (const id of changedFunctionIds(
|
|
2121
|
+
currentFunctions,
|
|
2122
|
+
baseFunctionsByCurrentPath.get(currentPath) ?? [],
|
|
2123
|
+
comparison.pathChanged
|
|
2124
|
+
)) {
|
|
2125
|
+
changedIds.add(id);
|
|
2126
|
+
}
|
|
2127
|
+
}
|
|
2128
|
+
return current.filter((callable) => changedIds.has(callable.id));
|
|
2129
|
+
}
|
|
2130
|
+
};
|
|
2131
|
+
function changedFunctionIds(current, base, pathChanged) {
|
|
2132
|
+
if (pathChanged) return current.map((callable) => callable.id);
|
|
2133
|
+
const currentGroups = groupBy(current, (callable) => `${callable.qualifiedName}\0${callable.kind}`);
|
|
2134
|
+
const baseGroups = groupBy(base, (callable) => `${callable.qualifiedName}\0${callable.kind}`);
|
|
2135
|
+
const changed = [];
|
|
2136
|
+
for (const [key, currentGroup] of currentGroups) {
|
|
2137
|
+
const baseGroup = baseGroups.get(key) ?? [];
|
|
2138
|
+
const unmatchedBase = new Set(baseGroup.map((_, index) => index));
|
|
2139
|
+
const unmatchedCurrent = new Set(currentGroup.map((_, index) => index));
|
|
2140
|
+
for (let currentIndex = 0; currentIndex < currentGroup.length; currentIndex += 1) {
|
|
2141
|
+
const exactBaseIndex = [...unmatchedBase].find(
|
|
2142
|
+
(baseIndex) => baseGroup[baseIndex].sourceHash === currentGroup[currentIndex].sourceHash
|
|
2143
|
+
);
|
|
2144
|
+
if (exactBaseIndex !== void 0) {
|
|
2145
|
+
unmatchedBase.delete(exactBaseIndex);
|
|
2146
|
+
unmatchedCurrent.delete(currentIndex);
|
|
2147
|
+
}
|
|
2148
|
+
}
|
|
2149
|
+
for (const currentIndex of unmatchedCurrent) changed.push(currentGroup[currentIndex].id);
|
|
2150
|
+
}
|
|
2151
|
+
return changed;
|
|
2152
|
+
}
|
|
2153
|
+
function normalizeProfile(profile) {
|
|
2154
|
+
return {
|
|
2155
|
+
provider: profile.provider,
|
|
2156
|
+
model: profile.model,
|
|
2157
|
+
dimensions: profile.dimensions,
|
|
2158
|
+
strategyVersion: profile.strategyVersion ?? "callable-v2"
|
|
2159
|
+
};
|
|
2160
|
+
}
|
|
2161
|
+
function descriptionProfileJson(profile) {
|
|
2162
|
+
return JSON.stringify({
|
|
2163
|
+
provider: profile.provider,
|
|
2164
|
+
model: profile.model,
|
|
2165
|
+
strategyVersion: profile.strategyVersion
|
|
2166
|
+
});
|
|
2167
|
+
}
|
|
2168
|
+
function embeddingKey(profile, operation, input) {
|
|
2169
|
+
return sha256(`${profile}\0${operation}\0${input}`);
|
|
2170
|
+
}
|
|
2171
|
+
function prepareDescription(prepared, embeddingProfile, description, database) {
|
|
2172
|
+
const key = embeddingKey(embeddingProfile, "document", description);
|
|
2173
|
+
let value = prepared.get(key);
|
|
2174
|
+
if (!value) {
|
|
2175
|
+
value = { key, description };
|
|
2176
|
+
const vector = database.cachedEmbedding(key);
|
|
2177
|
+
if (vector) value.vector = vector;
|
|
2178
|
+
prepared.set(key, value);
|
|
2179
|
+
}
|
|
2180
|
+
return value;
|
|
2181
|
+
}
|
|
2182
|
+
function emptyStats(checkpoint) {
|
|
2183
|
+
return {
|
|
2184
|
+
filesUpdated: 0,
|
|
2185
|
+
filesDeleted: 0,
|
|
2186
|
+
functionsAdded: 0,
|
|
2187
|
+
functionsUpdated: 0,
|
|
2188
|
+
functionsDeleted: 0,
|
|
2189
|
+
embeddingsCreated: 0,
|
|
2190
|
+
checkpoint
|
|
2191
|
+
};
|
|
2192
|
+
}
|
|
2193
|
+
function validateRenames(renames) {
|
|
2194
|
+
const sources = new Set(renames.map((rename) => rename.from));
|
|
2195
|
+
const targets = new Set(renames.map((rename) => rename.to));
|
|
2196
|
+
if (sources.size !== renames.length || targets.size !== renames.length) {
|
|
2197
|
+
throw new CodeIndexError("Rename sources and targets must be unique.");
|
|
2198
|
+
}
|
|
2199
|
+
if ([...sources].some((source) => targets.has(source))) {
|
|
2200
|
+
throw new CodeIndexError("Chained or cyclic renames must be applied in separate updates.");
|
|
2201
|
+
}
|
|
2202
|
+
}
|
|
2203
|
+
function groupBy(values, keyFor) {
|
|
2204
|
+
const groups = /* @__PURE__ */ new Map();
|
|
2205
|
+
for (const value of values) {
|
|
2206
|
+
const key = keyFor(value);
|
|
2207
|
+
const group = groups.get(key) ?? [];
|
|
2208
|
+
group.push(value);
|
|
2209
|
+
groups.set(key, group);
|
|
2210
|
+
}
|
|
2211
|
+
return groups;
|
|
2212
|
+
}
|
|
2213
|
+
|
|
2214
|
+
export {
|
|
2215
|
+
CodeIndex
|
|
2216
|
+
};
|
|
2217
|
+
//# sourceMappingURL=chunk-N6D66ASM.js.map
|