@wrongstack/tools 0.309.1 → 0.310.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/_regex.d.ts +6 -34
- package/dist/bash.js +3 -3
- package/dist/builtin.js +3011 -1677
- package/dist/codebase-index/binary-frame.d.ts +57 -8
- package/dist/codebase-index/codebase-incoming-calls-tool.d.ts +6 -0
- package/dist/codebase-index/codebase-outgoing-calls-tool.d.ts +6 -0
- package/dist/codebase-index/codebase-search-tool.d.ts +15 -5
- package/dist/codebase-index/index-service.d.ts +3 -19
- package/dist/codebase-index/index.js +2735 -1415
- package/dist/codebase-index/indexer.d.ts +3 -0
- package/dist/codebase-index/parser-batch.d.ts +53 -0
- package/dist/codebase-index/parser-dispatch.d.ts +32 -0
- package/dist/codebase-index/parser-output.d.ts +14 -0
- package/dist/codebase-index/parser-worker-pool.d.ts +57 -4
- package/dist/codebase-index/parser-worker-script.d.ts +5 -2
- package/dist/codebase-index/parser-worker-script.js +4042 -0
- package/dist/codebase-index/project-server-cache.d.ts +16 -0
- package/dist/codebase-index/project-server-client.d.ts +2 -2
- package/dist/codebase-index/project-server-query-cache.d.ts +88 -0
- package/dist/codebase-index/project-server.js +2846 -1308
- package/dist/codebase-index/py-parser.d.ts +5 -0
- package/dist/codebase-index/schema.d.ts +14 -1
- package/dist/codebase-index/sqlite-runtime.d.ts +2 -2
- package/dist/codebase-index/tree-sitter/queries.d.ts +30 -3
- package/dist/codebase-index/tree-sitter/visitor.d.ts +2 -1
- package/dist/codebase-index/vector-search.d.ts +12 -0
- package/dist/codebase-index/wal-maintenance.d.ts +58 -0
- package/dist/codebase-index/worker-protocol/contracts.d.ts +44 -0
- package/dist/codebase-index/worker-protocol.d.ts +17 -1
- package/dist/codebase-index/worker.js +2300 -1020
- package/dist/codebase-index/writer-admin.d.ts +11 -0
- package/dist/codebase-index/writer-helpers.d.ts +31 -1
- package/dist/codebase-index/writer-mutations.d.ts +0 -6
- package/dist/codebase-index/writer-schema.d.ts +2 -2
- package/dist/codebase-index/writer.d.ts +15 -0
- package/dist/edit.js +2511 -1203
- package/dist/exec.js +5 -3
- package/dist/grep.js +5 -124
- package/dist/index.js +3007 -1736
- package/dist/json.js +5 -124
- package/dist/kanban.js +130 -0
- package/dist/logs.js +5 -121
- package/dist/pack.js +3011 -1677
- package/dist/patch.js +2524 -1216
- package/dist/plan.js +106 -0
- package/dist/read.js +2506 -1198
- package/dist/replace.js +2487 -1295
- package/dist/search.js +6 -2
- package/dist/session-kanban.js +24 -16
- package/dist/task.js +106 -0
- package/dist/todo.js +106 -0
- package/dist/tool-tier.d.ts +11 -0
- package/dist/tool-tier.js +3019 -1677
- package/dist/tree.js +14 -3
- package/dist/win32.js +3 -3
- package/dist/write.js +2513 -1205
- package/package.json +5 -4
|
@@ -170,238 +170,133 @@ var init_languages = __esm({
|
|
|
170
170
|
}
|
|
171
171
|
});
|
|
172
172
|
|
|
173
|
-
// src/codebase-index/
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
tsLoad ??= import("@typescript/typescript6").then((m) => {
|
|
181
|
-
ts = m.default ?? m;
|
|
182
|
-
return ts;
|
|
183
|
-
});
|
|
184
|
-
return tsLoad;
|
|
185
|
-
}
|
|
186
|
-
function kindMap() {
|
|
187
|
-
kindMapCache ??= {
|
|
188
|
-
[ts.SyntaxKind.ClassDeclaration]: "class",
|
|
189
|
-
[ts.SyntaxKind.InterfaceDeclaration]: "interface",
|
|
190
|
-
[ts.SyntaxKind.EnumDeclaration]: "enum",
|
|
191
|
-
[ts.SyntaxKind.TypeAliasDeclaration]: "type",
|
|
192
|
-
[ts.SyntaxKind.FunctionDeclaration]: "function",
|
|
193
|
-
[ts.SyntaxKind.MethodDeclaration]: "method",
|
|
194
|
-
[ts.SyntaxKind.GetAccessor]: "property",
|
|
195
|
-
[ts.SyntaxKind.SetAccessor]: "property",
|
|
196
|
-
[ts.SyntaxKind.PropertyDeclaration]: "property",
|
|
197
|
-
[ts.SyntaxKind.Parameter]: "parameter",
|
|
198
|
-
[ts.SyntaxKind.NamespaceExportDeclaration]: "namespace"
|
|
199
|
-
};
|
|
200
|
-
return kindMapCache;
|
|
201
|
-
}
|
|
202
|
-
function kindOf(node) {
|
|
203
|
-
if (ts.isVariableDeclaration(node)) {
|
|
204
|
-
const parent = node.parent;
|
|
205
|
-
if (ts.isVariableDeclarationList(parent)) {
|
|
206
|
-
const flags = parent.flags;
|
|
207
|
-
if (flags & ts.NodeFlags.Let) return "let";
|
|
208
|
-
if (flags & ts.NodeFlags.Const) return "const";
|
|
209
|
-
return "var";
|
|
210
|
-
}
|
|
211
|
-
}
|
|
212
|
-
if (ts.isModuleDeclaration(node)) return "namespace";
|
|
213
|
-
return kindMap()[node.kind] ?? null;
|
|
214
|
-
}
|
|
215
|
-
function getSignature(printer, node, sourceFile) {
|
|
216
|
-
const raw = printer.printNode(ts.EmitHint.Unspecified, node, sourceFile);
|
|
217
|
-
return raw.replace(/\s+/g, " ").slice(0, 500);
|
|
218
|
-
}
|
|
219
|
-
function getJsDoc(node, sourceFile) {
|
|
220
|
-
const fullText = sourceFile.getFullText();
|
|
221
|
-
const nodePos = node.getFullStart();
|
|
222
|
-
const comments = ts.getLeadingCommentRanges(fullText, nodePos);
|
|
223
|
-
if (!comments) return "";
|
|
224
|
-
for (const range of comments) {
|
|
225
|
-
const commentText = fullText.slice(range.pos, range.end);
|
|
226
|
-
const trimmed = commentText.trim();
|
|
227
|
-
if (trimmed.startsWith("/**") && trimmed.endsWith("*/")) {
|
|
228
|
-
const inner = trimmed.slice(3, -2).replace(/^[ \t]*\*[ ]?/gm, "").trim();
|
|
229
|
-
return inner.split("\n")[0]?.trim().slice(0, 200) ?? "";
|
|
230
|
-
}
|
|
173
|
+
// src/codebase-index/import-extractor.ts
|
|
174
|
+
function lastSegment(specifier) {
|
|
175
|
+
const pathLike = /[/\\]|::/.test(specifier);
|
|
176
|
+
const segments = specifier.split(/[/\\]|::/).filter(Boolean);
|
|
177
|
+
let last = segments[segments.length - 1] ?? specifier;
|
|
178
|
+
if (last === "*" || last === "_") {
|
|
179
|
+
last = segments[segments.length - 2] ?? specifier;
|
|
231
180
|
}
|
|
232
|
-
return "";
|
|
181
|
+
if (pathLike) return last.replace(/\.[A-Za-z0-9]{1,8}$/, "") || last;
|
|
182
|
+
const dotted = last.split(".").filter((part) => part && part !== "*" && part !== "_");
|
|
183
|
+
return dotted[dotted.length - 1] ?? last;
|
|
233
184
|
}
|
|
234
|
-
function
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
if (node.name && ts.isIdentifier(node.name)) {
|
|
239
|
-
parts.push(node.name.text);
|
|
240
|
-
}
|
|
185
|
+
function newlineOffsets(content) {
|
|
186
|
+
const offsets = [];
|
|
187
|
+
for (let i = 0; i < content.length; i++) {
|
|
188
|
+
if (content.charCodeAt(i) === 10) offsets.push(i);
|
|
241
189
|
}
|
|
190
|
+
return offsets;
|
|
242
191
|
}
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
return { file, lang, symbols: [], mtimeMs: Date.now() };
|
|
251
|
-
}
|
|
252
|
-
const symbols = [];
|
|
253
|
-
const refs = [];
|
|
254
|
-
const printer = ts.createPrinter({});
|
|
255
|
-
function visit(node, funcDepth, scopeParts) {
|
|
256
|
-
const kind = kindOf(node);
|
|
257
|
-
if (kind) {
|
|
258
|
-
if ((kind === "const" || kind === "let" || kind === "var" || kind === "parameter") && funcDepth > 0) {
|
|
259
|
-
} else {
|
|
260
|
-
const nameNode = node.name;
|
|
261
|
-
if (!nameNode || !ts.isIdentifier(nameNode)) {
|
|
262
|
-
return;
|
|
263
|
-
}
|
|
264
|
-
const name = nameNode.text;
|
|
265
|
-
const pos2 = nameNode.getStart(sourceFile);
|
|
266
|
-
const { line: line2, character } = sourceFile.getLineAndCharacterOfPosition(pos2);
|
|
267
|
-
const scope = scopeParts.join(".");
|
|
268
|
-
const signature = getSignature(printer, node, sourceFile);
|
|
269
|
-
const docComment = getJsDoc(node, sourceFile);
|
|
270
|
-
const text = [name, signature, docComment].filter(Boolean).join(" | ");
|
|
271
|
-
symbols.push({
|
|
272
|
-
id: 0,
|
|
273
|
-
lang,
|
|
274
|
-
kind,
|
|
275
|
-
name,
|
|
276
|
-
file,
|
|
277
|
-
line: line2 + 1,
|
|
278
|
-
col: character,
|
|
279
|
-
signature,
|
|
280
|
-
docComment,
|
|
281
|
-
scope,
|
|
282
|
-
text
|
|
283
|
-
});
|
|
284
|
-
}
|
|
285
|
-
}
|
|
286
|
-
const pos = node.getStart(sourceFile);
|
|
287
|
-
const { line } = sourceFile.getLineAndCharacterOfPosition(pos);
|
|
288
|
-
const lineNum = line + 1;
|
|
289
|
-
if (ts.isCallExpression(node)) {
|
|
290
|
-
const expr = node.expression;
|
|
291
|
-
if (ts.isIdentifier(expr)) {
|
|
292
|
-
refs.push({ fromId: 0, toName: expr.text, callType: "call", line: lineNum });
|
|
293
|
-
}
|
|
294
|
-
} else if (ts.isPropertyAccessExpression(node)) {
|
|
295
|
-
if (ts.isIdentifier(node.expression)) {
|
|
296
|
-
refs.push({ fromId: 0, toName: node.expression.text, callType: "call", line: lineNum });
|
|
297
|
-
}
|
|
298
|
-
} else if (ts.isTypeReferenceNode(node)) {
|
|
299
|
-
const name = getTypeName(node.typeName);
|
|
300
|
-
if (name) refs.push({ fromId: 0, toName: name, callType: "type_ref", line: lineNum });
|
|
301
|
-
} else if (ts.isHeritageClause(node)) {
|
|
302
|
-
for (const t of node.types) {
|
|
303
|
-
const name = getTypeName(t.expression);
|
|
304
|
-
if (name)
|
|
305
|
-
refs.push({
|
|
306
|
-
fromId: 0,
|
|
307
|
-
toName: name,
|
|
308
|
-
callType: node.token === ts.SyntaxKind.ExtendsKeyword ? "inherit" : "implement",
|
|
309
|
-
line: lineNum
|
|
310
|
-
});
|
|
311
|
-
}
|
|
312
|
-
} else if (ts.isImportDeclaration(node)) {
|
|
313
|
-
emitImportSpecifierRefs(node, refs, lineNum);
|
|
314
|
-
} else if (ts.isExportDeclaration(node) && node.moduleSpecifier) {
|
|
315
|
-
emitExportSpecifierRefs(node, refs, lineNum);
|
|
316
|
-
}
|
|
317
|
-
const scopeIdx = scopeParts.length;
|
|
318
|
-
pushScopeName(node, scopeParts);
|
|
319
|
-
const childFuncDepth = ts.isFunctionLike(node) ? funcDepth + 1 : funcDepth;
|
|
320
|
-
ts.forEachChild(node, (child) => visit(child, childFuncDepth, scopeParts));
|
|
321
|
-
scopeParts.length = scopeIdx;
|
|
192
|
+
function lineAt(offsets, index) {
|
|
193
|
+
let low = 0;
|
|
194
|
+
let high = offsets.length;
|
|
195
|
+
while (low < high) {
|
|
196
|
+
const mid = low + high >>> 1;
|
|
197
|
+
if ((offsets[mid] ?? 0) < index) low = mid + 1;
|
|
198
|
+
else high = mid;
|
|
322
199
|
}
|
|
323
|
-
|
|
324
|
-
return { file, lang, symbols, refs: deduplicateRefs(refs), mtimeMs: Date.now() };
|
|
200
|
+
return low + 1;
|
|
325
201
|
}
|
|
326
|
-
function
|
|
327
|
-
|
|
328
|
-
if (ts.isQualifiedName(name)) return `${getTypeName(name.left)}.${name.right.text}`;
|
|
329
|
-
return "";
|
|
202
|
+
function hasImportPatterns(lang) {
|
|
203
|
+
return LANG_IMPORTS[lang] !== void 0;
|
|
330
204
|
}
|
|
331
|
-
function
|
|
205
|
+
function extractImports(opts) {
|
|
206
|
+
const patterns = LANG_IMPORTS[opts.lang];
|
|
207
|
+
if (!patterns || !opts.content) return [];
|
|
208
|
+
const limit = opts.maxImports ?? IMPORT_MAX_PER_FILE;
|
|
209
|
+
const content = opts.content.length > IMPORT_MAX_FILE_CHARS ? opts.content.slice(0, IMPORT_MAX_FILE_CHARS) : opts.content;
|
|
210
|
+
const refs = [];
|
|
332
211
|
const seen = /* @__PURE__ */ new Set();
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
if (!clause) {
|
|
347
|
-
if (module) {
|
|
348
|
-
refs.push({ fromId: 0, toName: module, callType: "import", line: lineNum, module });
|
|
349
|
-
}
|
|
350
|
-
return;
|
|
351
|
-
}
|
|
352
|
-
if (clause.name) {
|
|
353
|
-
refs.push({ fromId: 0, toName: clause.name.text, callType: "import", line: lineNum, module });
|
|
354
|
-
}
|
|
355
|
-
const bindings = clause.namedBindings;
|
|
356
|
-
if (!bindings) return;
|
|
357
|
-
if (ts.isNamedImports(bindings)) {
|
|
358
|
-
for (const element of bindings.elements) {
|
|
212
|
+
const offsets = newlineOffsets(content);
|
|
213
|
+
for (const pattern of patterns) {
|
|
214
|
+
const re = new RegExp(pattern.re.source, pattern.re.flags);
|
|
215
|
+
for (const match of content.matchAll(re)) {
|
|
216
|
+
if (refs.length >= limit) return refs;
|
|
217
|
+
const specifier = match[1]?.trim();
|
|
218
|
+
if (!specifier) continue;
|
|
219
|
+
const module = specifier;
|
|
220
|
+
const toName = pattern.name === "full" ? module : lastSegment(module);
|
|
221
|
+
if (!toName) continue;
|
|
222
|
+
const key = `${module}\0${toName}`;
|
|
223
|
+
if (seen.has(key)) continue;
|
|
224
|
+
seen.add(key);
|
|
359
225
|
refs.push({
|
|
360
226
|
fromId: 0,
|
|
361
|
-
toName
|
|
227
|
+
toName,
|
|
362
228
|
callType: "import",
|
|
363
|
-
line:
|
|
229
|
+
line: lineAt(offsets, match.index ?? 0),
|
|
230
|
+
lang: opts.lang,
|
|
364
231
|
module
|
|
365
232
|
});
|
|
366
233
|
}
|
|
367
|
-
} else if (ts.isNamespaceImport(bindings)) {
|
|
368
|
-
refs.push({
|
|
369
|
-
fromId: 0,
|
|
370
|
-
toName: bindings.name.text,
|
|
371
|
-
callType: "import",
|
|
372
|
-
line: lineNum,
|
|
373
|
-
module
|
|
374
|
-
});
|
|
375
234
|
}
|
|
235
|
+
return refs;
|
|
376
236
|
}
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
function emitExportSpecifierRefs(node, refs, lineNum) {
|
|
381
|
-
const module = moduleSpecifierOf(node.moduleSpecifier);
|
|
382
|
-
const clause = node.exportClause;
|
|
383
|
-
if (clause && ts.isNamespaceExport(clause)) {
|
|
384
|
-
refs.push({ fromId: 0, toName: clause.name.text, callType: "import", line: lineNum, module });
|
|
385
|
-
return;
|
|
386
|
-
}
|
|
387
|
-
if (clause && ts.isNamedExports(clause)) {
|
|
388
|
-
for (const element of clause.elements) {
|
|
389
|
-
const originalName = element.propertyName?.text ?? element.name.text;
|
|
390
|
-
refs.push({ fromId: 0, toName: originalName, callType: "import", line: lineNum, module });
|
|
391
|
-
}
|
|
392
|
-
return;
|
|
393
|
-
}
|
|
394
|
-
if (module) {
|
|
395
|
-
refs.push({ fromId: 0, toName: module, callType: "import", line: lineNum, module });
|
|
396
|
-
}
|
|
397
|
-
}
|
|
398
|
-
var ts, tsLoad, kindMapCache;
|
|
399
|
-
var init_ts_parser = __esm({
|
|
400
|
-
"src/codebase-index/ts-parser.ts"() {
|
|
237
|
+
var IMPORT_MAX_FILE_CHARS, IMPORT_MAX_PER_FILE, DOTTED_IMPORT, LANG_IMPORTS;
|
|
238
|
+
var init_import_extractor = __esm({
|
|
239
|
+
"src/codebase-index/import-extractor.ts"() {
|
|
401
240
|
"use strict";
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
241
|
+
IMPORT_MAX_FILE_CHARS = 512 * 1024;
|
|
242
|
+
IMPORT_MAX_PER_FILE = 400;
|
|
243
|
+
DOTTED_IMPORT = [
|
|
244
|
+
{ re: /^[ \t]*import\s+(?:static\s+)?([\w.]+(?:\.\*)?)\s*;?\s*$/gm }
|
|
245
|
+
];
|
|
246
|
+
LANG_IMPORTS = {
|
|
247
|
+
// Go and Python have real AST extractors; these patterns are the fallback for
|
|
248
|
+
// machines with no Go toolchain or Python interpreter installed, where the
|
|
249
|
+
// parser degrades to regex symbols and would otherwise contribute no edges.
|
|
250
|
+
go: [
|
|
251
|
+
{ re: /^[ \t]*import\s+(?:[\w.]+\s+)?"([^"]+)"/gm },
|
|
252
|
+
// Grouped form: inside `import ( … )` each line is an optional alias plus a
|
|
253
|
+
// quoted path. A stray match elsewhere resolves to no file and is dropped.
|
|
254
|
+
{ re: /^[ \t]*(?:[A-Za-z_.]\w*\s+)?"([^"]+)"\s*$/gm }
|
|
255
|
+
],
|
|
256
|
+
py: [{ re: /^[ \t]*import\s+([\w.]+)/gm }, { re: /^[ \t]*from\s+([.\w]+)\s+import\b/gm }],
|
|
257
|
+
rs: [
|
|
258
|
+
// use a::b::C; | use a::b::{C, D}; → the path before any brace
|
|
259
|
+
{ re: /^[ \t]*(?:pub\s+)?use\s+([\w:]+?)(?:::\{|\s*;|\s+as\b)/gm },
|
|
260
|
+
// mod foo; (a declaration *and* a dependency on foo.rs / foo/mod.rs)
|
|
261
|
+
{ re: /^[ \t]*(?:pub\s+)?mod\s+(\w+)\s*;/gm }
|
|
262
|
+
],
|
|
263
|
+
java: DOTTED_IMPORT,
|
|
264
|
+
kotlin: DOTTED_IMPORT,
|
|
265
|
+
scala: [{ re: /^[ \t]*import\s+([\w.]+(?:\.[_*])?)\s*$/gm }],
|
|
266
|
+
csharp: [
|
|
267
|
+
// using Foo.Bar; | using static Foo.Bar; | using Alias = Foo.Bar;
|
|
268
|
+
{ re: /^[ \t]*global\s+using\s+(?:static\s+)?([\w.]+)\s*;/gm, name: "full" },
|
|
269
|
+
{ re: /^[ \t]*using\s+(?:static\s+)?(?:\w+\s*=\s*)?([\w.]+)\s*;/gm, name: "full" }
|
|
270
|
+
],
|
|
271
|
+
// Quoted includes only: <stdio.h> is a system header with no indexed file.
|
|
272
|
+
c: [{ re: /^[ \t]*#\s*include\s+"([^"]+)"/gm }],
|
|
273
|
+
cpp: [{ re: /^[ \t]*#\s*include\s+"([^"]+)"/gm }],
|
|
274
|
+
ruby: [{ re: /^[ \t]*(?:require|require_relative|load)\s*\(?\s*['"]([^'"]+)['"]/gm }],
|
|
275
|
+
php: [
|
|
276
|
+
// `use A\B\C` imports the class C, which is what the index has a symbol
|
|
277
|
+
// for — the namespace symbol only covers the `A\B` prefix.
|
|
278
|
+
{ re: /^[ \t]*use\s+(?:function\s+|const\s+)?([\w\\]+)/gm },
|
|
279
|
+
{ re: /^[ \t]*(?:require|require_once|include|include_once)\s*\(?\s*['"]([^'"]+)['"]/gm }
|
|
280
|
+
],
|
|
281
|
+
swift: [{ re: /^[ \t]*import\s+(?:struct\s+|class\s+|func\s+)?([\w.]+)\s*$/gm }],
|
|
282
|
+
dart: [{ re: /^[ \t]*(?:import|export|part)\s+['"]([^'"]+)['"]/gm }],
|
|
283
|
+
lua: [{ re: /\brequire\s*\(?\s*['"]([^'"]+)['"]/g }],
|
|
284
|
+
elixir: [{ re: /^[ \t]*(?:alias|import|require|use)\s+([A-Z][\w.]*)/gm, name: "full" }],
|
|
285
|
+
haskell: [{ re: /^[ \t]*import\s+(?:qualified\s+)?([\w.]+)/gm, name: "full" }],
|
|
286
|
+
zig: [{ re: /@import\s*\(\s*"([^"]+)"\s*\)/g }],
|
|
287
|
+
proto: [{ re: /^[ \t]*import\s+(?:public\s+|weak\s+)?"([^"]+)"\s*;/gm }],
|
|
288
|
+
// `@import "x"`, `@use "x"`, `@forward "x"` — Sass and plain CSS alike.
|
|
289
|
+
css: [{ re: /^[ \t]*@(?:import|use|forward)\s+(?:url\()?\s*['"]([^'"]+)['"]/gm }],
|
|
290
|
+
// A .vue/.svelte file's <script> block is JS; the same ESM syntax applies.
|
|
291
|
+
vue: [{ re: /^[ \t]*import\s[^'"]*from\s*['"]([^'"]+)['"]/gm }],
|
|
292
|
+
svelte: [{ re: /^[ \t]*import\s[^'"]*from\s*['"]([^'"]+)['"]/gm }],
|
|
293
|
+
html: [
|
|
294
|
+
{ re: /<script[^>]+src\s*=\s*['"]([^'"]+)['"]/g },
|
|
295
|
+
{ re: /<link[^>]+href\s*=\s*['"]([^'"]+)['"]/g }
|
|
296
|
+
],
|
|
297
|
+
shell: [{ re: /^[ \t]*(?:source|\.)\s+(\S+)/gm }],
|
|
298
|
+
r: [{ re: /\b(?:source|library|require)\s*\(\s*['"]?([\w./-]+)['"]?\s*\)/g }]
|
|
299
|
+
};
|
|
405
300
|
}
|
|
406
301
|
});
|
|
407
302
|
|
|
@@ -487,6 +382,32 @@ function parseParserOutput(stdout, lang) {
|
|
|
487
382
|
refs: dedupeRefs(coerceRefs(record.refs, lang))
|
|
488
383
|
};
|
|
489
384
|
}
|
|
385
|
+
function parseParserBatchOutput(stdout, lang) {
|
|
386
|
+
const trimmed = stdout.trim();
|
|
387
|
+
if (!trimmed) return [];
|
|
388
|
+
let parsed;
|
|
389
|
+
try {
|
|
390
|
+
parsed = JSON.parse(trimmed);
|
|
391
|
+
} catch {
|
|
392
|
+
return [];
|
|
393
|
+
}
|
|
394
|
+
if (!parsed || typeof parsed !== "object") return [];
|
|
395
|
+
const results = parsed.results;
|
|
396
|
+
if (!Array.isArray(results)) return [];
|
|
397
|
+
return results.flatMap((entry) => {
|
|
398
|
+
if (typeof entry !== "object" || entry === null || Array.isArray(entry)) return [];
|
|
399
|
+
const candidate = entry;
|
|
400
|
+
if (typeof candidate.file !== "string" || !candidate.file) return [];
|
|
401
|
+
return [
|
|
402
|
+
{
|
|
403
|
+
file: candidate.file,
|
|
404
|
+
error: typeof candidate.error === "string" && candidate.error ? candidate.error : void 0,
|
|
405
|
+
symbols: coerceSymbols(candidate.symbols),
|
|
406
|
+
refs: dedupeRefs(coerceRefs(candidate.refs, lang))
|
|
407
|
+
}
|
|
408
|
+
];
|
|
409
|
+
});
|
|
410
|
+
}
|
|
490
411
|
function dedupeRefs(refs) {
|
|
491
412
|
const seen = /* @__PURE__ */ new Set();
|
|
492
413
|
return refs.filter((ref) => {
|
|
@@ -527,187 +448,172 @@ var init_spawn_gate = __esm({
|
|
|
527
448
|
}
|
|
528
449
|
});
|
|
529
450
|
|
|
530
|
-
// src/codebase-index/
|
|
531
|
-
var go_parser_exports = {};
|
|
532
|
-
__export(go_parser_exports, {
|
|
533
|
-
detectLang: () => detectLang,
|
|
534
|
-
parseSymbols: () => parseSymbols2
|
|
535
|
-
});
|
|
451
|
+
// src/codebase-index/parser-batch.ts
|
|
536
452
|
import { spawn } from "node:child_process";
|
|
453
|
+
import * as fsSync from "node:fs";
|
|
454
|
+
import * as fs4 from "node:fs/promises";
|
|
537
455
|
import * as os from "node:os";
|
|
538
456
|
import * as path6 from "node:path";
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
457
|
+
function chunkBatchFiles(files) {
|
|
458
|
+
const chunks = [];
|
|
459
|
+
let current = [];
|
|
460
|
+
let bytes = 0;
|
|
461
|
+
for (const file of files) {
|
|
462
|
+
const size = Buffer.byteLength(file.content, "utf8");
|
|
463
|
+
if (current.length > 0 && (current.length >= MAX_BATCH_FILES || bytes + size > MAX_BATCH_BYTES)) {
|
|
464
|
+
chunks.push(current);
|
|
465
|
+
current = [];
|
|
466
|
+
bytes = 0;
|
|
467
|
+
}
|
|
468
|
+
current.push(file);
|
|
469
|
+
bytes += size;
|
|
470
|
+
}
|
|
471
|
+
if (current.length > 0) chunks.push(current);
|
|
472
|
+
return chunks;
|
|
473
|
+
}
|
|
474
|
+
function batchTimeoutMs(fileCount) {
|
|
475
|
+
return Math.min(12e4, 15e3 + fileCount * 1500);
|
|
476
|
+
}
|
|
477
|
+
async function ensureScriptPath(cached, prefix, fileName, script) {
|
|
478
|
+
if (cached) return { path: cached, wrote: false };
|
|
479
|
+
const dir = await fs4.mkdtemp(path6.join(os.tmpdir(), prefix));
|
|
480
|
+
const scriptPath = path6.join(dir, fileName);
|
|
481
|
+
await fs4.writeFile(scriptPath, script, { encoding: "utf8", flag: "wx" });
|
|
482
|
+
process.once("exit", () => {
|
|
483
|
+
try {
|
|
484
|
+
fsSync.rmSync(dir, { recursive: true, force: true });
|
|
485
|
+
} catch {
|
|
546
486
|
}
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
} catch {
|
|
550
|
-
return fallbackParse(file, content, lang);
|
|
551
|
-
}
|
|
487
|
+
});
|
|
488
|
+
return { path: scriptPath, wrote: true };
|
|
552
489
|
}
|
|
553
|
-
function
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
const fn = /^func\s+(?:\([^)]*\)\s*)?([A-Za-z_]\w*)\s*\(/.exec(trimmed);
|
|
564
|
-
if (fn?.[1]) {
|
|
565
|
-
addFallbackSymbol(symbols, {
|
|
566
|
-
filePath,
|
|
567
|
-
lang,
|
|
568
|
-
kind: trimmed.startsWith("func (") ? "method" : "function",
|
|
569
|
-
name: fn[1],
|
|
570
|
-
line: idx + 1,
|
|
571
|
-
col,
|
|
572
|
-
signature: trimmed,
|
|
573
|
-
scope: packageName ? `${packageName}.${fn[1]}` : fn[1]
|
|
574
|
-
});
|
|
575
|
-
continue;
|
|
576
|
-
}
|
|
577
|
-
const typeDecl = /^type\s+([A-Za-z_]\w*)\b/.exec(trimmed);
|
|
578
|
-
if (typeDecl?.[1]) {
|
|
579
|
-
addFallbackSymbol(symbols, {
|
|
580
|
-
filePath,
|
|
581
|
-
lang,
|
|
582
|
-
kind: "type",
|
|
583
|
-
name: typeDecl[1],
|
|
584
|
-
line: idx + 1,
|
|
585
|
-
col,
|
|
586
|
-
signature: trimmed,
|
|
587
|
-
scope: packageName
|
|
588
|
-
});
|
|
589
|
-
continue;
|
|
590
|
-
}
|
|
591
|
-
const valueDecl = /^(const|var)\s+([A-Za-z_]\w*)\b/.exec(trimmed);
|
|
592
|
-
if (valueDecl?.[1] && valueDecl[2]) {
|
|
593
|
-
addFallbackSymbol(symbols, {
|
|
594
|
-
filePath,
|
|
595
|
-
lang,
|
|
596
|
-
kind: valueDecl[1],
|
|
597
|
-
name: valueDecl[2],
|
|
598
|
-
line: idx + 1,
|
|
599
|
-
col,
|
|
600
|
-
signature: trimmed,
|
|
601
|
-
scope: packageName
|
|
602
|
-
});
|
|
490
|
+
function runToolchainChild(binary, args, stdinPayload, timeoutMs) {
|
|
491
|
+
return new Promise((resolve2) => {
|
|
492
|
+
let settled = false;
|
|
493
|
+
let stdout = "";
|
|
494
|
+
let proc;
|
|
495
|
+
try {
|
|
496
|
+
proc = spawn(binary, args, { stdio: ["pipe", "pipe", "pipe"], windowsHide: true });
|
|
497
|
+
} catch {
|
|
498
|
+
resolve2(null);
|
|
499
|
+
return;
|
|
603
500
|
}
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
501
|
+
const finish = (value) => {
|
|
502
|
+
if (settled) return;
|
|
503
|
+
settled = true;
|
|
504
|
+
clearTimeout(timer);
|
|
505
|
+
resolve2(value);
|
|
506
|
+
};
|
|
507
|
+
const timer = setTimeout(() => {
|
|
508
|
+
proc.kill("SIGKILL");
|
|
509
|
+
finish(null);
|
|
510
|
+
}, timeoutMs);
|
|
511
|
+
timer.unref?.();
|
|
512
|
+
proc.on("error", () => finish(null));
|
|
513
|
+
proc.stdout?.on("data", (chunk) => {
|
|
514
|
+
stdout += chunk.toString();
|
|
515
|
+
});
|
|
516
|
+
proc.stderr?.resume();
|
|
517
|
+
proc.stdin?.on("error", () => {
|
|
518
|
+
});
|
|
519
|
+
proc.stdin?.write(stdinPayload);
|
|
520
|
+
proc.stdin?.end();
|
|
521
|
+
proc.on("close", (code) => finish({ code, stdout }));
|
|
620
522
|
});
|
|
621
523
|
}
|
|
622
|
-
function
|
|
623
|
-
const
|
|
624
|
-
|
|
625
|
-
const
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
524
|
+
async function runGoBatch(files, goBinary) {
|
|
525
|
+
const out = /* @__PURE__ */ new Map();
|
|
526
|
+
if (files.length === 0) return out;
|
|
527
|
+
const { path: scriptPath } = await ensureScriptPath(
|
|
528
|
+
_goBatchScriptPath,
|
|
529
|
+
"ws-go-parse",
|
|
530
|
+
"batch.go",
|
|
531
|
+
GO_BATCH_SCRIPT
|
|
532
|
+
);
|
|
533
|
+
_goBatchScriptPath = scriptPath;
|
|
534
|
+
const payload = JSON.stringify(files.map((f) => ({ file: f.file, content: f.content })));
|
|
535
|
+
const result = await withSpawnGate(
|
|
536
|
+
() => runToolchainChild(
|
|
537
|
+
goBinary ?? resolveWin32Command("go"),
|
|
538
|
+
["run", scriptPath],
|
|
539
|
+
payload,
|
|
540
|
+
batchTimeoutMs(files.length)
|
|
541
|
+
)
|
|
542
|
+
);
|
|
543
|
+
if (result?.code !== 0 || !result.stdout.trim()) return out;
|
|
544
|
+
for (const entry of parseParserBatchOutput(result.stdout, "go")) {
|
|
545
|
+
if (entry.error !== void 0) continue;
|
|
546
|
+
out.set(entry.file, {
|
|
547
|
+
file: entry.file,
|
|
548
|
+
lang: "go",
|
|
549
|
+
symbols: entry.symbols.map((s) => ({
|
|
550
|
+
id: 0,
|
|
551
|
+
lang: "go",
|
|
552
|
+
kind: s.kind,
|
|
553
|
+
name: s.name,
|
|
554
|
+
file: entry.file,
|
|
555
|
+
line: s.line,
|
|
556
|
+
col: s.col,
|
|
557
|
+
signature: s.signature ?? "",
|
|
558
|
+
docComment: "",
|
|
559
|
+
scope: s.scope ?? "",
|
|
560
|
+
text: `${s.name} ${s.signature ?? ""}`.trim()
|
|
561
|
+
})),
|
|
562
|
+
refs: entry.refs,
|
|
563
|
+
mtimeMs: Date.now()
|
|
564
|
+
});
|
|
632
565
|
}
|
|
633
|
-
return
|
|
566
|
+
return out;
|
|
634
567
|
}
|
|
635
|
-
async function
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
proc.on("close", (code2) => {
|
|
672
|
-
if (settled) return;
|
|
673
|
-
settled = true;
|
|
674
|
-
clearTimeout(timer);
|
|
675
|
-
resolve2({ code: code2, stdout: stdout2 });
|
|
676
|
-
});
|
|
677
|
-
}
|
|
678
|
-
);
|
|
679
|
-
const { code, stdout } = goResult;
|
|
680
|
-
if (code !== 0 || !stdout.trim()) {
|
|
681
|
-
return { file: filePath, lang, symbols: [], mtimeMs: Date.now() };
|
|
682
|
-
}
|
|
683
|
-
const { symbols: rawSymbols, refs } = parseParserOutput(stdout, lang);
|
|
684
|
-
const symbols = rawSymbols.map((s) => ({
|
|
685
|
-
id: 0,
|
|
686
|
-
lang,
|
|
687
|
-
kind: s.kind,
|
|
688
|
-
name: s.name,
|
|
689
|
-
file: filePath,
|
|
690
|
-
line: s.line,
|
|
691
|
-
col: s.col,
|
|
692
|
-
signature: s.signature ?? "",
|
|
693
|
-
docComment: "",
|
|
694
|
-
scope: s.scope ?? "",
|
|
695
|
-
text: `${s.name} ${s.signature ?? ""}`.trim()
|
|
696
|
-
}));
|
|
697
|
-
return { file: filePath, lang, symbols, refs, mtimeMs: Date.now() };
|
|
698
|
-
} catch {
|
|
699
|
-
return { file: filePath, lang, symbols: [], mtimeMs: Date.now() };
|
|
568
|
+
async function runPyBatch(files, pythonBinary) {
|
|
569
|
+
const out = /* @__PURE__ */ new Map();
|
|
570
|
+
if (files.length === 0) return out;
|
|
571
|
+
const { path: scriptPath } = await ensureScriptPath(
|
|
572
|
+
_pyBatchScriptPath,
|
|
573
|
+
"ws-py-parse",
|
|
574
|
+
"batch.py",
|
|
575
|
+
PY_BATCH_SCRIPT
|
|
576
|
+
);
|
|
577
|
+
_pyBatchScriptPath = scriptPath;
|
|
578
|
+
const payload = JSON.stringify(files.map((f) => ({ file: f.file, content: f.content })));
|
|
579
|
+
const result = await withSpawnGate(
|
|
580
|
+
() => runToolchainChild(pythonBinary, [scriptPath], payload, batchTimeoutMs(files.length))
|
|
581
|
+
);
|
|
582
|
+
if (result?.code !== 0 || !result.stdout.trim()) return out;
|
|
583
|
+
for (const entry of parseParserBatchOutput(result.stdout, "py")) {
|
|
584
|
+
if (entry.error !== void 0) continue;
|
|
585
|
+
out.set(entry.file, {
|
|
586
|
+
file: entry.file,
|
|
587
|
+
lang: "py",
|
|
588
|
+
symbols: entry.symbols.map((s) => ({
|
|
589
|
+
id: 0,
|
|
590
|
+
lang: "py",
|
|
591
|
+
kind: s.kind,
|
|
592
|
+
name: s.name,
|
|
593
|
+
file: entry.file,
|
|
594
|
+
line: s.line,
|
|
595
|
+
col: s.col,
|
|
596
|
+
signature: s.signature ?? "",
|
|
597
|
+
docComment: "",
|
|
598
|
+
scope: s.scope ?? "",
|
|
599
|
+
text: `${s.name} ${s.signature ?? ""}`.trim()
|
|
600
|
+
})),
|
|
601
|
+
refs: entry.refs,
|
|
602
|
+
mtimeMs: Date.now()
|
|
603
|
+
});
|
|
700
604
|
}
|
|
605
|
+
return out;
|
|
701
606
|
}
|
|
702
|
-
var
|
|
703
|
-
var
|
|
704
|
-
"src/codebase-index/
|
|
607
|
+
var MAX_BATCH_FILES, MAX_BATCH_BYTES, GO_BATCH_SCRIPT, PY_BATCH_SCRIPT, _goBatchScriptPath, _pyBatchScriptPath;
|
|
608
|
+
var init_parser_batch = __esm({
|
|
609
|
+
"src/codebase-index/parser-batch.ts"() {
|
|
705
610
|
"use strict";
|
|
706
611
|
init_win32_resolve();
|
|
707
612
|
init_parser_output();
|
|
708
613
|
init_spawn_gate();
|
|
709
|
-
|
|
710
|
-
|
|
614
|
+
MAX_BATCH_FILES = 100;
|
|
615
|
+
MAX_BATCH_BYTES = 8 * 1024 * 1024;
|
|
616
|
+
GO_BATCH_SCRIPT = `
|
|
711
617
|
package main
|
|
712
618
|
|
|
713
619
|
import (
|
|
@@ -731,8 +637,6 @@ type Sym struct {
|
|
|
731
637
|
Scope string \`json:"scope"\`
|
|
732
638
|
}
|
|
733
639
|
|
|
734
|
-
// Ref is a cross-reference emitted alongside the symbols, so one \`go run\`
|
|
735
|
-
// yields both. Module is the import path for CallType "import", else empty.
|
|
736
640
|
type Ref struct {
|
|
737
641
|
ToName string \`json:"toName"\`
|
|
738
642
|
CallType string \`json:"callType"\`
|
|
@@ -740,57 +644,53 @@ type Ref struct {
|
|
|
740
644
|
Module string \`json:"module"\`
|
|
741
645
|
}
|
|
742
646
|
|
|
743
|
-
type
|
|
744
|
-
|
|
745
|
-
|
|
647
|
+
type FileResult struct {
|
|
648
|
+
File string \`json:"file"\`
|
|
649
|
+
Error string \`json:"error,omitempty"\`
|
|
650
|
+
Symbols []Sym \`json:"symbols"\`
|
|
651
|
+
Refs []Ref \`json:"refs"\`
|
|
746
652
|
}
|
|
747
653
|
|
|
748
|
-
|
|
749
|
-
|
|
654
|
+
type BatchResult struct {
|
|
655
|
+
Version int \`json:"version"\`
|
|
656
|
+
Results []FileResult \`json:"results"\`
|
|
750
657
|
}
|
|
751
658
|
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
659
|
+
type inputFile struct {
|
|
660
|
+
File string \`json:"file"\`
|
|
661
|
+
Content string \`json:"content"\`
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
func parseOne(name string, src []byte) FileResult {
|
|
665
|
+
res := FileResult{File: name, Symbols: []Sym{}, Refs: []Ref{}}
|
|
758
666
|
fset := token.NewFileSet()
|
|
759
667
|
node, err := parser.ParseFile(fset, "src.go", src, 0)
|
|
760
668
|
if err != nil {
|
|
761
|
-
|
|
762
|
-
return
|
|
669
|
+
res.Error = err.Error()
|
|
670
|
+
return res
|
|
763
671
|
}
|
|
764
|
-
|
|
765
|
-
var syms []Sym
|
|
766
|
-
|
|
767
|
-
// Package-level scope
|
|
768
672
|
pkgScope := node.Name.Name
|
|
769
|
-
|
|
770
|
-
// Collect all top-level declarations
|
|
771
673
|
for _, decl := range node.Decls {
|
|
772
674
|
switch d := decl.(type) {
|
|
773
675
|
case *ast.FuncDecl:
|
|
774
|
-
|
|
676
|
+
symName := d.Name.Name
|
|
775
677
|
kind := "function"
|
|
776
678
|
scope := pkgScope
|
|
777
679
|
if d.Recv != nil && len(d.Recv.List) > 0 {
|
|
778
|
-
scope = pkgScope + "." + recvTypeName(d.Recv.List[0].Type) + "." +
|
|
680
|
+
scope = pkgScope + "." + recvTypeName(d.Recv.List[0].Type) + "." + symName
|
|
779
681
|
kind = "method"
|
|
780
682
|
} else {
|
|
781
|
-
scope = pkgScope + "." +
|
|
683
|
+
scope = pkgScope + "." + symName
|
|
782
684
|
}
|
|
783
685
|
pos := fset.Position(d.Pos())
|
|
784
|
-
|
|
785
|
-
syms = append(syms, Sym{Name: name, Kind: kind, Line: pos.Line, Col: pos.Column, Signature: sig, Scope: scope})
|
|
786
|
-
|
|
686
|
+
res.Symbols = append(res.Symbols, Sym{Name: symName, Kind: kind, Line: pos.Line, Col: pos.Column, Signature: formatFuncSig(d), Scope: scope})
|
|
787
687
|
case *ast.GenDecl:
|
|
788
688
|
for _, spec := range d.Specs {
|
|
789
689
|
switch s := spec.(type) {
|
|
790
690
|
case *ast.TypeSpec:
|
|
791
|
-
|
|
691
|
+
typeName := s.Name.Name
|
|
792
692
|
pos := fset.Position(s.Pos())
|
|
793
|
-
sig := "type " +
|
|
693
|
+
sig := "type " + typeName
|
|
794
694
|
if s.TypeParams != nil {
|
|
795
695
|
sig += formatTypeParams(s.TypeParams)
|
|
796
696
|
}
|
|
@@ -801,64 +701,72 @@ func main() {
|
|
|
801
701
|
} else {
|
|
802
702
|
sig += " = " + formatType(s.Type)
|
|
803
703
|
}
|
|
804
|
-
|
|
805
|
-
|
|
704
|
+
res.Symbols = append(res.Symbols, Sym{Name: typeName, Kind: "type", Line: pos.Line, Col: pos.Column, Signature: sig, Scope: pkgScope})
|
|
806
705
|
case *ast.ValueSpec:
|
|
807
706
|
for _, n := range s.Names {
|
|
808
|
-
|
|
707
|
+
valueName := n.Name
|
|
809
708
|
pos := fset.Position(n.Pos())
|
|
810
709
|
kind := "var"
|
|
811
710
|
if d.Tok == token.CONST {
|
|
812
711
|
kind = "const"
|
|
813
712
|
}
|
|
814
|
-
sig := kind + " " +
|
|
713
|
+
sig := kind + " " + valueName
|
|
815
714
|
if s.Type != nil {
|
|
816
715
|
sig += " " + formatType(s.Type)
|
|
817
716
|
}
|
|
818
|
-
|
|
717
|
+
res.Symbols = append(res.Symbols, Sym{Name: valueName, Kind: kind, Line: pos.Line, Col: pos.Column, Signature: sig, Scope: pkgScope})
|
|
819
718
|
}
|
|
820
719
|
}
|
|
821
720
|
}
|
|
822
721
|
}
|
|
823
722
|
}
|
|
824
|
-
|
|
825
|
-
refs := []Ref{}
|
|
826
723
|
ast.Inspect(node, func(n ast.Node) bool {
|
|
827
724
|
switch expr := n.(type) {
|
|
828
725
|
case *ast.CallExpr:
|
|
829
726
|
line := fset.Position(expr.Pos()).Line
|
|
830
727
|
switch fun := expr.Fun.(type) {
|
|
831
728
|
case *ast.Ident:
|
|
832
|
-
|
|
729
|
+
res.Refs = append(res.Refs, Ref{ToName: fun.Name, CallType: "call", Line: line})
|
|
833
730
|
case *ast.SelectorExpr:
|
|
834
|
-
|
|
835
|
-
// declared symbol name, so it resolves the same way the TypeScript
|
|
836
|
-
// and Python extractors' call refs do.
|
|
837
|
-
refs = append(refs, Ref{ToName: fun.Sel.Name, CallType: "call", Line: line})
|
|
731
|
+
res.Refs = append(res.Refs, Ref{ToName: fun.Sel.Name, CallType: "call", Line: line})
|
|
838
732
|
}
|
|
839
733
|
case *ast.ImportSpec:
|
|
840
734
|
if expr.Path != nil {
|
|
841
735
|
if importPath, uerr := strconv.Unquote(expr.Path.Value); uerr == nil {
|
|
842
736
|
line := fset.Position(expr.Pos()).Line
|
|
843
|
-
// A Go import names a package, not a symbol; the package's
|
|
844
|
-
// last path segment is the name it is referenced by.
|
|
845
737
|
name := importPath
|
|
846
738
|
if idx := strings.LastIndex(importPath, "/"); idx >= 0 {
|
|
847
739
|
name = importPath[idx+1:]
|
|
848
740
|
}
|
|
849
|
-
|
|
741
|
+
res.Refs = append(res.Refs, Ref{ToName: name, CallType: "import", Line: line, Module: importPath})
|
|
850
742
|
}
|
|
851
743
|
}
|
|
852
744
|
}
|
|
853
745
|
return true
|
|
854
746
|
})
|
|
747
|
+
return res
|
|
748
|
+
}
|
|
855
749
|
|
|
856
|
-
|
|
857
|
-
|
|
750
|
+
func main() {
|
|
751
|
+
raw, err := io.ReadAll(os.Stdin)
|
|
752
|
+
if err != nil {
|
|
753
|
+
out, _ := json.Marshal(BatchResult{Version: 1, Results: []FileResult{}})
|
|
754
|
+
fmt.Print(string(out))
|
|
755
|
+
return
|
|
858
756
|
}
|
|
859
|
-
|
|
757
|
+
var inputs []inputFile
|
|
758
|
+
if err := json.Unmarshal(raw, &inputs); err != nil {
|
|
759
|
+
out, _ := json.Marshal(BatchResult{Version: 1, Results: []FileResult{}})
|
|
760
|
+
fmt.Print(string(out))
|
|
761
|
+
return
|
|
762
|
+
}
|
|
763
|
+
results := make([]FileResult, 0, len(inputs))
|
|
764
|
+
for _, in := range inputs {
|
|
765
|
+
results = append(results, parseOne(in.File, []byte(in.Content)))
|
|
766
|
+
}
|
|
767
|
+
data, err := json.Marshal(BatchResult{Version: 1, Results: results})
|
|
860
768
|
if err != nil {
|
|
861
|
-
fmt.Print(
|
|
769
|
+
fmt.Print("{\\"version\\":1,\\"results\\":[]}")
|
|
862
770
|
return
|
|
863
771
|
}
|
|
864
772
|
fmt.Print(string(data))
|
|
@@ -978,10 +886,8 @@ func formatType(t ast.Expr) string {
|
|
|
978
886
|
case *ast.BasicLit:
|
|
979
887
|
return v.Value
|
|
980
888
|
case *ast.IndexExpr:
|
|
981
|
-
// Generic instantiation with one type arg, e.g. Logger[int].
|
|
982
889
|
return formatType(v.X) + "[" + formatType(v.Index) + "]"
|
|
983
890
|
case *ast.IndexListExpr:
|
|
984
|
-
// Generic instantiation with multiple type args, e.g. Map[K, V].
|
|
985
891
|
args := make([]string, len(v.Indices))
|
|
986
892
|
for i, idx := range v.Indices {
|
|
987
893
|
args[i] = formatType(idx)
|
|
@@ -992,7 +898,187 @@ func formatType(t ast.Expr) string {
|
|
|
992
898
|
}
|
|
993
899
|
}
|
|
994
900
|
`;
|
|
995
|
-
|
|
901
|
+
PY_BATCH_SCRIPT = `import ast, json, sys
|
|
902
|
+
|
|
903
|
+
def get_name(node):
|
|
904
|
+
if isinstance(node, ast.Name):
|
|
905
|
+
return node.id
|
|
906
|
+
elif isinstance(node, ast.Attribute):
|
|
907
|
+
return get_name(node.value) + "." + node.attr
|
|
908
|
+
elif isinstance(node, ast.Subscript):
|
|
909
|
+
return get_name(node.value)
|
|
910
|
+
elif isinstance(node, ast.Call):
|
|
911
|
+
return get_name(node.func)
|
|
912
|
+
elif isinstance(node, ast.Constant):
|
|
913
|
+
return str(node.value)
|
|
914
|
+
return ""
|
|
915
|
+
|
|
916
|
+
def leaf_name(node):
|
|
917
|
+
if isinstance(node, ast.Attribute):
|
|
918
|
+
return node.attr
|
|
919
|
+
if isinstance(node, ast.Name):
|
|
920
|
+
return node.id
|
|
921
|
+
return get_name(node).split(".")[-1]
|
|
922
|
+
|
|
923
|
+
def is_private(name):
|
|
924
|
+
return name.startswith("__") and not name.endswith("__")
|
|
925
|
+
|
|
926
|
+
def parse_one(name, source, module_name):
|
|
927
|
+
result = {"file": name, "symbols": [], "refs": []}
|
|
928
|
+
try:
|
|
929
|
+
tree = ast.parse(source, filename=name)
|
|
930
|
+
except Exception as e:
|
|
931
|
+
result["error"] = str(e)
|
|
932
|
+
return result
|
|
933
|
+
syms = []
|
|
934
|
+
refs = []
|
|
935
|
+
scope_stack = [module_name]
|
|
936
|
+
|
|
937
|
+
def sym(d):
|
|
938
|
+
return {
|
|
939
|
+
"name": d["name"], "kind": d["kind"], "line": d["line"], "col": d["col"],
|
|
940
|
+
"signature": d["signature"], "scope": d["scope"],
|
|
941
|
+
}
|
|
942
|
+
|
|
943
|
+
class Visitor(ast.NodeVisitor):
|
|
944
|
+
def visit_ClassDef(self, node):
|
|
945
|
+
bases = [get_name(b) for b in node.bases]
|
|
946
|
+
sig = "class " + node.name
|
|
947
|
+
if bases:
|
|
948
|
+
sig += "(" + ", ".join(bases) + ")"
|
|
949
|
+
sig += ": ..."
|
|
950
|
+
syms.append(sym({
|
|
951
|
+
"name": node.name, "kind": "class", "line": node.lineno,
|
|
952
|
+
"col": node.col_offset, "signature": sig,
|
|
953
|
+
"scope": ".".join(scope_stack) + "." + node.name,
|
|
954
|
+
}))
|
|
955
|
+
scope_stack.append(node.name)
|
|
956
|
+
self.generic_visit(node)
|
|
957
|
+
scope_stack.pop()
|
|
958
|
+
|
|
959
|
+
def visit_FunctionDef(self, node):
|
|
960
|
+
args = ", ".join(a.arg for a in node.args.args)
|
|
961
|
+
returns = get_name(node.returns) if node.returns is not None else ""
|
|
962
|
+
is_async = isinstance(node, ast.AsyncFunctionDef)
|
|
963
|
+
kind = "function"
|
|
964
|
+
prefix = "def "
|
|
965
|
+
for dec in node.decorator_list:
|
|
966
|
+
d = get_name(dec)
|
|
967
|
+
if d.endswith(".staticmethod"):
|
|
968
|
+
kind = "staticmethod"
|
|
969
|
+
elif d.endswith(".classmethod"):
|
|
970
|
+
kind = "classmethod"
|
|
971
|
+
elif d == "property":
|
|
972
|
+
kind = "property"
|
|
973
|
+
if is_async:
|
|
974
|
+
kind = "async_" + kind
|
|
975
|
+
sig = f"{prefix}{node.name}({args})"
|
|
976
|
+
if returns:
|
|
977
|
+
sig += f" -> {returns}"
|
|
978
|
+
syms.append(sym({
|
|
979
|
+
"name": node.name, "kind": kind, "line": node.lineno,
|
|
980
|
+
"col": node.col_offset, "signature": sig,
|
|
981
|
+
"scope": ".".join(scope_stack) + "." + node.name,
|
|
982
|
+
}))
|
|
983
|
+
|
|
984
|
+
def visit_AsyncFunctionDef(self, node):
|
|
985
|
+
self.visit_FunctionDef(node)
|
|
986
|
+
|
|
987
|
+
def visit_Assign(self, node):
|
|
988
|
+
for target in node.targets:
|
|
989
|
+
if isinstance(target, ast.Name):
|
|
990
|
+
tname = target.id
|
|
991
|
+
if is_private(tname):
|
|
992
|
+
continue
|
|
993
|
+
kind = "const" if tname.isupper() else "var"
|
|
994
|
+
col = target.col_offset if hasattr(target, "col_offset") else 0
|
|
995
|
+
syms.append(sym({
|
|
996
|
+
"name": tname, "kind": kind, "line": node.lineno, "col": col,
|
|
997
|
+
"signature": f"{tname} = ...", "scope": ".".join(scope_stack),
|
|
998
|
+
}))
|
|
999
|
+
|
|
1000
|
+
def visit_AnnAssign(self, node):
|
|
1001
|
+
if isinstance(node.target, ast.Name):
|
|
1002
|
+
tname = node.target.id
|
|
1003
|
+
if is_private(tname):
|
|
1004
|
+
return
|
|
1005
|
+
kind = "const" if tname.isupper() else "var"
|
|
1006
|
+
col = node.target.col_offset if hasattr(node.target, "col_offset") else 0
|
|
1007
|
+
sig = f"{tname}: {get_name(node.annotation)}"
|
|
1008
|
+
if node.value:
|
|
1009
|
+
sig += " = ..."
|
|
1010
|
+
syms.append(sym({
|
|
1011
|
+
"name": tname, "kind": kind, "line": node.lineno, "col": col,
|
|
1012
|
+
"signature": sig, "scope": ".".join(scope_stack),
|
|
1013
|
+
}))
|
|
1014
|
+
|
|
1015
|
+
def visit_Import(self, node):
|
|
1016
|
+
# Parity with the single-file parser: imports are symbols too.
|
|
1017
|
+
for alias in node.names:
|
|
1018
|
+
name = alias.asname or alias.name
|
|
1019
|
+
syms.append(sym({
|
|
1020
|
+
"name": name, "kind": "import", "line": node.lineno,
|
|
1021
|
+
"col": node.col_offset, "signature": f"import {alias.name}",
|
|
1022
|
+
"scope": ".".join(scope_stack),
|
|
1023
|
+
}))
|
|
1024
|
+
|
|
1025
|
+
def visit_ImportFrom(self, node):
|
|
1026
|
+
module = node.module or ""
|
|
1027
|
+
for alias in node.names:
|
|
1028
|
+
name = alias.asname or alias.name
|
|
1029
|
+
syms.append(sym({
|
|
1030
|
+
"name": name, "kind": "import", "line": node.lineno,
|
|
1031
|
+
"col": node.col_offset, "signature": f"from {module} import {alias.name}",
|
|
1032
|
+
"scope": ".".join(scope_stack),
|
|
1033
|
+
}))
|
|
1034
|
+
|
|
1035
|
+
Visitor().visit(tree)
|
|
1036
|
+
|
|
1037
|
+
for node in ast.walk(tree):
|
|
1038
|
+
if isinstance(node, ast.Call):
|
|
1039
|
+
cname = leaf_name(node.func)
|
|
1040
|
+
if cname:
|
|
1041
|
+
refs.append({"toName": cname, "callType": "call", "line": node.lineno})
|
|
1042
|
+
elif isinstance(node, ast.Import):
|
|
1043
|
+
for alias in node.names:
|
|
1044
|
+
refs.append({
|
|
1045
|
+
"toName": alias.name.split(".")[-1], "callType": "import",
|
|
1046
|
+
"line": node.lineno, "module": alias.name,
|
|
1047
|
+
})
|
|
1048
|
+
elif isinstance(node, ast.ImportFrom):
|
|
1049
|
+
module = ("." * (node.level or 0)) + (node.module or "")
|
|
1050
|
+
for alias in node.names:
|
|
1051
|
+
refs.append({
|
|
1052
|
+
"toName": alias.name, "callType": "import",
|
|
1053
|
+
"line": node.lineno, "module": module,
|
|
1054
|
+
})
|
|
1055
|
+
elif isinstance(node, ast.ClassDef):
|
|
1056
|
+
for base in node.bases:
|
|
1057
|
+
bname = leaf_name(base)
|
|
1058
|
+
if bname:
|
|
1059
|
+
refs.append({"toName": bname, "callType": "inherit", "line": node.lineno})
|
|
1060
|
+
|
|
1061
|
+
result["symbols"] = syms
|
|
1062
|
+
result["refs"] = refs
|
|
1063
|
+
return result
|
|
1064
|
+
|
|
1065
|
+
def main():
|
|
1066
|
+
try:
|
|
1067
|
+
inputs = json.loads(sys.stdin.read())
|
|
1068
|
+
except Exception:
|
|
1069
|
+
print(json.dumps({"version": 1, "results": []}))
|
|
1070
|
+
return
|
|
1071
|
+
results = []
|
|
1072
|
+
for entry in inputs:
|
|
1073
|
+
name = entry.get("file", "")
|
|
1074
|
+
module_name = name.rsplit("/", 1)[-1].rsplit("\\\\", 1)[-1][:-3]
|
|
1075
|
+
results.append(parse_one(name, entry.get("content", ""), module_name))
|
|
1076
|
+
print(json.dumps({"version": 1, "results": results}))
|
|
1077
|
+
|
|
1078
|
+
main()
|
|
1079
|
+
`;
|
|
1080
|
+
_goBatchScriptPath = null;
|
|
1081
|
+
_pyBatchScriptPath = null;
|
|
996
1082
|
}
|
|
997
1083
|
});
|
|
998
1084
|
|
|
@@ -1002,7 +1088,7 @@ __export(generic_parser_exports, {
|
|
|
1002
1088
|
GENERIC_MAX_FILE_CHARS: () => GENERIC_MAX_FILE_CHARS,
|
|
1003
1089
|
GENERIC_MAX_SYMBOLS_DEFAULT: () => GENERIC_MAX_SYMBOLS_DEFAULT,
|
|
1004
1090
|
parseGeneric: () => parseGeneric,
|
|
1005
|
-
parseSymbols: () =>
|
|
1091
|
+
parseSymbols: () => parseSymbols
|
|
1006
1092
|
});
|
|
1007
1093
|
function patternsFor(lang) {
|
|
1008
1094
|
return LANG_PATTERNS[lang] ?? LANG_PATTERNS.other ?? [];
|
|
@@ -1090,7 +1176,7 @@ function parseGeneric(opts) {
|
|
|
1090
1176
|
}
|
|
1091
1177
|
return { file, lang, symbols, mtimeMs };
|
|
1092
1178
|
}
|
|
1093
|
-
async function
|
|
1179
|
+
async function parseSymbols(opts) {
|
|
1094
1180
|
return parseGeneric(opts);
|
|
1095
1181
|
}
|
|
1096
1182
|
var C_LIKE, LANG_PATTERNS, KEYWORDS, GENERIC_MAX_SYMBOLS_DEFAULT, GENERIC_MAX_FILE_CHARS;
|
|
@@ -1323,13 +1409,14 @@ var init_generic_parser = __esm({
|
|
|
1323
1409
|
var py_parser_exports = {};
|
|
1324
1410
|
__export(py_parser_exports, {
|
|
1325
1411
|
detectLang: () => detectLang,
|
|
1326
|
-
parseSymbols: () =>
|
|
1412
|
+
parseSymbols: () => parseSymbols2,
|
|
1413
|
+
resolvePythonBinary: () => resolvePythonBinary
|
|
1327
1414
|
});
|
|
1328
1415
|
import { spawn as spawn2 } from "node:child_process";
|
|
1329
1416
|
import * as fs5 from "node:fs/promises";
|
|
1330
1417
|
import * as os2 from "node:os";
|
|
1331
1418
|
import * as path7 from "node:path";
|
|
1332
|
-
async function
|
|
1419
|
+
async function parseSymbols2(opts) {
|
|
1333
1420
|
const { file, content, lang } = opts;
|
|
1334
1421
|
try {
|
|
1335
1422
|
const native = await withSpawnGate(() => syncPyParse(file, content, lang));
|
|
@@ -1403,6 +1490,10 @@ function spawnPyParser(pyBinary, scriptPath, filePath, content) {
|
|
|
1403
1490
|
});
|
|
1404
1491
|
});
|
|
1405
1492
|
}
|
|
1493
|
+
function resolvePythonBinary() {
|
|
1494
|
+
cachedPyBinary ??= resolvePython();
|
|
1495
|
+
return cachedPyBinary;
|
|
1496
|
+
}
|
|
1406
1497
|
async function syncPyParse(filePath, content, lang) {
|
|
1407
1498
|
try {
|
|
1408
1499
|
if (!_cachedScriptPath) {
|
|
@@ -1655,47 +1746,751 @@ class ModuleVisitor(ast.NodeVisitor):
|
|
|
1655
1746
|
scope=".".join(self.scope_stack),
|
|
1656
1747
|
))
|
|
1657
1748
|
|
|
1658
|
-
visitor = ModuleVisitor()
|
|
1659
|
-
visitor.visit(tree)
|
|
1749
|
+
visitor = ModuleVisitor()
|
|
1750
|
+
visitor.visit(tree)
|
|
1751
|
+
|
|
1752
|
+
# Refs need a separate full walk: ModuleVisitor deliberately does not descend
|
|
1753
|
+
# into function bodies (it would index locals as symbols), but that is exactly
|
|
1754
|
+
# where the calls are.
|
|
1755
|
+
for node in ast.walk(tree):
|
|
1756
|
+
if isinstance(node, ast.Call):
|
|
1757
|
+
name = leaf_name(node.func)
|
|
1758
|
+
if name:
|
|
1759
|
+
refs.append({"toName": name, "callType": "call", "line": node.lineno})
|
|
1760
|
+
elif isinstance(node, ast.Import):
|
|
1761
|
+
for alias in node.names:
|
|
1762
|
+
refs.append({
|
|
1763
|
+
"toName": alias.name.split(".")[-1],
|
|
1764
|
+
"callType": "import",
|
|
1765
|
+
"line": node.lineno,
|
|
1766
|
+
"module": alias.name,
|
|
1767
|
+
})
|
|
1768
|
+
elif isinstance(node, ast.ImportFrom):
|
|
1769
|
+
# PEP 328: node.level is the number of leading dots. Preserving them is
|
|
1770
|
+
# what lets the resolver walk up from the importing file's package \u2014
|
|
1771
|
+
# dropping them made \`from .foo import X\` indistinguishable from an
|
|
1772
|
+
# absolute \`foo\`.
|
|
1773
|
+
module = ("." * (node.level or 0)) + (node.module or "")
|
|
1774
|
+
for alias in node.names:
|
|
1775
|
+
refs.append({
|
|
1776
|
+
"toName": alias.name,
|
|
1777
|
+
"callType": "import",
|
|
1778
|
+
"line": node.lineno,
|
|
1779
|
+
"module": module,
|
|
1780
|
+
})
|
|
1781
|
+
elif isinstance(node, ast.ClassDef):
|
|
1782
|
+
for base in node.bases:
|
|
1783
|
+
name = leaf_name(base)
|
|
1784
|
+
if name:
|
|
1785
|
+
refs.append({"toName": name, "callType": "inherit", "line": node.lineno})
|
|
1786
|
+
|
|
1787
|
+
print(json.dumps({"symbols": [s.to_dict() for s in syms], "refs": refs}))
|
|
1788
|
+
`;
|
|
1789
|
+
_cachedScriptPath = null;
|
|
1790
|
+
}
|
|
1791
|
+
});
|
|
1792
|
+
|
|
1793
|
+
// src/codebase-index/ts-parser.ts
|
|
1794
|
+
var ts_parser_exports = {};
|
|
1795
|
+
__export(ts_parser_exports, {
|
|
1796
|
+
detectLang: () => detectLang,
|
|
1797
|
+
parseSymbols: () => parseSymbols3
|
|
1798
|
+
});
|
|
1799
|
+
function loadTypescript() {
|
|
1800
|
+
tsLoad ??= import("@typescript/typescript6").then((m) => {
|
|
1801
|
+
ts = m.default ?? m;
|
|
1802
|
+
return ts;
|
|
1803
|
+
});
|
|
1804
|
+
return tsLoad;
|
|
1805
|
+
}
|
|
1806
|
+
function kindMap() {
|
|
1807
|
+
kindMapCache ??= {
|
|
1808
|
+
[ts.SyntaxKind.ClassDeclaration]: "class",
|
|
1809
|
+
[ts.SyntaxKind.InterfaceDeclaration]: "interface",
|
|
1810
|
+
[ts.SyntaxKind.EnumDeclaration]: "enum",
|
|
1811
|
+
[ts.SyntaxKind.TypeAliasDeclaration]: "type",
|
|
1812
|
+
[ts.SyntaxKind.FunctionDeclaration]: "function",
|
|
1813
|
+
[ts.SyntaxKind.MethodDeclaration]: "method",
|
|
1814
|
+
[ts.SyntaxKind.GetAccessor]: "property",
|
|
1815
|
+
[ts.SyntaxKind.SetAccessor]: "property",
|
|
1816
|
+
[ts.SyntaxKind.PropertyDeclaration]: "property",
|
|
1817
|
+
[ts.SyntaxKind.Parameter]: "parameter",
|
|
1818
|
+
[ts.SyntaxKind.NamespaceExportDeclaration]: "namespace"
|
|
1819
|
+
};
|
|
1820
|
+
return kindMapCache;
|
|
1821
|
+
}
|
|
1822
|
+
function kindOf(node) {
|
|
1823
|
+
if (ts.isVariableDeclaration(node)) {
|
|
1824
|
+
const parent = node.parent;
|
|
1825
|
+
if (ts.isVariableDeclarationList(parent)) {
|
|
1826
|
+
const flags = parent.flags;
|
|
1827
|
+
if (flags & ts.NodeFlags.Let) return "let";
|
|
1828
|
+
if (flags & ts.NodeFlags.Const) return "const";
|
|
1829
|
+
return "var";
|
|
1830
|
+
}
|
|
1831
|
+
}
|
|
1832
|
+
if (ts.isModuleDeclaration(node)) return "namespace";
|
|
1833
|
+
return kindMap()[node.kind] ?? null;
|
|
1834
|
+
}
|
|
1835
|
+
function getSignature(printer, node, sourceFile) {
|
|
1836
|
+
const raw = printer.printNode(ts.EmitHint.Unspecified, node, sourceFile);
|
|
1837
|
+
return raw.replace(/\s+/g, " ").slice(0, 500);
|
|
1838
|
+
}
|
|
1839
|
+
function getJsDoc(node, sourceFile) {
|
|
1840
|
+
const fullText = sourceFile.getFullText();
|
|
1841
|
+
const nodePos = node.getFullStart();
|
|
1842
|
+
const comments = ts.getLeadingCommentRanges(fullText, nodePos);
|
|
1843
|
+
if (!comments) return "";
|
|
1844
|
+
for (const range of comments) {
|
|
1845
|
+
const commentText = fullText.slice(range.pos, range.end);
|
|
1846
|
+
const trimmed = commentText.trim();
|
|
1847
|
+
if (trimmed.startsWith("/**") && trimmed.endsWith("*/")) {
|
|
1848
|
+
const inner = trimmed.slice(3, -2).replace(/^[ \t]*\*[ ]?/gm, "").trim();
|
|
1849
|
+
return inner.split("\n")[0]?.trim().slice(0, 200) ?? "";
|
|
1850
|
+
}
|
|
1851
|
+
}
|
|
1852
|
+
return "";
|
|
1853
|
+
}
|
|
1854
|
+
function pushScopeName(node, parts) {
|
|
1855
|
+
if (ts.isClassDeclaration(node) || ts.isInterfaceDeclaration(node) || ts.isEnumDeclaration(node) || ts.isTypeAliasDeclaration(node)) {
|
|
1856
|
+
parts.push(node.name?.text ?? "Anon");
|
|
1857
|
+
} else if (ts.isMethodDeclaration(node) || ts.isGetAccessor(node) || ts.isSetAccessor(node) || ts.isPropertyDeclaration(node) || ts.isFunctionDeclaration(node)) {
|
|
1858
|
+
if (node.name && ts.isIdentifier(node.name)) {
|
|
1859
|
+
parts.push(node.name.text);
|
|
1860
|
+
}
|
|
1861
|
+
}
|
|
1862
|
+
}
|
|
1863
|
+
async function parseSymbols3(opts) {
|
|
1864
|
+
const { file, content, lang } = opts;
|
|
1865
|
+
await loadTypescript();
|
|
1866
|
+
let sourceFile;
|
|
1867
|
+
try {
|
|
1868
|
+
sourceFile = ts.createSourceFile(file, content, ts.ScriptTarget.Latest, true);
|
|
1869
|
+
} catch {
|
|
1870
|
+
return { file, lang, symbols: [], mtimeMs: Date.now() };
|
|
1871
|
+
}
|
|
1872
|
+
const symbols = [];
|
|
1873
|
+
const refs = [];
|
|
1874
|
+
const printer = ts.createPrinter({});
|
|
1875
|
+
function visit(node, funcDepth, scopeParts) {
|
|
1876
|
+
const kind = kindOf(node);
|
|
1877
|
+
if (kind) {
|
|
1878
|
+
if ((kind === "const" || kind === "let" || kind === "var" || kind === "parameter") && funcDepth > 0) {
|
|
1879
|
+
} else {
|
|
1880
|
+
const nameNode = node.name;
|
|
1881
|
+
if (!nameNode || !ts.isIdentifier(nameNode)) {
|
|
1882
|
+
return;
|
|
1883
|
+
}
|
|
1884
|
+
const name = nameNode.text;
|
|
1885
|
+
const pos2 = nameNode.getStart(sourceFile);
|
|
1886
|
+
const { line: line2, character } = sourceFile.getLineAndCharacterOfPosition(pos2);
|
|
1887
|
+
const scope = scopeParts.join(".");
|
|
1888
|
+
const signature = getSignature(printer, node, sourceFile);
|
|
1889
|
+
const docComment = getJsDoc(node, sourceFile);
|
|
1890
|
+
const text = [name, signature, docComment].filter(Boolean).join(" | ");
|
|
1891
|
+
symbols.push({
|
|
1892
|
+
id: 0,
|
|
1893
|
+
lang,
|
|
1894
|
+
kind,
|
|
1895
|
+
name,
|
|
1896
|
+
file,
|
|
1897
|
+
line: line2 + 1,
|
|
1898
|
+
col: character,
|
|
1899
|
+
signature,
|
|
1900
|
+
docComment,
|
|
1901
|
+
scope,
|
|
1902
|
+
text
|
|
1903
|
+
});
|
|
1904
|
+
}
|
|
1905
|
+
}
|
|
1906
|
+
const pos = node.getStart(sourceFile);
|
|
1907
|
+
const { line } = sourceFile.getLineAndCharacterOfPosition(pos);
|
|
1908
|
+
const lineNum = line + 1;
|
|
1909
|
+
if (ts.isCallExpression(node)) {
|
|
1910
|
+
const expr = node.expression;
|
|
1911
|
+
if (ts.isIdentifier(expr)) {
|
|
1912
|
+
refs.push({ fromId: 0, toName: expr.text, callType: "call", line: lineNum });
|
|
1913
|
+
}
|
|
1914
|
+
} else if (ts.isPropertyAccessExpression(node)) {
|
|
1915
|
+
if (ts.isIdentifier(node.expression)) {
|
|
1916
|
+
refs.push({ fromId: 0, toName: node.expression.text, callType: "call", line: lineNum });
|
|
1917
|
+
}
|
|
1918
|
+
} else if (ts.isTypeReferenceNode(node)) {
|
|
1919
|
+
const name = getTypeName(node.typeName);
|
|
1920
|
+
if (name) refs.push({ fromId: 0, toName: name, callType: "type_ref", line: lineNum });
|
|
1921
|
+
} else if (ts.isHeritageClause(node)) {
|
|
1922
|
+
for (const t of node.types) {
|
|
1923
|
+
const name = getTypeName(t.expression);
|
|
1924
|
+
if (name)
|
|
1925
|
+
refs.push({
|
|
1926
|
+
fromId: 0,
|
|
1927
|
+
toName: name,
|
|
1928
|
+
callType: node.token === ts.SyntaxKind.ExtendsKeyword ? "inherit" : "implement",
|
|
1929
|
+
line: lineNum
|
|
1930
|
+
});
|
|
1931
|
+
}
|
|
1932
|
+
} else if (ts.isImportDeclaration(node)) {
|
|
1933
|
+
emitImportSpecifierRefs(node, refs, lineNum);
|
|
1934
|
+
} else if (ts.isExportDeclaration(node) && node.moduleSpecifier) {
|
|
1935
|
+
emitExportSpecifierRefs(node, refs, lineNum);
|
|
1936
|
+
}
|
|
1937
|
+
const scopeIdx = scopeParts.length;
|
|
1938
|
+
pushScopeName(node, scopeParts);
|
|
1939
|
+
const childFuncDepth = ts.isFunctionLike(node) ? funcDepth + 1 : funcDepth;
|
|
1940
|
+
ts.forEachChild(node, (child) => visit(child, childFuncDepth, scopeParts));
|
|
1941
|
+
scopeParts.length = scopeIdx;
|
|
1942
|
+
}
|
|
1943
|
+
visit(sourceFile, 0, []);
|
|
1944
|
+
return { file, lang, symbols, refs: deduplicateRefs(refs), mtimeMs: Date.now() };
|
|
1945
|
+
}
|
|
1946
|
+
function getTypeName(name) {
|
|
1947
|
+
if (ts.isIdentifier(name)) return name.text;
|
|
1948
|
+
if (ts.isQualifiedName(name)) return `${getTypeName(name.left)}.${name.right.text}`;
|
|
1949
|
+
return "";
|
|
1950
|
+
}
|
|
1951
|
+
function deduplicateRefs(refs) {
|
|
1952
|
+
const seen = /* @__PURE__ */ new Set();
|
|
1953
|
+
return refs.filter((r) => {
|
|
1954
|
+
const key = `${r.toName}:${r.callType}:${r.line}:${r.module ?? ""}`;
|
|
1955
|
+
if (seen.has(key)) return false;
|
|
1956
|
+
seen.add(key);
|
|
1957
|
+
return true;
|
|
1958
|
+
});
|
|
1959
|
+
}
|
|
1960
|
+
function getImportSpecifierName(spec) {
|
|
1961
|
+
return spec.propertyName?.text ?? spec.name.text;
|
|
1962
|
+
}
|
|
1963
|
+
function emitImportSpecifierRefs(node, refs, lineNum) {
|
|
1964
|
+
const module = moduleSpecifierOf(node.moduleSpecifier);
|
|
1965
|
+
const clause = node.importClause;
|
|
1966
|
+
if (!clause) {
|
|
1967
|
+
if (module) {
|
|
1968
|
+
refs.push({ fromId: 0, toName: module, callType: "import", line: lineNum, module });
|
|
1969
|
+
}
|
|
1970
|
+
return;
|
|
1971
|
+
}
|
|
1972
|
+
if (clause.name) {
|
|
1973
|
+
refs.push({ fromId: 0, toName: clause.name.text, callType: "import", line: lineNum, module });
|
|
1974
|
+
}
|
|
1975
|
+
const bindings = clause.namedBindings;
|
|
1976
|
+
if (!bindings) return;
|
|
1977
|
+
if (ts.isNamedImports(bindings)) {
|
|
1978
|
+
for (const element of bindings.elements) {
|
|
1979
|
+
refs.push({
|
|
1980
|
+
fromId: 0,
|
|
1981
|
+
toName: getImportSpecifierName(element),
|
|
1982
|
+
callType: "import",
|
|
1983
|
+
line: lineNum,
|
|
1984
|
+
module
|
|
1985
|
+
});
|
|
1986
|
+
}
|
|
1987
|
+
} else if (ts.isNamespaceImport(bindings)) {
|
|
1988
|
+
refs.push({
|
|
1989
|
+
fromId: 0,
|
|
1990
|
+
toName: bindings.name.text,
|
|
1991
|
+
callType: "import",
|
|
1992
|
+
line: lineNum,
|
|
1993
|
+
module
|
|
1994
|
+
});
|
|
1995
|
+
}
|
|
1996
|
+
}
|
|
1997
|
+
function moduleSpecifierOf(node) {
|
|
1998
|
+
return node && ts.isStringLiteral(node) ? node.text : void 0;
|
|
1999
|
+
}
|
|
2000
|
+
function emitExportSpecifierRefs(node, refs, lineNum) {
|
|
2001
|
+
const module = moduleSpecifierOf(node.moduleSpecifier);
|
|
2002
|
+
const clause = node.exportClause;
|
|
2003
|
+
if (clause && ts.isNamespaceExport(clause)) {
|
|
2004
|
+
refs.push({ fromId: 0, toName: clause.name.text, callType: "import", line: lineNum, module });
|
|
2005
|
+
return;
|
|
2006
|
+
}
|
|
2007
|
+
if (clause && ts.isNamedExports(clause)) {
|
|
2008
|
+
for (const element of clause.elements) {
|
|
2009
|
+
const originalName = element.propertyName?.text ?? element.name.text;
|
|
2010
|
+
refs.push({ fromId: 0, toName: originalName, callType: "import", line: lineNum, module });
|
|
2011
|
+
}
|
|
2012
|
+
return;
|
|
2013
|
+
}
|
|
2014
|
+
if (module) {
|
|
2015
|
+
refs.push({ fromId: 0, toName: module, callType: "import", line: lineNum, module });
|
|
2016
|
+
}
|
|
2017
|
+
}
|
|
2018
|
+
var ts, tsLoad, kindMapCache;
|
|
2019
|
+
var init_ts_parser = __esm({
|
|
2020
|
+
"src/codebase-index/ts-parser.ts"() {
|
|
2021
|
+
"use strict";
|
|
2022
|
+
init_languages();
|
|
2023
|
+
tsLoad = null;
|
|
2024
|
+
kindMapCache = null;
|
|
2025
|
+
}
|
|
2026
|
+
});
|
|
2027
|
+
|
|
2028
|
+
// src/codebase-index/go-parser.ts
|
|
2029
|
+
var go_parser_exports = {};
|
|
2030
|
+
__export(go_parser_exports, {
|
|
2031
|
+
detectLang: () => detectLang,
|
|
2032
|
+
parseSymbols: () => parseSymbols4
|
|
2033
|
+
});
|
|
2034
|
+
import { spawn as spawn3 } from "node:child_process";
|
|
2035
|
+
import * as os3 from "node:os";
|
|
2036
|
+
import * as path8 from "node:path";
|
|
2037
|
+
import * as fs6 from "node:fs/promises";
|
|
2038
|
+
async function parseSymbols4(opts) {
|
|
2039
|
+
const { file, content, lang } = opts;
|
|
2040
|
+
try {
|
|
2041
|
+
const parsed = await withSpawnGate(() => syncGoParse(file, content, lang));
|
|
2042
|
+
if (parsed.symbols.length > 0) {
|
|
2043
|
+
return parsed;
|
|
2044
|
+
}
|
|
2045
|
+
const fallback = fallbackParse(file, content, lang);
|
|
2046
|
+
return parsed.refs?.length ? { ...fallback, refs: parsed.refs } : fallback;
|
|
2047
|
+
} catch {
|
|
2048
|
+
return fallbackParse(file, content, lang);
|
|
2049
|
+
}
|
|
2050
|
+
}
|
|
2051
|
+
function fallbackParse(filePath, content, lang) {
|
|
2052
|
+
if (!/^\s*package\s+[A-Za-z_]\w*/m.test(content) || hasUnbalancedDelimiters(content)) {
|
|
2053
|
+
return { file: filePath, lang, symbols: [], mtimeMs: Date.now() };
|
|
2054
|
+
}
|
|
2055
|
+
const symbols = [];
|
|
2056
|
+
const packageName = content.match(/^\s*package\s+([A-Za-z_]\w*)/m)?.[1] ?? "";
|
|
2057
|
+
const lines = content.split(/\r?\n/);
|
|
2058
|
+
for (const [idx, line] of lines.entries()) {
|
|
2059
|
+
const trimmed = line.trimStart();
|
|
2060
|
+
const col = line.length - trimmed.length + 1;
|
|
2061
|
+
const fn = /^func\s+(?:\([^)]*\)\s*)?([A-Za-z_]\w*)\s*\(/.exec(trimmed);
|
|
2062
|
+
if (fn?.[1]) {
|
|
2063
|
+
addFallbackSymbol(symbols, {
|
|
2064
|
+
filePath,
|
|
2065
|
+
lang,
|
|
2066
|
+
kind: trimmed.startsWith("func (") ? "method" : "function",
|
|
2067
|
+
name: fn[1],
|
|
2068
|
+
line: idx + 1,
|
|
2069
|
+
col,
|
|
2070
|
+
signature: trimmed,
|
|
2071
|
+
scope: packageName ? `${packageName}.${fn[1]}` : fn[1]
|
|
2072
|
+
});
|
|
2073
|
+
continue;
|
|
2074
|
+
}
|
|
2075
|
+
const typeDecl = /^type\s+([A-Za-z_]\w*)\b/.exec(trimmed);
|
|
2076
|
+
if (typeDecl?.[1]) {
|
|
2077
|
+
addFallbackSymbol(symbols, {
|
|
2078
|
+
filePath,
|
|
2079
|
+
lang,
|
|
2080
|
+
kind: "type",
|
|
2081
|
+
name: typeDecl[1],
|
|
2082
|
+
line: idx + 1,
|
|
2083
|
+
col,
|
|
2084
|
+
signature: trimmed,
|
|
2085
|
+
scope: packageName
|
|
2086
|
+
});
|
|
2087
|
+
continue;
|
|
2088
|
+
}
|
|
2089
|
+
const valueDecl = /^(const|var)\s+([A-Za-z_]\w*)\b/.exec(trimmed);
|
|
2090
|
+
if (valueDecl?.[1] && valueDecl[2]) {
|
|
2091
|
+
addFallbackSymbol(symbols, {
|
|
2092
|
+
filePath,
|
|
2093
|
+
lang,
|
|
2094
|
+
kind: valueDecl[1],
|
|
2095
|
+
name: valueDecl[2],
|
|
2096
|
+
line: idx + 1,
|
|
2097
|
+
col,
|
|
2098
|
+
signature: trimmed,
|
|
2099
|
+
scope: packageName
|
|
2100
|
+
});
|
|
2101
|
+
}
|
|
2102
|
+
}
|
|
2103
|
+
return { file: filePath, lang, symbols, mtimeMs: Date.now() };
|
|
2104
|
+
}
|
|
2105
|
+
function addFallbackSymbol(symbols, opts) {
|
|
2106
|
+
symbols.push({
|
|
2107
|
+
id: 0,
|
|
2108
|
+
lang: opts.lang,
|
|
2109
|
+
kind: opts.kind,
|
|
2110
|
+
name: opts.name,
|
|
2111
|
+
file: opts.filePath,
|
|
2112
|
+
line: opts.line,
|
|
2113
|
+
col: opts.col,
|
|
2114
|
+
signature: opts.signature,
|
|
2115
|
+
docComment: "",
|
|
2116
|
+
scope: opts.scope,
|
|
2117
|
+
text: `${opts.name} ${opts.signature}`.trim()
|
|
2118
|
+
});
|
|
2119
|
+
}
|
|
2120
|
+
function hasUnbalancedDelimiters(content) {
|
|
2121
|
+
const pairs = { "(": ")", "[": "]", "{": "}" };
|
|
2122
|
+
const closers = new Set(Object.values(pairs));
|
|
2123
|
+
const stack = [];
|
|
2124
|
+
for (const ch of content) {
|
|
2125
|
+
if (pairs[ch]) {
|
|
2126
|
+
stack.push(pairs[ch]);
|
|
2127
|
+
} else if (closers.has(ch) && stack.pop() !== ch) {
|
|
2128
|
+
return true;
|
|
2129
|
+
}
|
|
2130
|
+
}
|
|
2131
|
+
return stack.length > 0;
|
|
2132
|
+
}
|
|
2133
|
+
async function syncGoParse(filePath, content, lang) {
|
|
2134
|
+
try {
|
|
2135
|
+
let scriptPath = _cachedGoScriptPath;
|
|
2136
|
+
if (!scriptPath) {
|
|
2137
|
+
const tmpDir = await fs6.mkdtemp(path8.join(os3.tmpdir(), "ws-go-parse-"));
|
|
2138
|
+
scriptPath = path8.join(tmpDir, "parse.go");
|
|
2139
|
+
await fs6.writeFile(scriptPath, GO_PARSE_SCRIPT, "utf8");
|
|
2140
|
+
_cachedGoScriptPath = scriptPath;
|
|
2141
|
+
}
|
|
2142
|
+
const goBinary = resolveWin32Command("go");
|
|
2143
|
+
const goResult = await new Promise(
|
|
2144
|
+
(resolve2, reject) => {
|
|
2145
|
+
let settled = false;
|
|
2146
|
+
const proc = spawn3(goBinary, ["run", scriptPath], {
|
|
2147
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
2148
|
+
windowsHide: true
|
|
2149
|
+
});
|
|
2150
|
+
proc.on("error", (err) => {
|
|
2151
|
+
if (settled) return;
|
|
2152
|
+
settled = true;
|
|
2153
|
+
reject(err);
|
|
2154
|
+
});
|
|
2155
|
+
let stdout2 = "";
|
|
2156
|
+
proc.stdout?.on("data", (chunk) => {
|
|
2157
|
+
stdout2 += chunk.toString();
|
|
2158
|
+
});
|
|
2159
|
+
proc.stderr?.resume();
|
|
2160
|
+
proc.stdin?.write(content);
|
|
2161
|
+
proc.stdin?.end();
|
|
2162
|
+
const timer = setTimeout(() => {
|
|
2163
|
+
if (settled) return;
|
|
2164
|
+
settled = true;
|
|
2165
|
+
proc.kill("SIGKILL");
|
|
2166
|
+
reject(new Error("timeout"));
|
|
2167
|
+
}, 15e3);
|
|
2168
|
+
timer.unref?.();
|
|
2169
|
+
proc.on("close", (code2) => {
|
|
2170
|
+
if (settled) return;
|
|
2171
|
+
settled = true;
|
|
2172
|
+
clearTimeout(timer);
|
|
2173
|
+
resolve2({ code: code2, stdout: stdout2 });
|
|
2174
|
+
});
|
|
2175
|
+
}
|
|
2176
|
+
);
|
|
2177
|
+
const { code, stdout } = goResult;
|
|
2178
|
+
if (code !== 0 || !stdout.trim()) {
|
|
2179
|
+
return { file: filePath, lang, symbols: [], mtimeMs: Date.now() };
|
|
2180
|
+
}
|
|
2181
|
+
const { symbols: rawSymbols, refs } = parseParserOutput(stdout, lang);
|
|
2182
|
+
const symbols = rawSymbols.map((s) => ({
|
|
2183
|
+
id: 0,
|
|
2184
|
+
lang,
|
|
2185
|
+
kind: s.kind,
|
|
2186
|
+
name: s.name,
|
|
2187
|
+
file: filePath,
|
|
2188
|
+
line: s.line,
|
|
2189
|
+
col: s.col,
|
|
2190
|
+
signature: s.signature ?? "",
|
|
2191
|
+
docComment: "",
|
|
2192
|
+
scope: s.scope ?? "",
|
|
2193
|
+
text: `${s.name} ${s.signature ?? ""}`.trim()
|
|
2194
|
+
}));
|
|
2195
|
+
return { file: filePath, lang, symbols, refs, mtimeMs: Date.now() };
|
|
2196
|
+
} catch {
|
|
2197
|
+
return { file: filePath, lang, symbols: [], mtimeMs: Date.now() };
|
|
2198
|
+
}
|
|
2199
|
+
}
|
|
2200
|
+
var GO_PARSE_SCRIPT, _cachedGoScriptPath;
|
|
2201
|
+
var init_go_parser = __esm({
|
|
2202
|
+
"src/codebase-index/go-parser.ts"() {
|
|
2203
|
+
"use strict";
|
|
2204
|
+
init_win32_resolve();
|
|
2205
|
+
init_parser_output();
|
|
2206
|
+
init_spawn_gate();
|
|
2207
|
+
init_languages();
|
|
2208
|
+
GO_PARSE_SCRIPT = `
|
|
2209
|
+
package main
|
|
2210
|
+
|
|
2211
|
+
import (
|
|
2212
|
+
"encoding/json"
|
|
2213
|
+
"fmt"
|
|
2214
|
+
"go/ast"
|
|
2215
|
+
"go/parser"
|
|
2216
|
+
"go/token"
|
|
2217
|
+
"io"
|
|
2218
|
+
"os"
|
|
2219
|
+
"strconv"
|
|
2220
|
+
"strings"
|
|
2221
|
+
)
|
|
2222
|
+
|
|
2223
|
+
type Sym struct {
|
|
2224
|
+
Name string \`json:"name"\`
|
|
2225
|
+
Kind string \`json:"kind"\`
|
|
2226
|
+
Line int \`json:"line"\`
|
|
2227
|
+
Col int \`json:"col"\`
|
|
2228
|
+
Signature string \`json:"signature"\`
|
|
2229
|
+
Scope string \`json:"scope"\`
|
|
2230
|
+
}
|
|
2231
|
+
|
|
2232
|
+
// Ref is a cross-reference emitted alongside the symbols, so one \`go run\`
|
|
2233
|
+
// yields both. Module is the import path for CallType "import", else empty.
|
|
2234
|
+
type Ref struct {
|
|
2235
|
+
ToName string \`json:"toName"\`
|
|
2236
|
+
CallType string \`json:"callType"\`
|
|
2237
|
+
Line int \`json:"line"\`
|
|
2238
|
+
Module string \`json:"module"\`
|
|
2239
|
+
}
|
|
2240
|
+
|
|
2241
|
+
type Result struct {
|
|
2242
|
+
Symbols []Sym \`json:"symbols"\`
|
|
2243
|
+
Refs []Ref \`json:"refs"\`
|
|
2244
|
+
}
|
|
2245
|
+
|
|
2246
|
+
func emptyResult() string {
|
|
2247
|
+
return "{\\"symbols\\":[],\\"refs\\":[]}"
|
|
2248
|
+
}
|
|
2249
|
+
|
|
2250
|
+
func main() {
|
|
2251
|
+
src, err := io.ReadAll(os.Stdin)
|
|
2252
|
+
if err != nil {
|
|
2253
|
+
fmt.Print(emptyResult())
|
|
2254
|
+
return
|
|
2255
|
+
}
|
|
2256
|
+
fset := token.NewFileSet()
|
|
2257
|
+
node, err := parser.ParseFile(fset, "src.go", src, 0)
|
|
2258
|
+
if err != nil {
|
|
2259
|
+
fmt.Print(emptyResult())
|
|
2260
|
+
return
|
|
2261
|
+
}
|
|
2262
|
+
|
|
2263
|
+
var syms []Sym
|
|
2264
|
+
|
|
2265
|
+
// Package-level scope
|
|
2266
|
+
pkgScope := node.Name.Name
|
|
2267
|
+
|
|
2268
|
+
// Collect all top-level declarations
|
|
2269
|
+
for _, decl := range node.Decls {
|
|
2270
|
+
switch d := decl.(type) {
|
|
2271
|
+
case *ast.FuncDecl:
|
|
2272
|
+
name := d.Name.Name
|
|
2273
|
+
kind := "function"
|
|
2274
|
+
scope := pkgScope
|
|
2275
|
+
if d.Recv != nil && len(d.Recv.List) > 0 {
|
|
2276
|
+
scope = pkgScope + "." + recvTypeName(d.Recv.List[0].Type) + "." + name
|
|
2277
|
+
kind = "method"
|
|
2278
|
+
} else {
|
|
2279
|
+
scope = pkgScope + "." + name
|
|
2280
|
+
}
|
|
2281
|
+
pos := fset.Position(d.Pos())
|
|
2282
|
+
sig := formatFuncSig(d)
|
|
2283
|
+
syms = append(syms, Sym{Name: name, Kind: kind, Line: pos.Line, Col: pos.Column, Signature: sig, Scope: scope})
|
|
2284
|
+
|
|
2285
|
+
case *ast.GenDecl:
|
|
2286
|
+
for _, spec := range d.Specs {
|
|
2287
|
+
switch s := spec.(type) {
|
|
2288
|
+
case *ast.TypeSpec:
|
|
2289
|
+
name := s.Name.Name
|
|
2290
|
+
pos := fset.Position(s.Pos())
|
|
2291
|
+
sig := "type " + name
|
|
2292
|
+
if s.TypeParams != nil {
|
|
2293
|
+
sig += formatTypeParams(s.TypeParams)
|
|
2294
|
+
}
|
|
2295
|
+
if st, ok := s.Type.(*ast.StructType); ok {
|
|
2296
|
+
sig += " = struct { " + formatFields(st.Fields.List) + " }"
|
|
2297
|
+
} else if it, ok := s.Type.(*ast.InterfaceType); ok {
|
|
2298
|
+
sig += " = interface { " + formatMethods(it.Methods.List) + " }"
|
|
2299
|
+
} else {
|
|
2300
|
+
sig += " = " + formatType(s.Type)
|
|
2301
|
+
}
|
|
2302
|
+
syms = append(syms, Sym{Name: name, Kind: "type", Line: pos.Line, Col: pos.Column, Signature: sig, Scope: pkgScope})
|
|
2303
|
+
|
|
2304
|
+
case *ast.ValueSpec:
|
|
2305
|
+
for _, n := range s.Names {
|
|
2306
|
+
name := n.Name
|
|
2307
|
+
pos := fset.Position(n.Pos())
|
|
2308
|
+
kind := "var"
|
|
2309
|
+
if d.Tok == token.CONST {
|
|
2310
|
+
kind = "const"
|
|
2311
|
+
}
|
|
2312
|
+
sig := kind + " " + name
|
|
2313
|
+
if s.Type != nil {
|
|
2314
|
+
sig += " " + formatType(s.Type)
|
|
2315
|
+
}
|
|
2316
|
+
syms = append(syms, Sym{Name: name, Kind: kind, Line: pos.Line, Col: pos.Column, Signature: sig, Scope: pkgScope})
|
|
2317
|
+
}
|
|
2318
|
+
}
|
|
2319
|
+
}
|
|
2320
|
+
}
|
|
2321
|
+
}
|
|
2322
|
+
|
|
2323
|
+
refs := []Ref{}
|
|
2324
|
+
ast.Inspect(node, func(n ast.Node) bool {
|
|
2325
|
+
switch expr := n.(type) {
|
|
2326
|
+
case *ast.CallExpr:
|
|
2327
|
+
line := fset.Position(expr.Pos()).Line
|
|
2328
|
+
switch fun := expr.Fun.(type) {
|
|
2329
|
+
case *ast.Ident:
|
|
2330
|
+
refs = append(refs, Ref{ToName: fun.Name, CallType: "call", Line: line})
|
|
2331
|
+
case *ast.SelectorExpr:
|
|
2332
|
+
// Record the selected name (\`Join\` of \`filepath.Join\`): it is the
|
|
2333
|
+
// declared symbol name, so it resolves the same way the TypeScript
|
|
2334
|
+
// and Python extractors' call refs do.
|
|
2335
|
+
refs = append(refs, Ref{ToName: fun.Sel.Name, CallType: "call", Line: line})
|
|
2336
|
+
}
|
|
2337
|
+
case *ast.ImportSpec:
|
|
2338
|
+
if expr.Path != nil {
|
|
2339
|
+
if importPath, uerr := strconv.Unquote(expr.Path.Value); uerr == nil {
|
|
2340
|
+
line := fset.Position(expr.Pos()).Line
|
|
2341
|
+
// A Go import names a package, not a symbol; the package's
|
|
2342
|
+
// last path segment is the name it is referenced by.
|
|
2343
|
+
name := importPath
|
|
2344
|
+
if idx := strings.LastIndex(importPath, "/"); idx >= 0 {
|
|
2345
|
+
name = importPath[idx+1:]
|
|
2346
|
+
}
|
|
2347
|
+
refs = append(refs, Ref{ToName: name, CallType: "import", Line: line, Module: importPath})
|
|
2348
|
+
}
|
|
2349
|
+
}
|
|
2350
|
+
}
|
|
2351
|
+
return true
|
|
2352
|
+
})
|
|
2353
|
+
|
|
2354
|
+
if syms == nil {
|
|
2355
|
+
syms = []Sym{}
|
|
2356
|
+
}
|
|
2357
|
+
data, err := json.Marshal(Result{Symbols: syms, Refs: refs})
|
|
2358
|
+
if err != nil {
|
|
2359
|
+
fmt.Print(emptyResult())
|
|
2360
|
+
return
|
|
2361
|
+
}
|
|
2362
|
+
fmt.Print(string(data))
|
|
2363
|
+
}
|
|
2364
|
+
|
|
2365
|
+
func recvTypeName(t ast.Expr) string {
|
|
2366
|
+
switch v := t.(type) {
|
|
2367
|
+
case *ast.Ident:
|
|
2368
|
+
return v.Name
|
|
2369
|
+
case *ast.StarExpr:
|
|
2370
|
+
return recvTypeName(v.X)
|
|
2371
|
+
default:
|
|
2372
|
+
return "?"
|
|
2373
|
+
}
|
|
2374
|
+
}
|
|
2375
|
+
|
|
2376
|
+
func formatFuncSig(d *ast.FuncDecl) string {
|
|
2377
|
+
scope := ""
|
|
2378
|
+
if d.Recv != nil && len(d.Recv.List) > 0 {
|
|
2379
|
+
scope = "(" + formatFieldList(d.Recv.List) + ") "
|
|
2380
|
+
}
|
|
2381
|
+
scope += formatFuncType(d.Type)
|
|
2382
|
+
return "func " + scope
|
|
2383
|
+
}
|
|
2384
|
+
|
|
2385
|
+
func formatFuncType(f *ast.FuncType) string {
|
|
2386
|
+
params := formatFieldList(f.Params.List)
|
|
2387
|
+
results := ""
|
|
2388
|
+
if f.Results != nil {
|
|
2389
|
+
results = " -> " + formatFieldList(f.Results.List)
|
|
2390
|
+
}
|
|
2391
|
+
return params + results
|
|
2392
|
+
}
|
|
2393
|
+
|
|
2394
|
+
func formatFieldList(fields []*ast.Field) string {
|
|
2395
|
+
if len(fields) == 0 {
|
|
2396
|
+
return "()"
|
|
2397
|
+
}
|
|
2398
|
+
names := make([]string, 0, len(fields))
|
|
2399
|
+
for _, f := range fields {
|
|
2400
|
+
name := ""
|
|
2401
|
+
if len(f.Names) > 0 {
|
|
2402
|
+
name = f.Names[0].Name
|
|
2403
|
+
}
|
|
2404
|
+
t := formatType(f.Type)
|
|
2405
|
+
if name != "" {
|
|
2406
|
+
names = append(names, name+" "+t)
|
|
2407
|
+
} else {
|
|
2408
|
+
names = append(names, t)
|
|
2409
|
+
}
|
|
2410
|
+
}
|
|
2411
|
+
return "(" + strings.Join(names, ", ") + ")"
|
|
2412
|
+
}
|
|
2413
|
+
|
|
2414
|
+
func formatFields(fields []*ast.Field) string {
|
|
2415
|
+
lines := make([]string, 0)
|
|
2416
|
+
for _, f := range fields {
|
|
2417
|
+
name := ""
|
|
2418
|
+
if len(f.Names) > 0 {
|
|
2419
|
+
name = f.Names[0].Name
|
|
2420
|
+
}
|
|
2421
|
+
t := formatType(f.Type)
|
|
2422
|
+
if name != "" {
|
|
2423
|
+
lines = append(lines, name+" "+t)
|
|
2424
|
+
} else {
|
|
2425
|
+
lines = append(lines, t)
|
|
2426
|
+
}
|
|
2427
|
+
}
|
|
2428
|
+
return strings.Join(lines, "; ")
|
|
2429
|
+
}
|
|
2430
|
+
|
|
2431
|
+
func formatMethods(fields []*ast.Field) string {
|
|
2432
|
+
return formatFields(fields)
|
|
2433
|
+
}
|
|
1660
2434
|
|
|
1661
|
-
|
|
1662
|
-
|
|
1663
|
-
|
|
1664
|
-
|
|
1665
|
-
|
|
1666
|
-
|
|
1667
|
-
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
1675
|
-
"module": alias.name,
|
|
1676
|
-
})
|
|
1677
|
-
elif isinstance(node, ast.ImportFrom):
|
|
1678
|
-
# PEP 328: node.level is the number of leading dots. Preserving them is
|
|
1679
|
-
# what lets the resolver walk up from the importing file's package \u2014
|
|
1680
|
-
# dropping them made \`from .foo import X\` indistinguishable from an
|
|
1681
|
-
# absolute \`foo\`.
|
|
1682
|
-
module = ("." * (node.level or 0)) + (node.module or "")
|
|
1683
|
-
for alias in node.names:
|
|
1684
|
-
refs.append({
|
|
1685
|
-
"toName": alias.name,
|
|
1686
|
-
"callType": "import",
|
|
1687
|
-
"line": node.lineno,
|
|
1688
|
-
"module": module,
|
|
1689
|
-
})
|
|
1690
|
-
elif isinstance(node, ast.ClassDef):
|
|
1691
|
-
for base in node.bases:
|
|
1692
|
-
name = leaf_name(base)
|
|
1693
|
-
if name:
|
|
1694
|
-
refs.append({"toName": name, "callType": "inherit", "line": node.lineno})
|
|
2435
|
+
func formatTypeParams(tp *ast.FieldList) string {
|
|
2436
|
+
if tp == nil || len(tp.List) == 0 {
|
|
2437
|
+
return ""
|
|
2438
|
+
}
|
|
2439
|
+
params := make([]string, len(tp.List))
|
|
2440
|
+
for i, p := range tp.List {
|
|
2441
|
+
if len(p.Names) > 0 {
|
|
2442
|
+
params[i] = p.Names[0].Name
|
|
2443
|
+
} else {
|
|
2444
|
+
params[i] = "T"
|
|
2445
|
+
}
|
|
2446
|
+
}
|
|
2447
|
+
return "[" + strings.Join(params, ", ") + "]"
|
|
2448
|
+
}
|
|
1695
2449
|
|
|
1696
|
-
|
|
2450
|
+
func formatType(t ast.Expr) string {
|
|
2451
|
+
if t == nil {
|
|
2452
|
+
return "?"
|
|
2453
|
+
}
|
|
2454
|
+
switch v := t.(type) {
|
|
2455
|
+
case *ast.Ident:
|
|
2456
|
+
return v.Name
|
|
2457
|
+
case *ast.SelectorExpr:
|
|
2458
|
+
return formatType(v.X) + "." + v.Sel.Name
|
|
2459
|
+
case *ast.StarExpr:
|
|
2460
|
+
return "*" + formatType(v.X)
|
|
2461
|
+
case *ast.ArrayType:
|
|
2462
|
+
if v.Len == nil {
|
|
2463
|
+
return "[]" + formatType(v.Elt)
|
|
2464
|
+
}
|
|
2465
|
+
return "[...]" + formatType(v.Elt)
|
|
2466
|
+
case *ast.MapType:
|
|
2467
|
+
return "map[" + formatType(v.Key) + "]" + formatType(v.Value)
|
|
2468
|
+
case *ast.InterfaceType:
|
|
2469
|
+
return "interface{}"
|
|
2470
|
+
case *ast.StructType:
|
|
2471
|
+
return "struct{}"
|
|
2472
|
+
case *ast.FuncType:
|
|
2473
|
+
return formatFuncType(v)
|
|
2474
|
+
case *ast.ChanType:
|
|
2475
|
+
return "chan " + formatType(v.Value)
|
|
2476
|
+
case *ast.BasicLit:
|
|
2477
|
+
return v.Value
|
|
2478
|
+
case *ast.IndexExpr:
|
|
2479
|
+
// Generic instantiation with one type arg, e.g. Logger[int].
|
|
2480
|
+
return formatType(v.X) + "[" + formatType(v.Index) + "]"
|
|
2481
|
+
case *ast.IndexListExpr:
|
|
2482
|
+
// Generic instantiation with multiple type args, e.g. Map[K, V].
|
|
2483
|
+
args := make([]string, len(v.Indices))
|
|
2484
|
+
for i, idx := range v.Indices {
|
|
2485
|
+
args[i] = formatType(idx)
|
|
2486
|
+
}
|
|
2487
|
+
return formatType(v.X) + "[" + strings.Join(args, ", ") + "]"
|
|
2488
|
+
default:
|
|
2489
|
+
return "?"
|
|
2490
|
+
}
|
|
2491
|
+
}
|
|
1697
2492
|
`;
|
|
1698
|
-
|
|
2493
|
+
_cachedGoScriptPath = null;
|
|
1699
2494
|
}
|
|
1700
2495
|
});
|
|
1701
2496
|
|
|
@@ -1791,7 +2586,7 @@ __export(json_parser_exports, {
|
|
|
1791
2586
|
parseSymbols: () => parseSymbols6
|
|
1792
2587
|
});
|
|
1793
2588
|
import { expectDefined as expectDefined2 } from "@wrongstack/core/utils";
|
|
1794
|
-
import * as
|
|
2589
|
+
import * as path9 from "node:path";
|
|
1795
2590
|
function parseSymbols6(opts) {
|
|
1796
2591
|
const { file, content, lang } = opts;
|
|
1797
2592
|
try {
|
|
@@ -1803,7 +2598,7 @@ function parseSymbols6(opts) {
|
|
|
1803
2598
|
function regexParse2(opts) {
|
|
1804
2599
|
const { file, content, lang } = opts;
|
|
1805
2600
|
const symbols = [];
|
|
1806
|
-
const basename4 =
|
|
2601
|
+
const basename4 = path9.basename(file).toLowerCase();
|
|
1807
2602
|
const isPackageJson = basename4 === "package.json";
|
|
1808
2603
|
const isTsconfig = basename4 === "tsconfig.json" || basename4 === "tsconfig.build.json";
|
|
1809
2604
|
const isJsonSchema = content.includes("$schema") || content.includes("$id") || content.includes("$ref");
|
|
@@ -1829,11 +2624,11 @@ function regexParse2(opts) {
|
|
|
1829
2624
|
const line = lineFromOffset(offset);
|
|
1830
2625
|
symbols.push(
|
|
1831
2626
|
makeSymbol({
|
|
1832
|
-
name:
|
|
2627
|
+
name: path9.basename(file),
|
|
1833
2628
|
kind: "object",
|
|
1834
2629
|
line,
|
|
1835
2630
|
col: 0,
|
|
1836
|
-
signature: `"${
|
|
2631
|
+
signature: `"${path9.basename(file)}" = { ... }`,
|
|
1837
2632
|
file,
|
|
1838
2633
|
lang
|
|
1839
2634
|
})
|
|
@@ -2161,6 +2956,106 @@ var init_yaml_parser = __esm({
|
|
|
2161
2956
|
});
|
|
2162
2957
|
|
|
2163
2958
|
// src/codebase-index/tree-sitter/queries.ts
|
|
2959
|
+
function parseGroupedUse(text) {
|
|
2960
|
+
const open = text.indexOf("{");
|
|
2961
|
+
const close = text.lastIndexOf("}");
|
|
2962
|
+
if (open < 0 || close <= open) return null;
|
|
2963
|
+
const prefix = text.slice(0, open).replace(/[\\/]+$/, "");
|
|
2964
|
+
const out = [];
|
|
2965
|
+
for (const rawMember of text.slice(open + 1, close).split(",")) {
|
|
2966
|
+
let member = rawMember.trim();
|
|
2967
|
+
if (!member) continue;
|
|
2968
|
+
member = member.replace(
|
|
2969
|
+
/^(static|final|type|class|struct|enum|protocol|var|func|let|typealias|function|const)\s+/i,
|
|
2970
|
+
""
|
|
2971
|
+
).trim();
|
|
2972
|
+
if (!member) continue;
|
|
2973
|
+
const aliasMatch = /\s+as\s+([A-Za-z_]\w*)\s*$/i.exec(member);
|
|
2974
|
+
if (aliasMatch) member = member.slice(0, aliasMatch.index).trim();
|
|
2975
|
+
if (!member) continue;
|
|
2976
|
+
const module = prefix ? `${prefix}\\${member}` : member;
|
|
2977
|
+
const toName = member.split(/[\\/]/).filter(Boolean).pop();
|
|
2978
|
+
if (toName) out.push({ toName, callType: "import", module });
|
|
2979
|
+
}
|
|
2980
|
+
return out.length ? out : null;
|
|
2981
|
+
}
|
|
2982
|
+
function importFromText(prefixes) {
|
|
2983
|
+
return (node) => {
|
|
2984
|
+
let text = node.text.replace(/\s+/g, " ").trim();
|
|
2985
|
+
for (const prefix of prefixes) {
|
|
2986
|
+
if (text.startsWith(prefix)) text = text.slice(prefix.length).trim();
|
|
2987
|
+
}
|
|
2988
|
+
text = text.replace(
|
|
2989
|
+
/^(static|final|type|class|struct|enum|protocol|var|func|let|typealias|function|const)\s+/i,
|
|
2990
|
+
""
|
|
2991
|
+
).trim();
|
|
2992
|
+
if (text.includes("{")) return parseGroupedUse(text);
|
|
2993
|
+
if (text.includes(",") && !text.includes("<") && !text.includes("=")) {
|
|
2994
|
+
const out = [];
|
|
2995
|
+
for (const clause of text.split(",")) {
|
|
2996
|
+
const one = oneImportClause(clause.trim());
|
|
2997
|
+
if (one) out.push(one);
|
|
2998
|
+
}
|
|
2999
|
+
return out.length ? out : null;
|
|
3000
|
+
}
|
|
3001
|
+
const single = oneImportClause(text);
|
|
3002
|
+
return single ? [single] : null;
|
|
3003
|
+
};
|
|
3004
|
+
}
|
|
3005
|
+
function oneImportClause(rawClause) {
|
|
3006
|
+
let text = rawClause;
|
|
3007
|
+
text = text.replace(
|
|
3008
|
+
/^(static|final|type|class|struct|enum|protocol|var|func|let|typealias|function|const)\s+/i,
|
|
3009
|
+
""
|
|
3010
|
+
).trim();
|
|
3011
|
+
text = text.replace(/[;}]+$/g, "").trim();
|
|
3012
|
+
const aliasMatch = /\s+as\s+([A-Za-z_]\w*)\s*$/i.exec(text);
|
|
3013
|
+
if (aliasMatch) text = text.slice(0, aliasMatch.index).trim();
|
|
3014
|
+
const eqMatch = /^([A-Za-z_]\w*)\s*=\s*(.+)$/.exec(text);
|
|
3015
|
+
if (eqMatch) text = eqMatch[2].trim();
|
|
3016
|
+
if (!text) return null;
|
|
3017
|
+
if (text.endsWith("*")) text = text.slice(0, -1).replace(/[.]$/, "");
|
|
3018
|
+
if (!text) return null;
|
|
3019
|
+
const module = text;
|
|
3020
|
+
const toName = module.split(/[.\\/]/).filter(Boolean).pop()?.replace(/<.*>$/s, "");
|
|
3021
|
+
if (!toName) return null;
|
|
3022
|
+
return { toName, callType: "import", module };
|
|
3023
|
+
}
|
|
3024
|
+
function heritageLeaf(node, depth) {
|
|
3025
|
+
if (depth > 6) return null;
|
|
3026
|
+
const named = node.childForFieldName("name");
|
|
3027
|
+
if (named) {
|
|
3028
|
+
if (named.type === "scoped_type_identifier" || named.type === "qualified_name" || named.type === "scope_resolution" || named.type === "user_type") {
|
|
3029
|
+
return heritageLeaf(named, depth + 1);
|
|
3030
|
+
}
|
|
3031
|
+
return named.text;
|
|
3032
|
+
}
|
|
3033
|
+
const children = [];
|
|
3034
|
+
for (let i = 0; i < node.namedChildCount; i++) {
|
|
3035
|
+
const c = node.namedChild(i);
|
|
3036
|
+
if (c) children.push(c);
|
|
3037
|
+
}
|
|
3038
|
+
for (let i = children.length - 1; i >= 0; i--) {
|
|
3039
|
+
const c = children[i];
|
|
3040
|
+
if (c.type === "type_arguments" || c.type === "type_argument_list" || // cpp: (template_type arguments: (template_argument_list …)) — the
|
|
3041
|
+
// descriptor's type_identifier inside it is never the declared base.
|
|
3042
|
+
c.type === "template_argument_list" || c.type === "type_parameter_list" || c.type === "type_projection" || c.type === "value_arguments") {
|
|
3043
|
+
continue;
|
|
3044
|
+
}
|
|
3045
|
+
if (c.type === "type_identifier" || c.type === "identifier" || c.type === "constant" || c.type === "name") {
|
|
3046
|
+
return c.text;
|
|
3047
|
+
}
|
|
3048
|
+
if (c.type === "scoped_type_identifier" || c.type === "qualified_name" || c.type === "scope_resolution" || c.type === "user_type") {
|
|
3049
|
+
return heritageLeaf(c, depth + 1);
|
|
3050
|
+
}
|
|
3051
|
+
}
|
|
3052
|
+
return leafSegment(
|
|
3053
|
+
node.text.replace(/\\/g, ".").replace(/::/g, ".").replace(/<[^<>]*>$/, "")
|
|
3054
|
+
);
|
|
3055
|
+
}
|
|
3056
|
+
function leafSegment(text) {
|
|
3057
|
+
return text.split(".").filter(Boolean).pop() ?? text;
|
|
3058
|
+
}
|
|
2164
3059
|
function getQueries(lang) {
|
|
2165
3060
|
return LANG_QUERIES[lang] ?? DEFAULT_QUERIES;
|
|
2166
3061
|
}
|
|
@@ -2172,10 +3067,127 @@ function readFirstString(node) {
|
|
|
2172
3067
|
const child = node.namedChild(0);
|
|
2173
3068
|
return child ? readFirstString(child) : null;
|
|
2174
3069
|
}
|
|
2175
|
-
var DEFAULT_QUERIES, LANG_QUERIES;
|
|
3070
|
+
var heritageExtractor, cCallExtractor, rubyCallExtractor, firstIdentifierCallExtractor, cIncludeExtractor, phpConstructorExtractor, DEFAULT_QUERIES, LANG_QUERIES;
|
|
2176
3071
|
var init_queries = __esm({
|
|
2177
3072
|
"src/codebase-index/tree-sitter/queries.ts"() {
|
|
2178
3073
|
"use strict";
|
|
3074
|
+
heritageExtractor = (node) => {
|
|
3075
|
+
const out = [];
|
|
3076
|
+
const SKIP_SUBTREES = /* @__PURE__ */ new Set([
|
|
3077
|
+
"type_arguments",
|
|
3078
|
+
"type_argument_list",
|
|
3079
|
+
// tree-sitter-cpp names its argument subtree template_argument_list —
|
|
3080
|
+
// verified AST: (base_class_clause (template_type name:
|
|
3081
|
+
// (type_identifier) arguments: (template_argument_list
|
|
3082
|
+
// (type_descriptor type: (type_identifier))))). Without this entry
|
|
3083
|
+
// `class D : Base<Foo>` recurses into the descriptor and emits Foo as a
|
|
3084
|
+
// phantom inherit ref.
|
|
3085
|
+
"template_argument_list",
|
|
3086
|
+
"type_parameter_list",
|
|
3087
|
+
"type_projection",
|
|
3088
|
+
"value_arguments"
|
|
3089
|
+
]);
|
|
3090
|
+
const collect = (current, depth) => {
|
|
3091
|
+
if (depth > 4) return;
|
|
3092
|
+
for (let i = 0; i < current.namedChildCount; i++) {
|
|
3093
|
+
const child = current.namedChild(i);
|
|
3094
|
+
if (!child) continue;
|
|
3095
|
+
if (SKIP_SUBTREES.has(child.type)) continue;
|
|
3096
|
+
if (child.type === "type_identifier" || child.type === "identifier" || child.type === "named_type" || child.type === "type" || // PHP heritage carries `name`; Ruby a `constant`.
|
|
3097
|
+
child.type === "constant" || child.type === "name") {
|
|
3098
|
+
const name = child.type === "named_type" ? leafSegment(child.text) : child.text;
|
|
3099
|
+
if (name) out.push({ toName: name });
|
|
3100
|
+
continue;
|
|
3101
|
+
}
|
|
3102
|
+
if (child.type === "generic_type" || child.type === "generic_name") {
|
|
3103
|
+
for (let j = 0; j < child.namedChildCount; j++) {
|
|
3104
|
+
const inner = child.namedChild(j);
|
|
3105
|
+
if (inner && !SKIP_SUBTREES.has(inner.type) && (inner.type === "type_identifier" || inner.type === "identifier" || inner.type === "name")) {
|
|
3106
|
+
out.push({ toName: inner.text });
|
|
3107
|
+
break;
|
|
3108
|
+
}
|
|
3109
|
+
}
|
|
3110
|
+
continue;
|
|
3111
|
+
}
|
|
3112
|
+
if (child.type === "qualified_name" || child.type === "scoped_type_identifier" || child.type === "user_type" || child.type === "scope_resolution") {
|
|
3113
|
+
const leaf = heritageLeaf(child, 0);
|
|
3114
|
+
if (leaf) out.push({ toName: leaf });
|
|
3115
|
+
continue;
|
|
3116
|
+
}
|
|
3117
|
+
collect(child, depth + 1);
|
|
3118
|
+
}
|
|
3119
|
+
};
|
|
3120
|
+
collect(node, 0);
|
|
3121
|
+
return out;
|
|
3122
|
+
};
|
|
3123
|
+
cCallExtractor = (node) => {
|
|
3124
|
+
const fn = node.childForFieldName("function");
|
|
3125
|
+
if (!fn) return null;
|
|
3126
|
+
if (fn.type === "field_expression") {
|
|
3127
|
+
const field = fn.childForFieldName("field");
|
|
3128
|
+
if (field) return [{ toName: field.text, callType: "call" }];
|
|
3129
|
+
const seg = fn.text.split("->").filter(Boolean).pop();
|
|
3130
|
+
if (seg) return [{ toName: leafSegment(seg.split(".")[0] ?? seg), callType: "call" }];
|
|
3131
|
+
return null;
|
|
3132
|
+
}
|
|
3133
|
+
if (fn.type === "qualified_identifier") {
|
|
3134
|
+
const name = fn.childForFieldName("name");
|
|
3135
|
+
if (name) return [{ toName: name.text, callType: "call" }];
|
|
3136
|
+
const seg = fn.text.split("::").filter(Boolean).pop();
|
|
3137
|
+
if (seg) return [{ toName: seg.split(/[<(]/)[0].trim(), callType: "call" }];
|
|
3138
|
+
return null;
|
|
3139
|
+
}
|
|
3140
|
+
return [{ toName: fn.text.split(/[<(]/)[0].trim(), callType: "call" }];
|
|
3141
|
+
};
|
|
3142
|
+
rubyCallExtractor = (node) => {
|
|
3143
|
+
const emissions = [];
|
|
3144
|
+
const method = node.childForFieldName("method");
|
|
3145
|
+
if (method) {
|
|
3146
|
+
const name = method.text;
|
|
3147
|
+
if (name && !name.includes(" ")) emissions.push({ toName: name, callType: "call" });
|
|
3148
|
+
if (name === "require" || name === "require_relative") {
|
|
3149
|
+
const args = node.childForFieldName("arguments");
|
|
3150
|
+
const first = args?.namedChild(0);
|
|
3151
|
+
if (first) {
|
|
3152
|
+
const raw = first.text.replace(/^['"]|['"]$/g, "");
|
|
3153
|
+
const toName = raw.split("/").filter(Boolean).pop();
|
|
3154
|
+
if (toName) emissions.push({ toName, callType: "import", module: raw });
|
|
3155
|
+
}
|
|
3156
|
+
}
|
|
3157
|
+
}
|
|
3158
|
+
return emissions;
|
|
3159
|
+
};
|
|
3160
|
+
firstIdentifierCallExtractor = (node) => {
|
|
3161
|
+
for (let i = 0; i < node.namedChildCount; i++) {
|
|
3162
|
+
const child = node.namedChild(i);
|
|
3163
|
+
if (child && (child.type === "simple_identifier" || child.type === "identifier")) {
|
|
3164
|
+
return [{ toName: child.text, callType: "call" }];
|
|
3165
|
+
}
|
|
3166
|
+
}
|
|
3167
|
+
const first = node.namedChild(0);
|
|
3168
|
+
if (!first) return null;
|
|
3169
|
+
const leaf = leafSegment(first.text.split(/[<(]/)[0] ?? first.text);
|
|
3170
|
+
if (!leaf) return null;
|
|
3171
|
+
return [{ toName: leaf, callType: "call" }];
|
|
3172
|
+
};
|
|
3173
|
+
cIncludeExtractor = (node) => {
|
|
3174
|
+
const raw = node.text.replace(/^#\s*include\s*/i, "").trim();
|
|
3175
|
+
const module = raw.replace(/^["'<]|["'>]$/g, "");
|
|
3176
|
+
if (!module) return null;
|
|
3177
|
+
const toName = module.split("/").pop()?.replace(/\.h$/, "");
|
|
3178
|
+
if (!toName) return null;
|
|
3179
|
+
return [{ toName, callType: "import", module }];
|
|
3180
|
+
};
|
|
3181
|
+
phpConstructorExtractor = (node) => {
|
|
3182
|
+
for (let i = 0; i < node.namedChildCount; i++) {
|
|
3183
|
+
const child = node.namedChild(i);
|
|
3184
|
+
if (child && (child.type === "qualified_name" || child.type === "name")) {
|
|
3185
|
+
const leaf = child.text.split(/[\\]/).filter(Boolean).pop();
|
|
3186
|
+
if (leaf) return [{ toName: leaf, callType: "call" }];
|
|
3187
|
+
}
|
|
3188
|
+
}
|
|
3189
|
+
return null;
|
|
3190
|
+
};
|
|
2179
3191
|
DEFAULT_QUERIES = {
|
|
2180
3192
|
declKinds: {}
|
|
2181
3193
|
};
|
|
@@ -2208,7 +3220,13 @@ var init_queries = __esm({
|
|
|
2208
3220
|
"struct_specifier",
|
|
2209
3221
|
"union_specifier",
|
|
2210
3222
|
"enum_specifier"
|
|
2211
|
-
])
|
|
3223
|
+
]),
|
|
3224
|
+
refRules: {
|
|
3225
|
+
// `obj->run()` and `Cls::stat()` carry structured function fields —
|
|
3226
|
+
// cCallExtractor handles all three AST shapes.
|
|
3227
|
+
call_expression: { callType: "call", nameExtractor: cCallExtractor },
|
|
3228
|
+
preproc_include: { callType: "import", nameExtractor: cIncludeExtractor }
|
|
3229
|
+
}
|
|
2212
3230
|
},
|
|
2213
3231
|
cpp: {
|
|
2214
3232
|
declKinds: {
|
|
@@ -2239,7 +3257,13 @@ var init_queries = __esm({
|
|
|
2239
3257
|
"union_specifier",
|
|
2240
3258
|
"enum_specifier",
|
|
2241
3259
|
"namespace_definition"
|
|
2242
|
-
])
|
|
3260
|
+
]),
|
|
3261
|
+
refRules: {
|
|
3262
|
+
call_expression: { callType: "call", nameExtractor: cCallExtractor },
|
|
3263
|
+
preproc_include: { callType: "import", nameExtractor: cIncludeExtractor },
|
|
3264
|
+
// `class Foo : public Bar, private Baz` — the base-class clause.
|
|
3265
|
+
base_class_clause: { callType: "inherit", nameExtractor: heritageExtractor }
|
|
3266
|
+
}
|
|
2243
3267
|
},
|
|
2244
3268
|
java: {
|
|
2245
3269
|
declKinds: {
|
|
@@ -2274,7 +3298,22 @@ var init_queries = __esm({
|
|
|
2274
3298
|
"interface_declaration",
|
|
2275
3299
|
"enum_declaration",
|
|
2276
3300
|
"record_declaration"
|
|
2277
|
-
])
|
|
3301
|
+
]),
|
|
3302
|
+
refRules: {
|
|
3303
|
+
method_invocation: { callType: "call", field: "name" },
|
|
3304
|
+
object_creation_expression: { callType: "call", field: "type" },
|
|
3305
|
+
// Verified AST: `superclass: (superclass (type_identifier))` and
|
|
3306
|
+
// `interfaces: (super_interfaces (type_list ...))` — no underscores.
|
|
3307
|
+
superclass: { callType: "inherit", nameExtractor: heritageExtractor },
|
|
3308
|
+
super_interfaces: {
|
|
3309
|
+
callType: "implement",
|
|
3310
|
+
nameExtractor: heritageExtractor
|
|
3311
|
+
},
|
|
3312
|
+
import_declaration: {
|
|
3313
|
+
callType: "import",
|
|
3314
|
+
nameExtractor: importFromText(["import "])
|
|
3315
|
+
}
|
|
3316
|
+
}
|
|
2278
3317
|
},
|
|
2279
3318
|
csharp: {
|
|
2280
3319
|
// C# 10+ `namespace Foo.Bar;` produces this node type. The legacy block
|
|
@@ -2312,7 +3351,18 @@ var init_queries = __esm({
|
|
|
2312
3351
|
"struct_declaration",
|
|
2313
3352
|
"enum_declaration",
|
|
2314
3353
|
"record_declaration"
|
|
2315
|
-
])
|
|
3354
|
+
]),
|
|
3355
|
+
refRules: {
|
|
3356
|
+
// Verified AST: `invocation_expression function: (identifier)` — the
|
|
3357
|
+
// callee field is `function` (C-style), not `name`.
|
|
3358
|
+
invocation_expression: { callType: "call", field: "function" },
|
|
3359
|
+
object_creation_expression: { callType: "call", field: "type" },
|
|
3360
|
+
base_list: { callType: "inherit", nameExtractor: heritageExtractor },
|
|
3361
|
+
using_directive: {
|
|
3362
|
+
callType: "import",
|
|
3363
|
+
nameExtractor: importFromText(["using "])
|
|
3364
|
+
}
|
|
3365
|
+
}
|
|
2316
3366
|
},
|
|
2317
3367
|
php: {
|
|
2318
3368
|
declKinds: {
|
|
@@ -2331,7 +3381,7 @@ var init_queries = __esm({
|
|
|
2331
3381
|
interface_declaration: "name",
|
|
2332
3382
|
trait_declaration: "name",
|
|
2333
3383
|
enum_declaration: "name",
|
|
2334
|
-
|
|
3384
|
+
namespace_definition: "name"
|
|
2335
3385
|
},
|
|
2336
3386
|
scopeNodes: /* @__PURE__ */ new Set([
|
|
2337
3387
|
"program",
|
|
@@ -2340,7 +3390,24 @@ var init_queries = __esm({
|
|
|
2340
3390
|
"interface_declaration",
|
|
2341
3391
|
"trait_declaration",
|
|
2342
3392
|
"enum_declaration"
|
|
2343
|
-
])
|
|
3393
|
+
]),
|
|
3394
|
+
refRules: {
|
|
3395
|
+
function_call_expression: { callType: "call", field: "function" },
|
|
3396
|
+
// Verified AST: `new App\Model\User()` carries a BARE qualified_name
|
|
3397
|
+
// child (no `name:` field), so the field default never fires.
|
|
3398
|
+
object_creation_expression: { callType: "call", nameExtractor: phpConstructorExtractor },
|
|
3399
|
+
base_clause: { callType: "inherit", nameExtractor: heritageExtractor },
|
|
3400
|
+
class_interface_clause: {
|
|
3401
|
+
callType: "implement",
|
|
3402
|
+
nameExtractor: heritageExtractor
|
|
3403
|
+
},
|
|
3404
|
+
// Verified AST: `namespace_use_declaration (namespace_use_clause
|
|
3405
|
+
// (qualified_name ...))` — not `use_declaration`.
|
|
3406
|
+
namespace_use_declaration: {
|
|
3407
|
+
callType: "import",
|
|
3408
|
+
nameExtractor: importFromText(["use "])
|
|
3409
|
+
}
|
|
3410
|
+
}
|
|
2344
3411
|
},
|
|
2345
3412
|
// ─── Scripting / mobile ────────────────────────────────────────────────────
|
|
2346
3413
|
ruby: {
|
|
@@ -2358,7 +3425,13 @@ var init_queries = __esm({
|
|
|
2358
3425
|
module: "name",
|
|
2359
3426
|
constant: "name"
|
|
2360
3427
|
},
|
|
2361
|
-
scopeNodes: /* @__PURE__ */ new Set(["program", "class", "module", "singleton_method", "method"])
|
|
3428
|
+
scopeNodes: /* @__PURE__ */ new Set(["program", "class", "module", "singleton_method", "method"]),
|
|
3429
|
+
refRules: {
|
|
3430
|
+
// `call` covers both `foo(...)` and `obj.foo(...)` — the extractor
|
|
3431
|
+
// records the method leaf, plus `require`/`require_relative` imports.
|
|
3432
|
+
call: { callType: "call", nameExtractor: rubyCallExtractor },
|
|
3433
|
+
superclass: { callType: "inherit", nameExtractor: heritageExtractor }
|
|
3434
|
+
}
|
|
2362
3435
|
},
|
|
2363
3436
|
swift: {
|
|
2364
3437
|
declKinds: {
|
|
@@ -2391,7 +3464,18 @@ var init_queries = __esm({
|
|
|
2391
3464
|
"protocol_declaration",
|
|
2392
3465
|
"actor_declaration",
|
|
2393
3466
|
"extension_declaration"
|
|
2394
|
-
])
|
|
3467
|
+
]),
|
|
3468
|
+
refRules: {
|
|
3469
|
+
// Verified AST: `call_expression (simple_identifier) (call_suffix …)` —
|
|
3470
|
+
// the callee is a bare first child, no field name.
|
|
3471
|
+
call_expression: { callType: "call", nameExtractor: firstIdentifierCallExtractor },
|
|
3472
|
+
// Verified AST: `inheritance_specifier inherits_from: (user_type …)`.
|
|
3473
|
+
inheritance_specifier: { callType: "inherit", nameExtractor: heritageExtractor },
|
|
3474
|
+
import_declaration: {
|
|
3475
|
+
callType: "import",
|
|
3476
|
+
nameExtractor: importFromText(["import ", "import type ", "@testable import "])
|
|
3477
|
+
}
|
|
3478
|
+
}
|
|
2395
3479
|
},
|
|
2396
3480
|
kotlin: {
|
|
2397
3481
|
declKinds: {
|
|
@@ -2416,7 +3500,17 @@ var init_queries = __esm({
|
|
|
2416
3500
|
"object_declaration",
|
|
2417
3501
|
"interface_declaration",
|
|
2418
3502
|
"function_declaration"
|
|
2419
|
-
])
|
|
3503
|
+
]),
|
|
3504
|
+
refRules: {
|
|
3505
|
+
call_expression: { callType: "call", nameExtractor: firstIdentifierCallExtractor },
|
|
3506
|
+
// Verified AST: `delegation_specifier (user_type (type_identifier))` —
|
|
3507
|
+
// the `: Handler` / `: Base()` clause.
|
|
3508
|
+
delegation_specifier: { callType: "inherit", nameExtractor: heritageExtractor },
|
|
3509
|
+
import_header: {
|
|
3510
|
+
callType: "import",
|
|
3511
|
+
nameExtractor: importFromText(["import "])
|
|
3512
|
+
}
|
|
3513
|
+
}
|
|
2420
3514
|
},
|
|
2421
3515
|
elixir: {
|
|
2422
3516
|
declKinds: {
|
|
@@ -2488,7 +3582,38 @@ function visitTree(tree, content, file, lang, queries) {
|
|
|
2488
3582
|
const boundedContent = content.length > TREE_SITTER_MAX_FILE_CHARS ? content.slice(0, TREE_SITTER_MAX_FILE_CHARS) : content;
|
|
2489
3583
|
const nlOffsets = newlineOffsets3(boundedContent);
|
|
2490
3584
|
const symbols = [];
|
|
3585
|
+
const refs = [];
|
|
3586
|
+
const seenRefs = /* @__PURE__ */ new Set();
|
|
2491
3587
|
const scopeStack = [];
|
|
3588
|
+
function emitRefsForNode(node, rule) {
|
|
3589
|
+
const emissions = rule.nameExtractor?.(node) ?? defaultRefTarget(node, rule);
|
|
3590
|
+
if (!emissions) return;
|
|
3591
|
+
const { line } = lineColAt2(nlOffsets, node.startIndex);
|
|
3592
|
+
for (const emission of emissions) {
|
|
3593
|
+
if (!emission.toName) continue;
|
|
3594
|
+
const callType = emission.callType ?? rule.callType;
|
|
3595
|
+
const key = `${emission.toName}:${callType}:${line}:${emission.module ?? ""}:${node.startIndex}`;
|
|
3596
|
+
if (seenRefs.has(key)) continue;
|
|
3597
|
+
seenRefs.add(key);
|
|
3598
|
+
refs.push({
|
|
3599
|
+
fromId: 0,
|
|
3600
|
+
// assignRefsToSymbols attaches owners after insertion
|
|
3601
|
+
toName: emission.toName.slice(0, 200),
|
|
3602
|
+
callType,
|
|
3603
|
+
line,
|
|
3604
|
+
lang,
|
|
3605
|
+
module: emission.module
|
|
3606
|
+
});
|
|
3607
|
+
}
|
|
3608
|
+
}
|
|
3609
|
+
function defaultRefTarget(node, rule) {
|
|
3610
|
+
if (!rule.field) return null;
|
|
3611
|
+
const field = node.childForFieldName(rule.field);
|
|
3612
|
+
if (!field) return null;
|
|
3613
|
+
const leaf = field.text.split(/[.:\\]/).filter(Boolean).pop()?.split(/[<(]/)[0];
|
|
3614
|
+
if (!leaf) return null;
|
|
3615
|
+
return [{ toName: leaf.trim() }];
|
|
3616
|
+
}
|
|
2492
3617
|
function visit(node, depth) {
|
|
2493
3618
|
if (symbols.length >= TREE_SITTER_MAX_SYMBOLS) return;
|
|
2494
3619
|
if (node.isMissing || node.isError) {
|
|
@@ -2507,6 +3632,8 @@ function visitTree(tree, content, file, lang, queries) {
|
|
|
2507
3632
|
);
|
|
2508
3633
|
if (emitted) symbols.push(emitted);
|
|
2509
3634
|
}
|
|
3635
|
+
const refRule = queries.refRules?.[node.type];
|
|
3636
|
+
if (refRule) emitRefsForNode(node, refRule);
|
|
2510
3637
|
}
|
|
2511
3638
|
const pushesScope = queries.scopeNodes?.has(node.type) ?? false;
|
|
2512
3639
|
const pushIdx = pushesScope ? pushScope(scopeStack, node, queries) : -1;
|
|
@@ -2524,7 +3651,7 @@ function visitTree(tree, content, file, lang, queries) {
|
|
|
2524
3651
|
if (pushIdx !== -1) scopeStack.pop();
|
|
2525
3652
|
}
|
|
2526
3653
|
visit(tree.rootNode, 0);
|
|
2527
|
-
return { symbols };
|
|
3654
|
+
return { symbols, refs };
|
|
2528
3655
|
}
|
|
2529
3656
|
function pushScope(scopeStack, node, queries) {
|
|
2530
3657
|
const name = extractName(node, queries);
|
|
@@ -2611,7 +3738,7 @@ __export(tree_sitter_parser_exports, {
|
|
|
2611
3738
|
parseSymbols: () => parseSymbols8,
|
|
2612
3739
|
parseTreeSitterAst: () => parseTreeSitterAst
|
|
2613
3740
|
});
|
|
2614
|
-
import * as
|
|
3741
|
+
import * as path10 from "node:path";
|
|
2615
3742
|
import { fileURLToPath } from "node:url";
|
|
2616
3743
|
function optInEnabled(env) {
|
|
2617
3744
|
return process.env[env] === "1" || process.env[env] === "true";
|
|
@@ -2634,7 +3761,7 @@ async function loadLanguage(lang) {
|
|
|
2634
3761
|
if (!grammarName) {
|
|
2635
3762
|
throw new Error(`tree-sitter: no grammar registered for lang "${lang}"`);
|
|
2636
3763
|
}
|
|
2637
|
-
const wasmPath =
|
|
3764
|
+
const wasmPath = path10.join(WASM_DIR, grammarName, `tree-sitter-${grammarName}.wasm`);
|
|
2638
3765
|
const { Language, init } = await getRuntime();
|
|
2639
3766
|
await init();
|
|
2640
3767
|
const languageObj = await Language.load(wasmPath);
|
|
@@ -2655,7 +3782,7 @@ function isTreeSitterSupported(lang) {
|
|
|
2655
3782
|
function getGrammarWasmPath(lang) {
|
|
2656
3783
|
const name = resolveGrammarName(lang);
|
|
2657
3784
|
if (!name) return void 0;
|
|
2658
|
-
return
|
|
3785
|
+
return path10.join(WASM_DIR, name, `tree-sitter-${name}.wasm`);
|
|
2659
3786
|
}
|
|
2660
3787
|
async function parseSymbols8(opts) {
|
|
2661
3788
|
const { file, content, lang } = opts;
|
|
@@ -2671,10 +3798,10 @@ async function parseSymbols8(opts) {
|
|
|
2671
3798
|
if (!tree) {
|
|
2672
3799
|
return { file, lang, symbols: [], mtimeMs: Date.now() };
|
|
2673
3800
|
}
|
|
2674
|
-
const { symbols } = visitTree(tree, content, file, lang, getQueries(lang));
|
|
3801
|
+
const { symbols, refs } = visitTree(tree, content, file, lang, getQueries(lang));
|
|
2675
3802
|
parser.delete();
|
|
2676
3803
|
tree.delete();
|
|
2677
|
-
return { file, lang, symbols, refs
|
|
3804
|
+
return { file, lang, symbols, refs, mtimeMs: Date.now() };
|
|
2678
3805
|
} catch {
|
|
2679
3806
|
return { file, lang, symbols: [], mtimeMs: Date.now() };
|
|
2680
3807
|
}
|
|
@@ -2707,7 +3834,7 @@ async function parseTreeSitterAst(opts) {
|
|
|
2707
3834
|
try {
|
|
2708
3835
|
const { Parser, Language, init } = await getRuntime();
|
|
2709
3836
|
await init();
|
|
2710
|
-
const wasmPath =
|
|
3837
|
+
const wasmPath = path10.join(WASM_DIR, grammar, `tree-sitter-${grammar}.wasm`);
|
|
2711
3838
|
const languageObj = await Language.load(wasmPath);
|
|
2712
3839
|
const parser = new Parser();
|
|
2713
3840
|
parser.setLanguage(languageObj);
|
|
@@ -2728,7 +3855,7 @@ var init_tree_sitter_parser = __esm({
|
|
|
2728
3855
|
init_queries();
|
|
2729
3856
|
init_visitor();
|
|
2730
3857
|
WASM_DIR = fileURLToPath(new URL("./wasm/", import.meta.url));
|
|
2731
|
-
RUNTIME_WASM =
|
|
3858
|
+
RUNTIME_WASM = path10.join(WASM_DIR, "tree-sitter-runtime.wasm");
|
|
2732
3859
|
LANG_TO_GRAMMAR = {
|
|
2733
3860
|
c: "c",
|
|
2734
3861
|
cpp: "cpp",
|
|
@@ -2752,6 +3879,146 @@ var init_tree_sitter_parser = __esm({
|
|
|
2752
3879
|
}
|
|
2753
3880
|
});
|
|
2754
3881
|
|
|
3882
|
+
// src/codebase-index/parser-dispatch.ts
|
|
3883
|
+
var parser_dispatch_exports = {};
|
|
3884
|
+
__export(parser_dispatch_exports, {
|
|
3885
|
+
parseFileContent: () => parseFileContent,
|
|
3886
|
+
parseFilesContent: () => parseFilesContent
|
|
3887
|
+
});
|
|
3888
|
+
async function parseFileContent(file, content, lang) {
|
|
3889
|
+
const parsed = await dispatch(file, content, lang);
|
|
3890
|
+
return withRelations(parsed, content, lang);
|
|
3891
|
+
}
|
|
3892
|
+
async function parseFilesContent(files) {
|
|
3893
|
+
if (files.length === 0) return [];
|
|
3894
|
+
const slots = files.map(() => ({ result: null }));
|
|
3895
|
+
const batchingEnabled = process.env["WRONGSTACK_TOOLCHAIN_BATCH"] !== "0";
|
|
3896
|
+
if (batchingEnabled) {
|
|
3897
|
+
const goFiles = [];
|
|
3898
|
+
const pyFiles = [];
|
|
3899
|
+
files.forEach((f, index) => {
|
|
3900
|
+
if (f.lang === "go") goFiles.push({ ...f, index });
|
|
3901
|
+
else if (f.lang === "py") pyFiles.push({ ...f, index });
|
|
3902
|
+
});
|
|
3903
|
+
if (goFiles.length > 0) {
|
|
3904
|
+
await applyBatchResults(slots, goFiles, (chunks) => runGoBatch(chunks), "go");
|
|
3905
|
+
}
|
|
3906
|
+
if (pyFiles.length > 0) {
|
|
3907
|
+
const pyBinary = await resolvePythonBinary();
|
|
3908
|
+
if (pyBinary) {
|
|
3909
|
+
await applyBatchResults(slots, pyFiles, (chunks) => runPyBatch(chunks, pyBinary), "py");
|
|
3910
|
+
}
|
|
3911
|
+
}
|
|
3912
|
+
}
|
|
3913
|
+
const jobs = [];
|
|
3914
|
+
for (let i = 0; i < files.length; i++) {
|
|
3915
|
+
if (slots[i].result !== null) continue;
|
|
3916
|
+
const { file, content, lang } = files[i];
|
|
3917
|
+
const slot = slots[i];
|
|
3918
|
+
jobs.push(
|
|
3919
|
+
(async () => {
|
|
3920
|
+
try {
|
|
3921
|
+
slot.result = await parseFileContent(file, content, lang);
|
|
3922
|
+
} catch (err) {
|
|
3923
|
+
slot.error = err instanceof Error ? err.message : String(err);
|
|
3924
|
+
}
|
|
3925
|
+
})()
|
|
3926
|
+
);
|
|
3927
|
+
}
|
|
3928
|
+
await Promise.all(jobs);
|
|
3929
|
+
return slots;
|
|
3930
|
+
}
|
|
3931
|
+
async function applyBatchResults(slots, batchFiles, runBatch, lang) {
|
|
3932
|
+
for (const chunk of chunkBatchFiles(batchFiles)) {
|
|
3933
|
+
let byFile = null;
|
|
3934
|
+
try {
|
|
3935
|
+
byFile = await runBatch(chunk);
|
|
3936
|
+
} catch {
|
|
3937
|
+
byFile = null;
|
|
3938
|
+
}
|
|
3939
|
+
if (!byFile) continue;
|
|
3940
|
+
for (const item of chunk) {
|
|
3941
|
+
const parsed = byFile.get(item.file);
|
|
3942
|
+
if (!parsed) continue;
|
|
3943
|
+
slots[item.index] = { result: withRelations(parsed, item.content, lang) };
|
|
3944
|
+
}
|
|
3945
|
+
}
|
|
3946
|
+
}
|
|
3947
|
+
async function dispatch(file, content, lang) {
|
|
3948
|
+
switch (lang) {
|
|
3949
|
+
case "ts":
|
|
3950
|
+
case "tsx":
|
|
3951
|
+
case "js":
|
|
3952
|
+
case "jsx": {
|
|
3953
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_ts_parser(), ts_parser_exports));
|
|
3954
|
+
return parseSymbols9({ file, content, lang });
|
|
3955
|
+
}
|
|
3956
|
+
case "go": {
|
|
3957
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_go_parser(), go_parser_exports));
|
|
3958
|
+
return parseSymbols9({ file, content, lang: "go" });
|
|
3959
|
+
}
|
|
3960
|
+
case "py": {
|
|
3961
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_py_parser(), py_parser_exports));
|
|
3962
|
+
return parseSymbols9({ file, content, lang: "py" });
|
|
3963
|
+
}
|
|
3964
|
+
case "rs": {
|
|
3965
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_rs_parser(), rs_parser_exports));
|
|
3966
|
+
return parseSymbols9({ file, content, lang: "rs" });
|
|
3967
|
+
}
|
|
3968
|
+
case "json": {
|
|
3969
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_json_parser(), json_parser_exports));
|
|
3970
|
+
return parseSymbols9({ file, content, lang: "json" });
|
|
3971
|
+
}
|
|
3972
|
+
case "yaml": {
|
|
3973
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_yaml_parser(), yaml_parser_exports));
|
|
3974
|
+
return parseSymbols9({ file, content, lang: "yaml" });
|
|
3975
|
+
}
|
|
3976
|
+
// Phase 1: ten languages now route through the Tree-Sitter WASM
|
|
3977
|
+
// universal parser (`tree-sitter-parser.ts`). The dispatch falls back to
|
|
3978
|
+
// the regex extractor in `generic-parser.ts` whenever WASM loading fails
|
|
3979
|
+
// or the parser returns zero symbols — preserving the indexable-file
|
|
3980
|
+
// contract that "missing a parser must never mean skipping the file".
|
|
3981
|
+
case "c":
|
|
3982
|
+
case "cpp":
|
|
3983
|
+
case "java":
|
|
3984
|
+
case "csharp":
|
|
3985
|
+
case "php":
|
|
3986
|
+
case "ruby":
|
|
3987
|
+
case "swift":
|
|
3988
|
+
case "kotlin":
|
|
3989
|
+
case "shell":
|
|
3990
|
+
case "elixir": {
|
|
3991
|
+
try {
|
|
3992
|
+
const { parseSymbols: parseSymbols10 } = await Promise.resolve().then(() => (init_tree_sitter_parser(), tree_sitter_parser_exports));
|
|
3993
|
+
const parsed = await parseSymbols10({ file, content, lang });
|
|
3994
|
+
if (parsed.symbols.length > 0) return parsed;
|
|
3995
|
+
} catch {
|
|
3996
|
+
}
|
|
3997
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_generic_parser(), generic_parser_exports));
|
|
3998
|
+
return parseSymbols9({ file, content, lang });
|
|
3999
|
+
}
|
|
4000
|
+
default: {
|
|
4001
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_generic_parser(), generic_parser_exports));
|
|
4002
|
+
return parseSymbols9({ file, content, lang });
|
|
4003
|
+
}
|
|
4004
|
+
}
|
|
4005
|
+
}
|
|
4006
|
+
function withRelations(parsed, content, lang) {
|
|
4007
|
+
let refs = parsed.refs ?? [];
|
|
4008
|
+
if (refs.length === 0 && hasImportPatterns(lang)) {
|
|
4009
|
+
refs = extractImports({ content, lang });
|
|
4010
|
+
}
|
|
4011
|
+
return { ...parsed, refs: refs.map((ref) => ref.lang ? ref : { ...ref, lang }) };
|
|
4012
|
+
}
|
|
4013
|
+
var init_parser_dispatch = __esm({
|
|
4014
|
+
"src/codebase-index/parser-dispatch.ts"() {
|
|
4015
|
+
"use strict";
|
|
4016
|
+
init_import_extractor();
|
|
4017
|
+
init_parser_batch();
|
|
4018
|
+
init_py_parser();
|
|
4019
|
+
}
|
|
4020
|
+
});
|
|
4021
|
+
|
|
2755
4022
|
// src/codebase-index/worker.ts
|
|
2756
4023
|
import { parentPort } from "node:worker_threads";
|
|
2757
4024
|
|
|
@@ -2759,9 +4026,9 @@ import { parentPort } from "node:worker_threads";
|
|
|
2759
4026
|
import { expectDefined as expectDefined5 } from "@wrongstack/core/utils";
|
|
2760
4027
|
import { execFile } from "node:child_process";
|
|
2761
4028
|
import { createHash } from "node:crypto";
|
|
2762
|
-
import * as
|
|
4029
|
+
import * as fs10 from "node:fs/promises";
|
|
2763
4030
|
import { availableParallelism } from "node:os";
|
|
2764
|
-
import * as
|
|
4031
|
+
import * as path13 from "node:path";
|
|
2765
4032
|
import {
|
|
2766
4033
|
DEFAULT_WALK_IGNORE_DIRS,
|
|
2767
4034
|
indexParallelBatchSize,
|
|
@@ -3505,207 +4772,20 @@ var ModuleResolver = class {
|
|
|
3505
4772
|
}
|
|
3506
4773
|
};
|
|
3507
4774
|
|
|
3508
|
-
// src/codebase-index/
|
|
3509
|
-
|
|
3510
|
-
var IMPORT_MAX_PER_FILE = 400;
|
|
3511
|
-
var DOTTED_IMPORT = [
|
|
3512
|
-
{ re: /^[ \t]*import\s+(?:static\s+)?([\w.]+(?:\.\*)?)\s*;?\s*$/gm }
|
|
3513
|
-
];
|
|
3514
|
-
var LANG_IMPORTS = {
|
|
3515
|
-
// Go and Python have real AST extractors; these patterns are the fallback for
|
|
3516
|
-
// machines with no Go toolchain or Python interpreter installed, where the
|
|
3517
|
-
// parser degrades to regex symbols and would otherwise contribute no edges.
|
|
3518
|
-
go: [
|
|
3519
|
-
{ re: /^[ \t]*import\s+(?:[\w.]+\s+)?"([^"]+)"/gm },
|
|
3520
|
-
// Grouped form: inside `import ( … )` each line is an optional alias plus a
|
|
3521
|
-
// quoted path. A stray match elsewhere resolves to no file and is dropped.
|
|
3522
|
-
{ re: /^[ \t]*(?:[A-Za-z_.]\w*\s+)?"([^"]+)"\s*$/gm }
|
|
3523
|
-
],
|
|
3524
|
-
py: [{ re: /^[ \t]*import\s+([\w.]+)/gm }, { re: /^[ \t]*from\s+([.\w]+)\s+import\b/gm }],
|
|
3525
|
-
rs: [
|
|
3526
|
-
// use a::b::C; | use a::b::{C, D}; → the path before any brace
|
|
3527
|
-
{ re: /^[ \t]*(?:pub\s+)?use\s+([\w:]+?)(?:::\{|\s*;|\s+as\b)/gm },
|
|
3528
|
-
// mod foo; (a declaration *and* a dependency on foo.rs / foo/mod.rs)
|
|
3529
|
-
{ re: /^[ \t]*(?:pub\s+)?mod\s+(\w+)\s*;/gm }
|
|
3530
|
-
],
|
|
3531
|
-
java: DOTTED_IMPORT,
|
|
3532
|
-
kotlin: DOTTED_IMPORT,
|
|
3533
|
-
scala: [{ re: /^[ \t]*import\s+([\w.]+(?:\.[_*])?)\s*$/gm }],
|
|
3534
|
-
csharp: [
|
|
3535
|
-
// using Foo.Bar; | using static Foo.Bar; | using Alias = Foo.Bar;
|
|
3536
|
-
{ re: /^[ \t]*global\s+using\s+(?:static\s+)?([\w.]+)\s*;/gm, name: "full" },
|
|
3537
|
-
{ re: /^[ \t]*using\s+(?:static\s+)?(?:\w+\s*=\s*)?([\w.]+)\s*;/gm, name: "full" }
|
|
3538
|
-
],
|
|
3539
|
-
// Quoted includes only: <stdio.h> is a system header with no indexed file.
|
|
3540
|
-
c: [{ re: /^[ \t]*#\s*include\s+"([^"]+)"/gm }],
|
|
3541
|
-
cpp: [{ re: /^[ \t]*#\s*include\s+"([^"]+)"/gm }],
|
|
3542
|
-
ruby: [{ re: /^[ \t]*(?:require|require_relative|load)\s*\(?\s*['"]([^'"]+)['"]/gm }],
|
|
3543
|
-
php: [
|
|
3544
|
-
// `use A\B\C` imports the class C, which is what the index has a symbol
|
|
3545
|
-
// for — the namespace symbol only covers the `A\B` prefix.
|
|
3546
|
-
{ re: /^[ \t]*use\s+(?:function\s+|const\s+)?([\w\\]+)/gm },
|
|
3547
|
-
{ re: /^[ \t]*(?:require|require_once|include|include_once)\s*\(?\s*['"]([^'"]+)['"]/gm }
|
|
3548
|
-
],
|
|
3549
|
-
swift: [{ re: /^[ \t]*import\s+(?:struct\s+|class\s+|func\s+)?([\w.]+)\s*$/gm }],
|
|
3550
|
-
dart: [{ re: /^[ \t]*(?:import|export|part)\s+['"]([^'"]+)['"]/gm }],
|
|
3551
|
-
lua: [{ re: /\brequire\s*\(?\s*['"]([^'"]+)['"]/g }],
|
|
3552
|
-
elixir: [{ re: /^[ \t]*(?:alias|import|require|use)\s+([A-Z][\w.]*)/gm, name: "full" }],
|
|
3553
|
-
haskell: [{ re: /^[ \t]*import\s+(?:qualified\s+)?([\w.]+)/gm, name: "full" }],
|
|
3554
|
-
zig: [{ re: /@import\s*\(\s*"([^"]+)"\s*\)/g }],
|
|
3555
|
-
proto: [{ re: /^[ \t]*import\s+(?:public\s+|weak\s+)?"([^"]+)"\s*;/gm }],
|
|
3556
|
-
// `@import "x"`, `@use "x"`, `@forward "x"` — Sass and plain CSS alike.
|
|
3557
|
-
css: [{ re: /^[ \t]*@(?:import|use|forward)\s+(?:url\()?\s*['"]([^'"]+)['"]/gm }],
|
|
3558
|
-
// A .vue/.svelte file's <script> block is JS; the same ESM syntax applies.
|
|
3559
|
-
vue: [{ re: /^[ \t]*import\s[^'"]*from\s*['"]([^'"]+)['"]/gm }],
|
|
3560
|
-
svelte: [{ re: /^[ \t]*import\s[^'"]*from\s*['"]([^'"]+)['"]/gm }],
|
|
3561
|
-
html: [
|
|
3562
|
-
{ re: /<script[^>]+src\s*=\s*['"]([^'"]+)['"]/g },
|
|
3563
|
-
{ re: /<link[^>]+href\s*=\s*['"]([^'"]+)['"]/g }
|
|
3564
|
-
],
|
|
3565
|
-
shell: [{ re: /^[ \t]*(?:source|\.)\s+(\S+)/gm }],
|
|
3566
|
-
r: [{ re: /\b(?:source|library|require)\s*\(\s*['"]?([\w./-]+)['"]?\s*\)/g }]
|
|
3567
|
-
};
|
|
3568
|
-
function lastSegment(specifier) {
|
|
3569
|
-
const pathLike = /[/\\]|::/.test(specifier);
|
|
3570
|
-
const segments = specifier.split(/[/\\]|::/).filter(Boolean);
|
|
3571
|
-
let last = segments[segments.length - 1] ?? specifier;
|
|
3572
|
-
if (last === "*" || last === "_") {
|
|
3573
|
-
last = segments[segments.length - 2] ?? specifier;
|
|
3574
|
-
}
|
|
3575
|
-
if (pathLike) return last.replace(/\.[A-Za-z0-9]{1,8}$/, "") || last;
|
|
3576
|
-
const dotted = last.split(".").filter((part) => part && part !== "*" && part !== "_");
|
|
3577
|
-
return dotted[dotted.length - 1] ?? last;
|
|
3578
|
-
}
|
|
3579
|
-
function newlineOffsets(content) {
|
|
3580
|
-
const offsets = [];
|
|
3581
|
-
for (let i = 0; i < content.length; i++) {
|
|
3582
|
-
if (content.charCodeAt(i) === 10) offsets.push(i);
|
|
3583
|
-
}
|
|
3584
|
-
return offsets;
|
|
3585
|
-
}
|
|
3586
|
-
function lineAt(offsets, index) {
|
|
3587
|
-
let low = 0;
|
|
3588
|
-
let high = offsets.length;
|
|
3589
|
-
while (low < high) {
|
|
3590
|
-
const mid = low + high >>> 1;
|
|
3591
|
-
if ((offsets[mid] ?? 0) < index) low = mid + 1;
|
|
3592
|
-
else high = mid;
|
|
3593
|
-
}
|
|
3594
|
-
return low + 1;
|
|
3595
|
-
}
|
|
3596
|
-
function hasImportPatterns(lang) {
|
|
3597
|
-
return LANG_IMPORTS[lang] !== void 0;
|
|
3598
|
-
}
|
|
3599
|
-
function extractImports(opts) {
|
|
3600
|
-
const patterns = LANG_IMPORTS[opts.lang];
|
|
3601
|
-
if (!patterns || !opts.content) return [];
|
|
3602
|
-
const limit = opts.maxImports ?? IMPORT_MAX_PER_FILE;
|
|
3603
|
-
const content = opts.content.length > IMPORT_MAX_FILE_CHARS ? opts.content.slice(0, IMPORT_MAX_FILE_CHARS) : opts.content;
|
|
3604
|
-
const refs = [];
|
|
3605
|
-
const seen = /* @__PURE__ */ new Set();
|
|
3606
|
-
const offsets = newlineOffsets(content);
|
|
3607
|
-
for (const pattern of patterns) {
|
|
3608
|
-
const re = new RegExp(pattern.re.source, pattern.re.flags);
|
|
3609
|
-
for (const match of content.matchAll(re)) {
|
|
3610
|
-
if (refs.length >= limit) return refs;
|
|
3611
|
-
const specifier = match[1]?.trim();
|
|
3612
|
-
if (!specifier) continue;
|
|
3613
|
-
const module = specifier;
|
|
3614
|
-
const toName = pattern.name === "full" ? module : lastSegment(module);
|
|
3615
|
-
if (!toName) continue;
|
|
3616
|
-
const key = `${module}\0${toName}`;
|
|
3617
|
-
if (seen.has(key)) continue;
|
|
3618
|
-
seen.add(key);
|
|
3619
|
-
refs.push({
|
|
3620
|
-
fromId: 0,
|
|
3621
|
-
toName,
|
|
3622
|
-
callType: "import",
|
|
3623
|
-
line: lineAt(offsets, match.index ?? 0),
|
|
3624
|
-
lang: opts.lang,
|
|
3625
|
-
module
|
|
3626
|
-
});
|
|
3627
|
-
}
|
|
3628
|
-
}
|
|
3629
|
-
return refs;
|
|
3630
|
-
}
|
|
3631
|
-
|
|
3632
|
-
// src/codebase-index/parser-dispatch.ts
|
|
3633
|
-
async function parseFileContent(file, content, lang) {
|
|
3634
|
-
const parsed = await dispatch(file, content, lang);
|
|
3635
|
-
return withRelations(parsed, content, lang);
|
|
3636
|
-
}
|
|
3637
|
-
async function dispatch(file, content, lang) {
|
|
3638
|
-
switch (lang) {
|
|
3639
|
-
case "ts":
|
|
3640
|
-
case "tsx":
|
|
3641
|
-
case "js":
|
|
3642
|
-
case "jsx": {
|
|
3643
|
-
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_ts_parser(), ts_parser_exports));
|
|
3644
|
-
return parseSymbols9({ file, content, lang });
|
|
3645
|
-
}
|
|
3646
|
-
case "go": {
|
|
3647
|
-
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_go_parser(), go_parser_exports));
|
|
3648
|
-
return parseSymbols9({ file, content, lang: "go" });
|
|
3649
|
-
}
|
|
3650
|
-
case "py": {
|
|
3651
|
-
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_py_parser(), py_parser_exports));
|
|
3652
|
-
return parseSymbols9({ file, content, lang: "py" });
|
|
3653
|
-
}
|
|
3654
|
-
case "rs": {
|
|
3655
|
-
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_rs_parser(), rs_parser_exports));
|
|
3656
|
-
return parseSymbols9({ file, content, lang: "rs" });
|
|
3657
|
-
}
|
|
3658
|
-
case "json": {
|
|
3659
|
-
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_json_parser(), json_parser_exports));
|
|
3660
|
-
return parseSymbols9({ file, content, lang: "json" });
|
|
3661
|
-
}
|
|
3662
|
-
case "yaml": {
|
|
3663
|
-
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_yaml_parser(), yaml_parser_exports));
|
|
3664
|
-
return parseSymbols9({ file, content, lang: "yaml" });
|
|
3665
|
-
}
|
|
3666
|
-
// Phase 1: ten languages now route through the Tree-Sitter WASM
|
|
3667
|
-
// universal parser (`tree-sitter-parser.ts`). The dispatch falls back to
|
|
3668
|
-
// the regex extractor in `generic-parser.ts` whenever WASM loading fails
|
|
3669
|
-
// or the parser returns zero symbols — preserving the indexable-file
|
|
3670
|
-
// contract that "missing a parser must never mean skipping the file".
|
|
3671
|
-
case "c":
|
|
3672
|
-
case "cpp":
|
|
3673
|
-
case "java":
|
|
3674
|
-
case "csharp":
|
|
3675
|
-
case "php":
|
|
3676
|
-
case "ruby":
|
|
3677
|
-
case "swift":
|
|
3678
|
-
case "kotlin":
|
|
3679
|
-
case "shell":
|
|
3680
|
-
case "elixir": {
|
|
3681
|
-
try {
|
|
3682
|
-
const { parseSymbols: parseSymbols10 } = await Promise.resolve().then(() => (init_tree_sitter_parser(), tree_sitter_parser_exports));
|
|
3683
|
-
const parsed = await parseSymbols10({ file, content, lang });
|
|
3684
|
-
if (parsed.symbols.length > 0) return parsed;
|
|
3685
|
-
} catch {
|
|
3686
|
-
}
|
|
3687
|
-
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_generic_parser(), generic_parser_exports));
|
|
3688
|
-
return parseSymbols9({ file, content, lang });
|
|
3689
|
-
}
|
|
3690
|
-
default: {
|
|
3691
|
-
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_generic_parser(), generic_parser_exports));
|
|
3692
|
-
return parseSymbols9({ file, content, lang });
|
|
3693
|
-
}
|
|
3694
|
-
}
|
|
3695
|
-
}
|
|
3696
|
-
function withRelations(parsed, content, lang) {
|
|
3697
|
-
let refs = parsed.refs ?? [];
|
|
3698
|
-
if (refs.length === 0 && hasImportPatterns(lang)) {
|
|
3699
|
-
refs = extractImports({ content, lang });
|
|
3700
|
-
}
|
|
3701
|
-
return { ...parsed, refs: refs.map((ref) => ref.lang ? ref : { ...ref, lang }) };
|
|
3702
|
-
}
|
|
4775
|
+
// src/codebase-index/indexer.ts
|
|
4776
|
+
init_parser_dispatch();
|
|
3703
4777
|
|
|
3704
4778
|
// src/codebase-index/parser-worker-pool.ts
|
|
4779
|
+
import * as fs7 from "node:fs";
|
|
4780
|
+
import { fileURLToPath as fileURLToPath2, pathToFileURL } from "node:url";
|
|
3705
4781
|
import { Worker } from "node:worker_threads";
|
|
3706
|
-
import { fileURLToPath as fileURLToPath2 } from "node:url";
|
|
3707
|
-
import * as fs6 from "node:fs";
|
|
3708
4782
|
var WORKER_POOL_THRESHOLD = 500;
|
|
4783
|
+
function resolveWorkerPoolThreshold() {
|
|
4784
|
+
const raw = process.env["WRONGSTACK_INDEX_WORKER_THRESHOLD"];
|
|
4785
|
+
if (raw === void 0) return WORKER_POOL_THRESHOLD;
|
|
4786
|
+
if (!/^\d+$/.test(raw)) return WORKER_POOL_THRESHOLD;
|
|
4787
|
+
return Number.parseInt(raw, 10);
|
|
4788
|
+
}
|
|
3709
4789
|
var ParserWorkerPool = class {
|
|
3710
4790
|
constructor(maxWorkers = defaultWorkerCount()) {
|
|
3711
4791
|
this.maxWorkers = maxWorkers;
|
|
@@ -3749,7 +4829,8 @@ var ParserWorkerPool = class {
|
|
|
3749
4829
|
w.unref();
|
|
3750
4830
|
w.on("message", (msg) => this.handleMessage(msg));
|
|
3751
4831
|
w.on("error", (err) => this.handleError(err, w));
|
|
3752
|
-
|
|
4832
|
+
w.on("exit", () => this.retireByReference(w));
|
|
4833
|
+
this.workers.push({ worker: w, workerId: w.threadId, busy: false });
|
|
3753
4834
|
} catch {
|
|
3754
4835
|
if (this.workers.length === 0) {
|
|
3755
4836
|
this.unavailable = true;
|
|
@@ -3765,7 +4846,7 @@ var ParserWorkerPool = class {
|
|
|
3765
4846
|
}
|
|
3766
4847
|
/**
|
|
3767
4848
|
* Parse files in parallel across the worker pool. Returns a flat
|
|
3768
|
-
* `FileSymbols[]` in completion order (caller
|
|
4849
|
+
* `FileSymbols[]` in completion order (caller matches by file path).
|
|
3769
4850
|
*
|
|
3770
4851
|
* Content is pre-read by the main thread (for the content-hash check)
|
|
3771
4852
|
* and passed to workers to avoid a second disk read. Files are
|
|
@@ -3786,21 +4867,19 @@ var ParserWorkerPool = class {
|
|
|
3786
4867
|
chunks[i % workerCount].push(files[i]);
|
|
3787
4868
|
}
|
|
3788
4869
|
return new Promise((resolve2, reject) => {
|
|
4870
|
+
const pendingChunks = /* @__PURE__ */ new Map();
|
|
3789
4871
|
this.pending.set(batchId, {
|
|
3790
4872
|
resolve: resolve2,
|
|
3791
4873
|
reject,
|
|
3792
4874
|
accumulated: [],
|
|
3793
|
-
|
|
3794
|
-
|
|
4875
|
+
pendingChunks,
|
|
4876
|
+
settled: false
|
|
3795
4877
|
});
|
|
3796
4878
|
for (let i = 0; i < workerCount; i++) {
|
|
3797
4879
|
const pw = this.workers[i];
|
|
3798
4880
|
pw.busy = true;
|
|
3799
|
-
pw.
|
|
3800
|
-
|
|
3801
|
-
id: batchId,
|
|
3802
|
-
files: chunks[i]
|
|
3803
|
-
});
|
|
4881
|
+
pendingChunks.set(pw.workerId, chunks[i]);
|
|
4882
|
+
pw.worker.postMessage({ type: "parse", id: batchId, files: chunks[i] });
|
|
3804
4883
|
}
|
|
3805
4884
|
});
|
|
3806
4885
|
}
|
|
@@ -3809,6 +4888,12 @@ var ParserWorkerPool = class {
|
|
|
3809
4888
|
const workers = this.workers.map((w) => w.worker);
|
|
3810
4889
|
this.workers = [];
|
|
3811
4890
|
this.unavailable = false;
|
|
4891
|
+
for (const [, p] of this.pending) {
|
|
4892
|
+
if (p.settled) continue;
|
|
4893
|
+
p.settled = true;
|
|
4894
|
+
p.reject(new Error("ParserWorkerPool shut down"));
|
|
4895
|
+
}
|
|
4896
|
+
this.pending.clear();
|
|
3812
4897
|
for (const w of workers) {
|
|
3813
4898
|
try {
|
|
3814
4899
|
w.postMessage({ type: "shutdown" });
|
|
@@ -3829,39 +4914,117 @@ var ParserWorkerPool = class {
|
|
|
3829
4914
|
})
|
|
3830
4915
|
)
|
|
3831
4916
|
);
|
|
3832
|
-
for (const [, p] of this.pending) p.reject(new Error("ParserWorkerPool shut down"));
|
|
3833
|
-
this.pending.clear();
|
|
3834
4917
|
}
|
|
3835
4918
|
handleMessage(msg) {
|
|
3836
4919
|
const batch = this.pending.get(msg.id);
|
|
3837
4920
|
if (!batch) return;
|
|
4921
|
+
const worker = this.workers.find((w) => w.workerId === msg.workerId);
|
|
4922
|
+
if (worker) worker.busy = false;
|
|
4923
|
+
batch.pendingChunks.delete(msg.workerId);
|
|
3838
4924
|
batch.accumulated.push(...msg.results);
|
|
3839
|
-
batch.
|
|
3840
|
-
|
|
3841
|
-
|
|
3842
|
-
if (batch.completedWorkers >= batch.expectedWorkers) {
|
|
4925
|
+
if (batch.pendingChunks.size === 0) {
|
|
4926
|
+
if (batch.settled) return;
|
|
4927
|
+
batch.settled = true;
|
|
3843
4928
|
this.pending.delete(msg.id);
|
|
3844
4929
|
batch.resolve(batch.accumulated);
|
|
3845
4930
|
}
|
|
3846
4931
|
}
|
|
3847
|
-
|
|
3848
|
-
|
|
3849
|
-
|
|
3850
|
-
|
|
3851
|
-
|
|
3852
|
-
|
|
4932
|
+
/**
|
|
4933
|
+
* Remove a worker from the pool and salvage any chunk it still owed.
|
|
4934
|
+
*
|
|
4935
|
+
* Idempotent by workerId — `error` and `exit` can both fire for one
|
|
4936
|
+
* death, and a worker may die while no batch references it. When the
|
|
4937
|
+
* dead worker owed files to an in-flight batch and other workers remain,
|
|
4938
|
+
* those files are re-parsed inline on this thread (one fewer worker
|
|
4939
|
+
* should cost latency, not correctness). When it was the last worker,
|
|
4940
|
+
* every remaining batch rejects so the indexer's existing inline
|
|
4941
|
+
* fallback takes over the whole batch.
|
|
4942
|
+
*/
|
|
4943
|
+
retireWorker(workerId) {
|
|
4944
|
+
const entry = this.workers.find((w) => w.workerId === workerId);
|
|
4945
|
+
if (!entry) return;
|
|
4946
|
+
this.workers = this.workers.filter((w) => w.workerId !== workerId);
|
|
4947
|
+
for (const [batchId, batch] of [...this.pending]) {
|
|
4948
|
+
const orphaned = batch.pendingChunks.get(workerId);
|
|
4949
|
+
if (!orphaned) continue;
|
|
4950
|
+
if (this.workers.length === 0) {
|
|
4951
|
+
this.pending.delete(batchId);
|
|
4952
|
+
this.unavailable = true;
|
|
4953
|
+
if (!batch.settled) {
|
|
4954
|
+
batch.settled = true;
|
|
4955
|
+
batch.reject(new Error("ParserWorkerPool: all workers died mid-batch"));
|
|
4956
|
+
}
|
|
4957
|
+
continue;
|
|
4958
|
+
}
|
|
4959
|
+
void this.reparseInline(batch, orphaned, workerId);
|
|
4960
|
+
}
|
|
4961
|
+
}
|
|
4962
|
+
/**
|
|
4963
|
+
* Salvage path: re-parse an orphaned chunk on this thread. Files that
|
|
4964
|
+
* fail here stay absent from the results — same contract as a per-file
|
|
4965
|
+
* error inside a live worker (see handleMessage).
|
|
4966
|
+
*/
|
|
4967
|
+
async reparseInline(batch, orphaned, workerId) {
|
|
4968
|
+
try {
|
|
4969
|
+
const { parseFileContent: parseFileContent2 } = await Promise.resolve().then(() => (init_parser_dispatch(), parser_dispatch_exports));
|
|
4970
|
+
for (const item of orphaned) {
|
|
4971
|
+
if (batch.settled) return;
|
|
4972
|
+
try {
|
|
4973
|
+
const parsed = await parseFileContent2(item.file, item.content, item.lang);
|
|
4974
|
+
batch.accumulated.push(parsed);
|
|
4975
|
+
} catch {
|
|
4976
|
+
}
|
|
4977
|
+
await new Promise((resolve2) => setImmediate(resolve2));
|
|
4978
|
+
}
|
|
4979
|
+
} finally {
|
|
4980
|
+
this.finishSalvage(batch, workerId);
|
|
4981
|
+
}
|
|
4982
|
+
}
|
|
4983
|
+
/**
|
|
4984
|
+
* Terminal tail of a salvage — runs on every exit path. Kept free of
|
|
4985
|
+
* control flow inside a `finally` (noUnsafeFinally): releases the
|
|
4986
|
+
* pending-marker and resolves the batch if this was its last chunk.
|
|
4987
|
+
*/
|
|
4988
|
+
finishSalvage(batch, workerId) {
|
|
4989
|
+
batch.pendingChunks.delete(workerId);
|
|
4990
|
+
if (batch.pendingChunks.size === 0) {
|
|
4991
|
+
for (const [batchId, tracked] of this.pending) {
|
|
4992
|
+
if (tracked === batch) {
|
|
4993
|
+
if (batch.settled) return;
|
|
4994
|
+
batch.settled = true;
|
|
4995
|
+
this.pending.delete(batchId);
|
|
4996
|
+
batch.resolve(batch.accumulated);
|
|
4997
|
+
return;
|
|
4998
|
+
}
|
|
4999
|
+
}
|
|
3853
5000
|
}
|
|
3854
5001
|
}
|
|
5002
|
+
/**
|
|
5003
|
+
* Retire by worker object rather than threadId. `threadId` is -1 before
|
|
5004
|
+
* the worker emits `online`, so a death during script load would make a
|
|
5005
|
+
* threadId-keyed lookup silently no-op and leak the entry (with its
|
|
5006
|
+
* pending chunk) — reference identity is correct in every case.
|
|
5007
|
+
*/
|
|
5008
|
+
retireByReference(source) {
|
|
5009
|
+
const entry = this.workers.find((w) => w.worker === source);
|
|
5010
|
+
if (entry) this.retireWorker(entry.workerId);
|
|
5011
|
+
}
|
|
5012
|
+
handleError(err, source) {
|
|
5013
|
+
void err;
|
|
5014
|
+
this.retireByReference(source);
|
|
5015
|
+
}
|
|
3855
5016
|
};
|
|
3856
5017
|
function defaultWorkerCount() {
|
|
3857
5018
|
const cores = globalThis.navigator?.hardwareConcurrency ?? 4;
|
|
3858
5019
|
return Math.max(1, Math.min(4, cores - 1));
|
|
3859
5020
|
}
|
|
3860
5021
|
function resolveWorkerScriptUrl() {
|
|
5022
|
+
const override = process.env["WRONGSTACK_PARSER_WORKER_SCRIPT"];
|
|
5023
|
+
if (override) return pathToFileURL(override);
|
|
3861
5024
|
for (const rel of ["./parser-worker-script.js", "./codebase-index/parser-worker-script.js"]) {
|
|
3862
5025
|
try {
|
|
3863
5026
|
const url = new URL(rel, import.meta.url);
|
|
3864
|
-
if (url.protocol === "file:" &&
|
|
5027
|
+
if (url.protocol === "file:" && fs7.existsSync(fileURLToPath2(url))) return url;
|
|
3865
5028
|
} catch {
|
|
3866
5029
|
}
|
|
3867
5030
|
}
|
|
@@ -3874,8 +5037,8 @@ function getParserPool() {
|
|
|
3874
5037
|
}
|
|
3875
5038
|
|
|
3876
5039
|
// src/codebase-index/writer.ts
|
|
3877
|
-
import * as
|
|
3878
|
-
import * as
|
|
5040
|
+
import * as fs9 from "node:fs";
|
|
5041
|
+
import * as path12 from "node:path";
|
|
3879
5042
|
|
|
3880
5043
|
// src/codebase-index/bm25.ts
|
|
3881
5044
|
var K1 = 1.5;
|
|
@@ -3970,11 +5133,11 @@ var Bm25Index = class {
|
|
|
3970
5133
|
init_languages();
|
|
3971
5134
|
|
|
3972
5135
|
// src/codebase-index/schema.ts
|
|
3973
|
-
var SCHEMA_VERSION =
|
|
5136
|
+
var SCHEMA_VERSION = 5;
|
|
3974
5137
|
|
|
3975
5138
|
// src/codebase-index/sqlite-runtime.ts
|
|
3976
|
-
import { createRequire } from "node:module";
|
|
3977
5139
|
import { toErrorMessage } from "@wrongstack/core/utils";
|
|
5140
|
+
import { loadRuntimeDatabaseSync } from "@wrongstack/persistence";
|
|
3978
5141
|
|
|
3979
5142
|
// src/codebase-index/circuit-breaker.ts
|
|
3980
5143
|
var LockError = class extends Error {
|
|
@@ -4068,11 +5231,10 @@ function loadDatabaseSync() {
|
|
|
4068
5231
|
if (DatabaseSyncCtor) return DatabaseSyncCtor;
|
|
4069
5232
|
silenceSqliteExperimentalWarning();
|
|
4070
5233
|
try {
|
|
4071
|
-
|
|
4072
|
-
DatabaseSyncCtor = req("node:sqlite").DatabaseSync;
|
|
5234
|
+
DatabaseSyncCtor = loadRuntimeDatabaseSync();
|
|
4073
5235
|
} catch (err) {
|
|
4074
5236
|
throw new Error(
|
|
4075
|
-
`The codebase index needs
|
|
5237
|
+
`The codebase index needs node:sqlite (Node >= 22.5) or bun:sqlite. This runtime doesn't provide it: ${toErrorMessage(err)}`
|
|
4076
5238
|
);
|
|
4077
5239
|
}
|
|
4078
5240
|
return DatabaseSyncCtor;
|
|
@@ -4096,28 +5258,106 @@ function sleepSync(ms) {
|
|
|
4096
5258
|
} catch {
|
|
4097
5259
|
}
|
|
4098
5260
|
}
|
|
4099
|
-
function runSqliteWithRetry(fn) {
|
|
4100
|
-
let lastError;
|
|
4101
|
-
for (let attempt = 0; attempt <= MAX_LOCK_RETRIES; attempt++) {
|
|
4102
|
-
try {
|
|
4103
|
-
return fn();
|
|
4104
|
-
} catch (err) {
|
|
4105
|
-
lastError = err;
|
|
4106
|
-
if (!isLockError(err)) throw err;
|
|
4107
|
-
if (attempt === MAX_LOCK_RETRIES) {
|
|
4108
|
-
const msg = lastError instanceof Error ? lastError.message : String(lastError);
|
|
4109
|
-
throw new LockError(`SQLite lock conflict after ${MAX_LOCK_RETRIES} retries: ${msg}`);
|
|
4110
|
-
}
|
|
4111
|
-
const delay = Math.min(LOCK_RETRY_BASE_DELAY_MS * 2 ** attempt, LOCK_RETRY_MAX_DELAY_MS);
|
|
4112
|
-
sleepSync(delay);
|
|
4113
|
-
}
|
|
5261
|
+
function runSqliteWithRetry(fn) {
|
|
5262
|
+
let lastError;
|
|
5263
|
+
for (let attempt = 0; attempt <= MAX_LOCK_RETRIES; attempt++) {
|
|
5264
|
+
try {
|
|
5265
|
+
return fn();
|
|
5266
|
+
} catch (err) {
|
|
5267
|
+
lastError = err;
|
|
5268
|
+
if (!isLockError(err)) throw err;
|
|
5269
|
+
if (attempt === MAX_LOCK_RETRIES) {
|
|
5270
|
+
const msg = lastError instanceof Error ? lastError.message : String(lastError);
|
|
5271
|
+
throw new LockError(`SQLite lock conflict after ${MAX_LOCK_RETRIES} retries: ${msg}`);
|
|
5272
|
+
}
|
|
5273
|
+
const delay = Math.min(LOCK_RETRY_BASE_DELAY_MS * 2 ** attempt, LOCK_RETRY_MAX_DELAY_MS);
|
|
5274
|
+
sleepSync(delay);
|
|
5275
|
+
}
|
|
5276
|
+
}
|
|
5277
|
+
throw lastError;
|
|
5278
|
+
}
|
|
5279
|
+
|
|
5280
|
+
// src/codebase-index/vector-search.ts
|
|
5281
|
+
var RRF_K = 60;
|
|
5282
|
+
function vectorEmbeddingEnabled() {
|
|
5283
|
+
return process.env["WRONGSTACK_INDEX_VECTORS"] === "1";
|
|
5284
|
+
}
|
|
5285
|
+
var VECTOR_DIMENSIONS = 384;
|
|
5286
|
+
var NGRAM_SIZE = 3;
|
|
5287
|
+
function embedText(text) {
|
|
5288
|
+
const vec = new Float32Array(VECTOR_DIMENSIONS);
|
|
5289
|
+
const normalized = text.toLowerCase().trim();
|
|
5290
|
+
if (normalized.length < NGRAM_SIZE) {
|
|
5291
|
+
const padded = ` ${normalized} `.slice(0, Math.max(NGRAM_SIZE, normalized.length + 2));
|
|
5292
|
+
for (let i = 0; i <= padded.length - NGRAM_SIZE; i++) {
|
|
5293
|
+
const ngram = padded.slice(i, i + NGRAM_SIZE);
|
|
5294
|
+
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
5295
|
+
vec[bucket] += 1;
|
|
5296
|
+
}
|
|
5297
|
+
} else {
|
|
5298
|
+
for (let i = 0; i <= normalized.length - NGRAM_SIZE; i++) {
|
|
5299
|
+
const ngram = normalized.slice(i, i + NGRAM_SIZE);
|
|
5300
|
+
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
5301
|
+
vec[bucket] += 1;
|
|
5302
|
+
}
|
|
5303
|
+
}
|
|
5304
|
+
let norm = 0;
|
|
5305
|
+
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
5306
|
+
norm += vec[i] * vec[i];
|
|
5307
|
+
}
|
|
5308
|
+
norm = Math.sqrt(norm);
|
|
5309
|
+
if (norm > 0) {
|
|
5310
|
+
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
5311
|
+
vec[i] /= norm;
|
|
5312
|
+
}
|
|
5313
|
+
}
|
|
5314
|
+
return vec;
|
|
5315
|
+
}
|
|
5316
|
+
function hashNgram(str) {
|
|
5317
|
+
let hash = 2166136261;
|
|
5318
|
+
for (let i = 0; i < str.length; i++) {
|
|
5319
|
+
hash ^= str.charCodeAt(i);
|
|
5320
|
+
hash = Math.imul(hash, 16777619);
|
|
5321
|
+
}
|
|
5322
|
+
return hash >>> 0;
|
|
5323
|
+
}
|
|
5324
|
+
function cosineSimilarity(a, b) {
|
|
5325
|
+
let dot = 0;
|
|
5326
|
+
const len = Math.min(a.length, b.length);
|
|
5327
|
+
for (let i = 0; i < len; i++) {
|
|
5328
|
+
dot += a[i] * b[i];
|
|
5329
|
+
}
|
|
5330
|
+
return dot;
|
|
5331
|
+
}
|
|
5332
|
+
function encodeVector(vec) {
|
|
5333
|
+
return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
|
|
5334
|
+
}
|
|
5335
|
+
function decodeVector(buf) {
|
|
5336
|
+
const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
5337
|
+
const copy = new Float32Array(buf.byteLength / 4);
|
|
5338
|
+
for (let i = 0; i < copy.length; i++) {
|
|
5339
|
+
copy[i] = view.getFloat32(i * 4, true);
|
|
5340
|
+
}
|
|
5341
|
+
return copy;
|
|
5342
|
+
}
|
|
5343
|
+
function reciprocalRankFusion(bm25Ranks, vectorRanks, k = RRF_K) {
|
|
5344
|
+
const allIds = /* @__PURE__ */ new Set([...bm25Ranks.keys(), ...vectorRanks.keys()]);
|
|
5345
|
+
const scored = [];
|
|
5346
|
+
for (const id of allIds) {
|
|
5347
|
+
const bm25Rank = bm25Ranks.get(id);
|
|
5348
|
+
const vecRank = vectorRanks.get(id);
|
|
5349
|
+
let score = 0;
|
|
5350
|
+
if (bm25Rank !== void 0) score += 1 / (k + bm25Rank);
|
|
5351
|
+
if (vecRank !== void 0) score += 1 / (k + vecRank);
|
|
5352
|
+
scored.push([id, score]);
|
|
4114
5353
|
}
|
|
4115
|
-
|
|
5354
|
+
scored.sort((a, b) => b[1] - a[1]);
|
|
5355
|
+
return scored;
|
|
4116
5356
|
}
|
|
4117
5357
|
|
|
4118
5358
|
// src/codebase-index/writer-admin.ts
|
|
4119
|
-
import * as
|
|
4120
|
-
import * as
|
|
5359
|
+
import * as fs8 from "node:fs";
|
|
5360
|
+
import * as path11 from "node:path";
|
|
4121
5361
|
var DB_FILE = "index.db";
|
|
4122
5362
|
function getAllIndexableWithStatement(stmt) {
|
|
4123
5363
|
return stmt("SELECT id, text FROM symbols").all().map(
|
|
@@ -4153,6 +5393,14 @@ function getMetadataWithStatement(stmt, key) {
|
|
|
4153
5393
|
const rows = stmt("SELECT value FROM metadata WHERE key = ?").all(key);
|
|
4154
5394
|
return rows[0]?.value;
|
|
4155
5395
|
}
|
|
5396
|
+
function getIndexSummaryWithStatement(stmt) {
|
|
5397
|
+
const fileRows = stmt("SELECT COUNT(*) AS n FROM files").all();
|
|
5398
|
+
const lastRows = stmt("SELECT value FROM metadata WHERE key = 'last_indexed'").all();
|
|
5399
|
+
return {
|
|
5400
|
+
totalFiles: fileRows[0] ? Number(fileRows[0].n) : 0,
|
|
5401
|
+
lastIndexed: lastRows.length ? Number(lastRows[0]?.value) : null
|
|
5402
|
+
};
|
|
5403
|
+
}
|
|
4156
5404
|
function getFileMetaWithStatement(stmt, file) {
|
|
4157
5405
|
const rows = stmt(
|
|
4158
5406
|
"SELECT file, lang, mtime_ms, symbol_count, last_indexed, content_hash FROM files WHERE file = ?"
|
|
@@ -4182,22 +5430,102 @@ function getAllFileMetasWithStatement(stmt) {
|
|
|
4182
5430
|
}
|
|
4183
5431
|
function getIndexDbSizeBytes(indexDir) {
|
|
4184
5432
|
try {
|
|
4185
|
-
return
|
|
5433
|
+
return fs8.statSync(path11.join(indexDir, DB_FILE)).size;
|
|
4186
5434
|
} catch {
|
|
4187
5435
|
return 0;
|
|
4188
5436
|
}
|
|
4189
5437
|
}
|
|
4190
5438
|
|
|
5439
|
+
// src/codebase-index/writer-helpers.ts
|
|
5440
|
+
import { resolveWstackPaths } from "@wrongstack/core/utils";
|
|
5441
|
+
function escapeLike(value) {
|
|
5442
|
+
return value.replace(/[\\%_]/g, (char) => `\\${char}`);
|
|
5443
|
+
}
|
|
5444
|
+
function ladderChunkSizes(total, max) {
|
|
5445
|
+
if (total <= 0) return [];
|
|
5446
|
+
const sizes = [];
|
|
5447
|
+
let remaining = total;
|
|
5448
|
+
while (remaining > 0) {
|
|
5449
|
+
const cap = Math.min(remaining, max);
|
|
5450
|
+
const pow = cap >= 1 ? 2 ** Math.floor(Math.log2(cap)) : 1;
|
|
5451
|
+
const take = Math.max(1, Math.min(pow, remaining));
|
|
5452
|
+
sizes.push(take);
|
|
5453
|
+
remaining -= take;
|
|
5454
|
+
}
|
|
5455
|
+
return sizes;
|
|
5456
|
+
}
|
|
5457
|
+
function nextPow2(count) {
|
|
5458
|
+
return count <= 1 ? 1 : 2 ** Math.ceil(Math.log2(count));
|
|
5459
|
+
}
|
|
5460
|
+
function padToInBucket(values) {
|
|
5461
|
+
if (values.length <= 1) return values.slice();
|
|
5462
|
+
const target = nextPow2(values.length);
|
|
5463
|
+
const padded = values.slice();
|
|
5464
|
+
while (padded.length < target) padded.push(padded[0]);
|
|
5465
|
+
return padded;
|
|
5466
|
+
}
|
|
5467
|
+
function placeholders(count) {
|
|
5468
|
+
return Array.from({ length: count }, () => "?").join(",");
|
|
5469
|
+
}
|
|
5470
|
+
function inListChunks(total, max) {
|
|
5471
|
+
if (total <= 0) return [];
|
|
5472
|
+
if (nextPow2(total) <= max) return [total];
|
|
5473
|
+
const powMax = Math.max(1, 2 ** Math.floor(Math.log2(max)));
|
|
5474
|
+
return ladderChunkSizes(total, powMax);
|
|
5475
|
+
}
|
|
5476
|
+
function posixIndexPath(file) {
|
|
5477
|
+
return file.replace(/\\/g, "/").replace(/^\.\//, "");
|
|
5478
|
+
}
|
|
5479
|
+
function indexedFileMatchSql(column = "file") {
|
|
5480
|
+
return `(${column} = ? OR replace(${column}, '\\', '/') = ? OR replace(${column}, '\\', '/') LIKE ? ESCAPE '\\')`;
|
|
5481
|
+
}
|
|
5482
|
+
function indexedFileMatchArgs(file) {
|
|
5483
|
+
const posix4 = posixIndexPath(file.trim());
|
|
5484
|
+
return [file, posix4, `%/${escapeLike(posix4)}`];
|
|
5485
|
+
}
|
|
5486
|
+
function matchesIndexedPackageFilter(storedFile, packageLabel, filter) {
|
|
5487
|
+
if (packageLabel === filter) return true;
|
|
5488
|
+
const posixFile = posixIndexPath(storedFile);
|
|
5489
|
+
const posixFilter = posixIndexPath(filter.trim());
|
|
5490
|
+
if (!posixFilter) return false;
|
|
5491
|
+
return posixFile === posixFilter || posixFile.endsWith(`/${posixFilter}`) || posixFile.includes(`/${posixFilter}/`);
|
|
5492
|
+
}
|
|
5493
|
+
function assignRefsToSymbols(refs, symbols) {
|
|
5494
|
+
if (refs.length === 0 || symbols.length === 0) return [];
|
|
5495
|
+
const ordered = [...symbols].sort((a, b) => a.line - b.line || a.col - b.col || a.id - b.id);
|
|
5496
|
+
const seen = /* @__PURE__ */ new Set();
|
|
5497
|
+
const assigned = [];
|
|
5498
|
+
for (const ref of refs) {
|
|
5499
|
+
let owner;
|
|
5500
|
+
for (const symbol of ordered) {
|
|
5501
|
+
if (symbol.line > ref.line) break;
|
|
5502
|
+
owner = symbol;
|
|
5503
|
+
}
|
|
5504
|
+
if (!owner && ref.callType === "import") owner = ordered[0];
|
|
5505
|
+
if (!owner || owner.id <= 0) continue;
|
|
5506
|
+
const key = `${owner.id}:${ref.toName}:${ref.callType}:${ref.module ?? ""}`;
|
|
5507
|
+
if (seen.has(key)) continue;
|
|
5508
|
+
seen.add(key);
|
|
5509
|
+
assigned.push({ ...ref, fromId: owner.id });
|
|
5510
|
+
}
|
|
5511
|
+
return assigned;
|
|
5512
|
+
}
|
|
5513
|
+
function resolveIndexDir(projectRoot, override) {
|
|
5514
|
+
return override ?? resolveWstackPaths({ projectRoot }).projectCodebaseIndex;
|
|
5515
|
+
}
|
|
5516
|
+
|
|
4191
5517
|
// src/codebase-index/writer-bulk-insert.ts
|
|
4192
5518
|
function bulkInsertSymbolsWithStatement(stmt, maxSqlVars, rows) {
|
|
4193
5519
|
if (rows.length === 0) return;
|
|
4194
|
-
const
|
|
4195
|
-
|
|
4196
|
-
|
|
4197
|
-
const
|
|
5520
|
+
const ladder = ladderChunkSizes(rows.length, Math.max(1, Math.floor(maxSqlVars / 12)));
|
|
5521
|
+
let cursor = 0;
|
|
5522
|
+
for (const take of ladder) {
|
|
5523
|
+
const chunk = rows.slice(cursor, cursor + take);
|
|
5524
|
+
cursor += take;
|
|
5525
|
+
const placeholders2 = chunk.map(() => "(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)").join(", ");
|
|
4198
5526
|
const insert = stmt(
|
|
4199
|
-
`INSERT INTO symbols(id, lang, kind, name, file, line, col, signature, doc_comment, scope, text
|
|
4200
|
-
VALUES ${
|
|
5527
|
+
`INSERT INTO symbols(id, lang, kind, name, file, line, col, signature, doc_comment, scope, text)
|
|
5528
|
+
VALUES ${placeholders2}`
|
|
4201
5529
|
);
|
|
4202
5530
|
const binds = [];
|
|
4203
5531
|
for (const r of chunk) {
|
|
@@ -4212,8 +5540,7 @@ function bulkInsertSymbolsWithStatement(stmt, maxSqlVars, rows) {
|
|
|
4212
5540
|
r.signature,
|
|
4213
5541
|
r.docComment,
|
|
4214
5542
|
r.scope,
|
|
4215
|
-
r.text
|
|
4216
|
-
r.file
|
|
5543
|
+
r.text
|
|
4217
5544
|
);
|
|
4218
5545
|
}
|
|
4219
5546
|
insert.run(...binds);
|
|
@@ -4221,11 +5548,13 @@ function bulkInsertSymbolsWithStatement(stmt, maxSqlVars, rows) {
|
|
|
4221
5548
|
}
|
|
4222
5549
|
function bulkInsertFtsWithStatement(stmt, maxSqlVars, ftsAvailable, rows) {
|
|
4223
5550
|
if (!ftsAvailable || rows.length === 0) return;
|
|
4224
|
-
const
|
|
4225
|
-
|
|
4226
|
-
|
|
4227
|
-
const
|
|
4228
|
-
|
|
5551
|
+
const ladder = ladderChunkSizes(rows.length, Math.max(1, Math.floor(maxSqlVars / 2)));
|
|
5552
|
+
let cursor = 0;
|
|
5553
|
+
for (const take of ladder) {
|
|
5554
|
+
const chunk = rows.slice(cursor, cursor + take);
|
|
5555
|
+
cursor += take;
|
|
5556
|
+
const placeholders2 = chunk.map(() => "(?, ?)").join(", ");
|
|
5557
|
+
const insert = stmt(`INSERT INTO symbols_fts(rowid, text) VALUES ${placeholders2}`);
|
|
4229
5558
|
const binds = [];
|
|
4230
5559
|
for (const r of chunk) binds.push(r.id, r.text);
|
|
4231
5560
|
insert.run(...binds);
|
|
@@ -4233,11 +5562,13 @@ function bulkInsertFtsWithStatement(stmt, maxSqlVars, ftsAvailable, rows) {
|
|
|
4233
5562
|
}
|
|
4234
5563
|
function bulkInsertVectorsWithStatement(stmt, maxSqlVars, rows) {
|
|
4235
5564
|
if (rows.length === 0) return;
|
|
4236
|
-
const
|
|
4237
|
-
|
|
4238
|
-
|
|
4239
|
-
const
|
|
4240
|
-
|
|
5565
|
+
const ladder = ladderChunkSizes(rows.length, Math.max(1, Math.floor(maxSqlVars / 2)));
|
|
5566
|
+
let cursor = 0;
|
|
5567
|
+
for (const take of ladder) {
|
|
5568
|
+
const chunk = rows.slice(cursor, cursor + take);
|
|
5569
|
+
cursor += take;
|
|
5570
|
+
const placeholders2 = chunk.map(() => "(?, ?)").join(", ");
|
|
5571
|
+
const insert = stmt(`INSERT INTO symbol_vectors(symbol_id, vector) VALUES ${placeholders2}`);
|
|
4241
5572
|
const binds = [];
|
|
4242
5573
|
for (const r of chunk) binds.push(r.id, r.vector);
|
|
4243
5574
|
insert.run(...binds);
|
|
@@ -4245,13 +5576,15 @@ function bulkInsertVectorsWithStatement(stmt, maxSqlVars, rows) {
|
|
|
4245
5576
|
}
|
|
4246
5577
|
function bulkInsertRefsWithStatement(stmt, maxSqlVars, refs) {
|
|
4247
5578
|
if (refs.length === 0) return;
|
|
4248
|
-
const
|
|
4249
|
-
|
|
4250
|
-
|
|
4251
|
-
const
|
|
5579
|
+
const ladder = ladderChunkSizes(refs.length, Math.max(1, Math.floor(maxSqlVars / 8)));
|
|
5580
|
+
let cursor = 0;
|
|
5581
|
+
for (const take of ladder) {
|
|
5582
|
+
const chunk = refs.slice(cursor, cursor + take);
|
|
5583
|
+
cursor += take;
|
|
5584
|
+
const placeholders2 = chunk.map(() => "(?, ?, ?, ?, ?, ?, ?, ?)").join(", ");
|
|
4252
5585
|
const insert = stmt(
|
|
4253
5586
|
`INSERT INTO refs(from_id, to_name, to_id, call_type, line, lang, module, to_file)
|
|
4254
|
-
VALUES ${
|
|
5587
|
+
VALUES ${placeholders2}`
|
|
4255
5588
|
);
|
|
4256
5589
|
const binds = [];
|
|
4257
5590
|
for (const ref of chunk) {
|
|
@@ -4405,52 +5738,6 @@ function materializeWeightedEdges(edgeMap, idPrefix) {
|
|
|
4405
5738
|
return edges;
|
|
4406
5739
|
}
|
|
4407
5740
|
|
|
4408
|
-
// src/codebase-index/writer-helpers.ts
|
|
4409
|
-
import { resolveWstackPaths } from "@wrongstack/core/utils";
|
|
4410
|
-
function escapeLike(value) {
|
|
4411
|
-
return value.replace(/[\\%_]/g, (char) => `\\${char}`);
|
|
4412
|
-
}
|
|
4413
|
-
function posixIndexPath(file) {
|
|
4414
|
-
return file.replace(/\\/g, "/").replace(/^\.\//, "");
|
|
4415
|
-
}
|
|
4416
|
-
function indexedFileMatchSql(column = "file") {
|
|
4417
|
-
return `(${column} = ? OR replace(${column}, '\\', '/') = ? OR replace(${column}, '\\', '/') LIKE ? ESCAPE '\\')`;
|
|
4418
|
-
}
|
|
4419
|
-
function indexedFileMatchArgs(file) {
|
|
4420
|
-
const posix4 = posixIndexPath(file.trim());
|
|
4421
|
-
return [file, posix4, `%/${escapeLike(posix4)}`];
|
|
4422
|
-
}
|
|
4423
|
-
function matchesIndexedPackageFilter(storedFile, packageLabel, filter) {
|
|
4424
|
-
if (packageLabel === filter) return true;
|
|
4425
|
-
const posixFile = posixIndexPath(storedFile);
|
|
4426
|
-
const posixFilter = posixIndexPath(filter.trim());
|
|
4427
|
-
if (!posixFilter) return false;
|
|
4428
|
-
return posixFile === posixFilter || posixFile.endsWith(`/${posixFilter}`) || posixFile.includes(`/${posixFilter}/`);
|
|
4429
|
-
}
|
|
4430
|
-
function assignRefsToSymbols(refs, symbols) {
|
|
4431
|
-
if (refs.length === 0 || symbols.length === 0) return [];
|
|
4432
|
-
const ordered = [...symbols].sort((a, b) => a.line - b.line || a.col - b.col || a.id - b.id);
|
|
4433
|
-
const seen = /* @__PURE__ */ new Set();
|
|
4434
|
-
const assigned = [];
|
|
4435
|
-
for (const ref of refs) {
|
|
4436
|
-
let owner;
|
|
4437
|
-
for (const symbol of ordered) {
|
|
4438
|
-
if (symbol.line > ref.line) break;
|
|
4439
|
-
owner = symbol;
|
|
4440
|
-
}
|
|
4441
|
-
if (!owner && ref.callType === "import") owner = ordered[0];
|
|
4442
|
-
if (!owner || owner.id <= 0) continue;
|
|
4443
|
-
const key = `${owner.id}:${ref.toName}:${ref.callType}:${ref.module ?? ""}`;
|
|
4444
|
-
if (seen.has(key)) continue;
|
|
4445
|
-
seen.add(key);
|
|
4446
|
-
assigned.push({ ...ref, fromId: owner.id });
|
|
4447
|
-
}
|
|
4448
|
-
return assigned;
|
|
4449
|
-
}
|
|
4450
|
-
function resolveIndexDir(projectRoot, override) {
|
|
4451
|
-
return override ?? resolveWstackPaths({ projectRoot }).projectCodebaseIndex;
|
|
4452
|
-
}
|
|
4453
|
-
|
|
4454
5741
|
// src/codebase-index/writer-ref-mapper.ts
|
|
4455
5742
|
function mapWriterRefRow(row) {
|
|
4456
5743
|
return {
|
|
@@ -4472,20 +5759,20 @@ function mapWriterRefRow(row) {
|
|
|
4472
5759
|
var MAX_SQL_VARS = 900;
|
|
4473
5760
|
function chunkedIdQuery(stmt, ids, buildSql, extraArgs = []) {
|
|
4474
5761
|
const results = [];
|
|
4475
|
-
for (
|
|
4476
|
-
const chunk = ids.slice(
|
|
4477
|
-
|
|
4478
|
-
const sql = buildSql(placeholders);
|
|
5762
|
+
for (const take of inListChunks(ids.length, MAX_SQL_VARS)) {
|
|
5763
|
+
const chunk = padToInBucket(ids.slice(0, take));
|
|
5764
|
+
ids = ids.slice(take);
|
|
5765
|
+
const sql = buildSql(placeholders(chunk.length));
|
|
4479
5766
|
results.push(...stmt(sql).all(...chunk, ...extraArgs));
|
|
4480
5767
|
}
|
|
4481
5768
|
return results;
|
|
4482
5769
|
}
|
|
4483
5770
|
function chunkedIdScalar(stmt, ids, buildSql, extraArgs = []) {
|
|
4484
5771
|
let total = 0;
|
|
4485
|
-
for (
|
|
4486
|
-
const chunk = ids.slice(
|
|
4487
|
-
|
|
4488
|
-
const sql = buildSql(placeholders);
|
|
5772
|
+
for (const take of inListChunks(ids.length, MAX_SQL_VARS)) {
|
|
5773
|
+
const chunk = padToInBucket(ids.slice(0, take));
|
|
5774
|
+
ids = ids.slice(take);
|
|
5775
|
+
const sql = buildSql(placeholders(chunk.length));
|
|
4489
5776
|
const rows = stmt(sql).all(...chunk, ...extraArgs);
|
|
4490
5777
|
total += rows[0]?.n ?? 0;
|
|
4491
5778
|
}
|
|
@@ -4514,16 +5801,24 @@ function resolveIndexedFiles(stmt, file) {
|
|
|
4514
5801
|
}
|
|
4515
5802
|
function resolveSymbolIds(stmt, symbolName, file) {
|
|
4516
5803
|
if (!file) {
|
|
4517
|
-
const
|
|
4518
|
-
|
|
5804
|
+
const rows = stmt("SELECT id FROM symbols WHERE name = ? ORDER BY id").all(
|
|
5805
|
+
symbolName
|
|
5806
|
+
);
|
|
5807
|
+
return rows.map((r) => r.id);
|
|
4519
5808
|
}
|
|
4520
5809
|
const indexedFiles = resolveIndexedFiles(stmt, file);
|
|
4521
5810
|
if (indexedFiles.length === 0) return [];
|
|
4522
|
-
const
|
|
4523
|
-
|
|
4524
|
-
|
|
4525
|
-
|
|
4526
|
-
|
|
5811
|
+
const ids = [];
|
|
5812
|
+
let cursor = 0;
|
|
5813
|
+
for (const take of inListChunks(indexedFiles.length, MAX_SQL_VARS)) {
|
|
5814
|
+
const files = padToInBucket(indexedFiles.slice(cursor, cursor + take));
|
|
5815
|
+
cursor += take;
|
|
5816
|
+
const rows = stmt(
|
|
5817
|
+
`SELECT id FROM symbols WHERE name = ? AND file IN (${placeholders(files.length)}) ORDER BY id`
|
|
5818
|
+
).all(symbolName, ...files);
|
|
5819
|
+
ids.push(...rows.map((r) => r.id));
|
|
5820
|
+
}
|
|
5821
|
+
return ids;
|
|
4527
5822
|
}
|
|
4528
5823
|
function findIncomingCallsByName(stmt, symbolName, file, limit) {
|
|
4529
5824
|
const targetIds = resolveSymbolIds(stmt, symbolName, file);
|
|
@@ -4957,9 +6252,9 @@ function getSymbolGraphWithStatement(stmt, fileFilter) {
|
|
|
4957
6252
|
const loadedIds = new Set(syms.map((s) => s.id));
|
|
4958
6253
|
const missingIds = [...relatedIds].filter((id) => !loadedIds.has(id));
|
|
4959
6254
|
if (missingIds.length > 0) {
|
|
4960
|
-
const
|
|
6255
|
+
const placeholders2 = missingIds.map(() => "?").join(",");
|
|
4961
6256
|
const extras = stmt(
|
|
4962
|
-
`SELECT id, name, kind, lang, file, line, signature, scope FROM symbols WHERE id IN (${
|
|
6257
|
+
`SELECT id, name, kind, lang, file, line, signature, scope FROM symbols WHERE id IN (${placeholders2})`
|
|
4963
6258
|
).all(...missingIds);
|
|
4964
6259
|
for (const s of extras) symById.set(s.id, s);
|
|
4965
6260
|
}
|
|
@@ -4972,81 +6267,6 @@ function getSymbolGraphWithStatement(stmt, fileFilter) {
|
|
|
4972
6267
|
return { nodes, edges };
|
|
4973
6268
|
}
|
|
4974
6269
|
|
|
4975
|
-
// src/codebase-index/vector-search.ts
|
|
4976
|
-
var RRF_K = 60;
|
|
4977
|
-
var VECTOR_DIMENSIONS = 384;
|
|
4978
|
-
var NGRAM_SIZE = 3;
|
|
4979
|
-
function embedText(text) {
|
|
4980
|
-
const vec = new Float32Array(VECTOR_DIMENSIONS);
|
|
4981
|
-
const normalized = text.toLowerCase().trim();
|
|
4982
|
-
if (normalized.length < NGRAM_SIZE) {
|
|
4983
|
-
const padded = ` ${normalized} `.slice(0, Math.max(NGRAM_SIZE, normalized.length + 2));
|
|
4984
|
-
for (let i = 0; i <= padded.length - NGRAM_SIZE; i++) {
|
|
4985
|
-
const ngram = padded.slice(i, i + NGRAM_SIZE);
|
|
4986
|
-
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
4987
|
-
vec[bucket] += 1;
|
|
4988
|
-
}
|
|
4989
|
-
} else {
|
|
4990
|
-
for (let i = 0; i <= normalized.length - NGRAM_SIZE; i++) {
|
|
4991
|
-
const ngram = normalized.slice(i, i + NGRAM_SIZE);
|
|
4992
|
-
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
4993
|
-
vec[bucket] += 1;
|
|
4994
|
-
}
|
|
4995
|
-
}
|
|
4996
|
-
let norm = 0;
|
|
4997
|
-
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
4998
|
-
norm += vec[i] * vec[i];
|
|
4999
|
-
}
|
|
5000
|
-
norm = Math.sqrt(norm);
|
|
5001
|
-
if (norm > 0) {
|
|
5002
|
-
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
5003
|
-
vec[i] /= norm;
|
|
5004
|
-
}
|
|
5005
|
-
}
|
|
5006
|
-
return vec;
|
|
5007
|
-
}
|
|
5008
|
-
function hashNgram(str) {
|
|
5009
|
-
let hash = 2166136261;
|
|
5010
|
-
for (let i = 0; i < str.length; i++) {
|
|
5011
|
-
hash ^= str.charCodeAt(i);
|
|
5012
|
-
hash = Math.imul(hash, 16777619);
|
|
5013
|
-
}
|
|
5014
|
-
return hash >>> 0;
|
|
5015
|
-
}
|
|
5016
|
-
function cosineSimilarity(a, b) {
|
|
5017
|
-
let dot = 0;
|
|
5018
|
-
const len = Math.min(a.length, b.length);
|
|
5019
|
-
for (let i = 0; i < len; i++) {
|
|
5020
|
-
dot += a[i] * b[i];
|
|
5021
|
-
}
|
|
5022
|
-
return dot;
|
|
5023
|
-
}
|
|
5024
|
-
function encodeVector(vec) {
|
|
5025
|
-
return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
|
|
5026
|
-
}
|
|
5027
|
-
function decodeVector(buf) {
|
|
5028
|
-
const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
5029
|
-
const copy = new Float32Array(buf.byteLength / 4);
|
|
5030
|
-
for (let i = 0; i < copy.length; i++) {
|
|
5031
|
-
copy[i] = view.getFloat32(i * 4, true);
|
|
5032
|
-
}
|
|
5033
|
-
return copy;
|
|
5034
|
-
}
|
|
5035
|
-
function reciprocalRankFusion(bm25Ranks, vectorRanks, k = RRF_K) {
|
|
5036
|
-
const allIds = /* @__PURE__ */ new Set([...bm25Ranks.keys(), ...vectorRanks.keys()]);
|
|
5037
|
-
const scored = [];
|
|
5038
|
-
for (const id of allIds) {
|
|
5039
|
-
const bm25Rank = bm25Ranks.get(id);
|
|
5040
|
-
const vecRank = vectorRanks.get(id);
|
|
5041
|
-
let score = 0;
|
|
5042
|
-
if (bm25Rank !== void 0) score += 1 / (k + bm25Rank);
|
|
5043
|
-
if (vecRank !== void 0) score += 1 / (k + vecRank);
|
|
5044
|
-
scored.push([id, score]);
|
|
5045
|
-
}
|
|
5046
|
-
scored.sort((a, b) => b[1] - a[1]);
|
|
5047
|
-
return scored;
|
|
5048
|
-
}
|
|
5049
|
-
|
|
5050
6270
|
// src/codebase-index/writer-mutations.ts
|
|
5051
6271
|
function commitBatchWithStatement(stmtFn, maxSqlVars, ftsAvailable, vectorsAvailable, allocateSymbolIds, invalidateIncomingRefsForFiles, resolveRefsForNamesUnsafe2, entries, options = {}) {
|
|
5052
6272
|
if (entries.length === 0 && (options.deleteForFiles?.length ?? 0) === 0) {
|
|
@@ -5058,24 +6278,29 @@ function commitBatchWithStatement(stmtFn, maxSqlVars, ftsAvailable, vectorsAvail
|
|
|
5058
6278
|
for (const ref of entry.refs) affectedNames.add(ref.toName);
|
|
5059
6279
|
}
|
|
5060
6280
|
if (options.deleteForFiles && options.deleteForFiles.length > 0) {
|
|
5061
|
-
const placeholders = options.deleteForFiles.map(() => "?").join(",");
|
|
5062
6281
|
for (const name of invalidateIncomingRefsForFiles(options.deleteForFiles)) {
|
|
5063
6282
|
affectedNames.add(name);
|
|
5064
6283
|
}
|
|
5065
|
-
|
|
5066
|
-
|
|
5067
|
-
|
|
5068
|
-
|
|
5069
|
-
|
|
5070
|
-
|
|
6284
|
+
let cursor = 0;
|
|
6285
|
+
for (const take of inListChunks(options.deleteForFiles.length, Math.floor(maxSqlVars / 4))) {
|
|
6286
|
+
const bucket = padToInBucket(options.deleteForFiles.slice(cursor, cursor + take));
|
|
6287
|
+
cursor += take;
|
|
6288
|
+
const ph = placeholders(bucket.length);
|
|
6289
|
+
if (ftsAvailable) {
|
|
6290
|
+
stmtFn(
|
|
6291
|
+
`DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file IN (${ph}))`
|
|
6292
|
+
).run(...bucket);
|
|
6293
|
+
}
|
|
6294
|
+
if (vectorsAvailable) {
|
|
6295
|
+
stmtFn(
|
|
6296
|
+
`DELETE FROM symbol_vectors WHERE symbol_id IN (SELECT id FROM symbols WHERE file IN (${ph}))`
|
|
6297
|
+
).run(...bucket);
|
|
6298
|
+
}
|
|
5071
6299
|
stmtFn(
|
|
5072
|
-
`DELETE FROM
|
|
5073
|
-
).run(...
|
|
6300
|
+
`DELETE FROM refs WHERE from_id IN (SELECT id FROM symbols WHERE file IN (${ph}))`
|
|
6301
|
+
).run(...bucket);
|
|
6302
|
+
stmtFn(`DELETE FROM symbols WHERE file IN (${ph})`).run(...bucket);
|
|
5074
6303
|
}
|
|
5075
|
-
stmtFn(
|
|
5076
|
-
`DELETE FROM refs WHERE from_id IN (SELECT id FROM symbols WHERE file IN (${placeholders}))`
|
|
5077
|
-
).run(...options.deleteForFiles);
|
|
5078
|
-
stmtFn(`DELETE FROM symbols WHERE file IN (${placeholders})`).run(...options.deleteForFiles);
|
|
5079
6304
|
}
|
|
5080
6305
|
const totalSymbols = entries.reduce((n, e) => n + e.symbols.length, 0);
|
|
5081
6306
|
let nextId = allocateSymbolIds(totalSymbols);
|
|
@@ -5107,12 +6332,14 @@ function commitBatchWithStatement(stmtFn, maxSqlVars, ftsAvailable, vectorsAvail
|
|
|
5107
6332
|
text: buildIndexableText(s.name, s.signature, s.docComment)
|
|
5108
6333
|
});
|
|
5109
6334
|
}
|
|
5110
|
-
|
|
5111
|
-
|
|
5112
|
-
|
|
5113
|
-
|
|
5114
|
-
|
|
5115
|
-
|
|
6335
|
+
if (vectorsAvailable) {
|
|
6336
|
+
vectorRows.push({
|
|
6337
|
+
id,
|
|
6338
|
+
vector: encodeVector(
|
|
6339
|
+
embedText(s.text || buildIndexableText(s.name, s.signature, s.docComment))
|
|
6340
|
+
)
|
|
6341
|
+
});
|
|
6342
|
+
}
|
|
5116
6343
|
const inserted = { ...s, id };
|
|
5117
6344
|
allInserted.push(inserted);
|
|
5118
6345
|
insertedForEntry.push(inserted);
|
|
@@ -5204,9 +6431,8 @@ var CORE_TABLES_SQL = `
|
|
|
5204
6431
|
signature TEXT NOT NULL DEFAULT '',
|
|
5205
6432
|
doc_comment TEXT NOT NULL DEFAULT '',
|
|
5206
6433
|
scope TEXT NOT NULL DEFAULT '',
|
|
5207
|
-
text TEXT NOT NULL DEFAULT ''
|
|
5208
|
-
|
|
5209
|
-
);
|
|
6434
|
+
text TEXT NOT NULL DEFAULT ''
|
|
6435
|
+
);
|
|
5210
6436
|
`;
|
|
5211
6437
|
var FILE_INDEX_SQL = [
|
|
5212
6438
|
"CREATE INDEX IF NOT EXISTS idx_f_package ON files(package)"
|
|
@@ -5217,7 +6443,6 @@ var SYMBOL_INDEX_SQL = [
|
|
|
5217
6443
|
"CREATE INDEX IF NOT EXISTS idx_s_lang ON symbols(lang)",
|
|
5218
6444
|
"CREATE INDEX IF NOT EXISTS idx_s_file ON symbols(file)",
|
|
5219
6445
|
"CREATE INDEX IF NOT EXISTS idx_s_lang_kind ON symbols(lang, kind)",
|
|
5220
|
-
"CREATE INDEX IF NOT EXISTS idx_s_file_fk ON symbols(file_fk)",
|
|
5221
6446
|
"CREATE INDEX IF NOT EXISTS idx_s_name_id ON symbols(name, id)"
|
|
5222
6447
|
];
|
|
5223
6448
|
var REFS_TABLE_SQL = `
|
|
@@ -5292,11 +6517,12 @@ function getUnresolvedImportsWithStatement(stmtFn, maxSqlVars, onlyFiles) {
|
|
|
5292
6517
|
return stmtFn(base).all();
|
|
5293
6518
|
}
|
|
5294
6519
|
const out = [];
|
|
5295
|
-
|
|
5296
|
-
|
|
5297
|
-
const
|
|
6520
|
+
let cursor = 0;
|
|
6521
|
+
for (const take of inListChunks(onlyFiles.length, maxSqlVars)) {
|
|
6522
|
+
const chunk = padToInBucket(onlyFiles.slice(cursor, cursor + take));
|
|
6523
|
+
cursor += take;
|
|
5298
6524
|
out.push(
|
|
5299
|
-
...stmtFn(`${base} AND s.file IN (${placeholders})`).all(...chunk)
|
|
6525
|
+
...stmtFn(`${base} AND s.file IN (${placeholders(chunk.length)})`).all(...chunk)
|
|
5300
6526
|
);
|
|
5301
6527
|
}
|
|
5302
6528
|
return out;
|
|
@@ -5367,16 +6593,18 @@ function applyImportResolutionsWithStatement(db, stmtFn, runWithRetry, maxSqlVar
|
|
|
5367
6593
|
)`
|
|
5368
6594
|
);
|
|
5369
6595
|
const chunkSize = Math.max(1, Math.floor(maxSqlVars / 4));
|
|
5370
|
-
|
|
5371
|
-
|
|
5372
|
-
const
|
|
6596
|
+
let cursor = 0;
|
|
6597
|
+
for (const take of ladderChunkSizes(resolutions.length, chunkSize)) {
|
|
6598
|
+
const chunk = resolutions.slice(cursor, cursor + take);
|
|
6599
|
+
cursor += take;
|
|
6600
|
+
const valuesPh = chunk.map(() => "(?, ?, ?, ?)").join(", ");
|
|
5373
6601
|
const binds = [];
|
|
5374
6602
|
for (const entry of chunk) {
|
|
5375
6603
|
binds.push(entry.fromFile, entry.lang, entry.module, entry.toFile);
|
|
5376
6604
|
}
|
|
5377
6605
|
stmtFn(
|
|
5378
6606
|
`INSERT INTO temp.import_resolution(from_file, lang, module, to_file)
|
|
5379
|
-
VALUES ${
|
|
6607
|
+
VALUES ${valuesPh}`
|
|
5380
6608
|
).run(...binds);
|
|
5381
6609
|
}
|
|
5382
6610
|
db.exec(
|
|
@@ -5413,9 +6641,11 @@ function resolveRefsForNamesUnsafe(stmtFn, maxSqlVars, names) {
|
|
|
5413
6641
|
const list = [...names].filter((name) => name.length > 0);
|
|
5414
6642
|
if (list.length === 0) return 0;
|
|
5415
6643
|
let total = 0;
|
|
5416
|
-
|
|
5417
|
-
|
|
5418
|
-
const
|
|
6644
|
+
let cursor = 0;
|
|
6645
|
+
for (const take of inListChunks(list.length, maxSqlVars)) {
|
|
6646
|
+
const chunk = padToInBucket(list.slice(cursor, cursor + take));
|
|
6647
|
+
cursor += take;
|
|
6648
|
+
const ph = placeholders(chunk.length);
|
|
5419
6649
|
try {
|
|
5420
6650
|
const result = stmtFn(
|
|
5421
6651
|
`UPDATE refs
|
|
@@ -5424,16 +6654,16 @@ function resolveRefsForNamesUnsafe(stmtFn, maxSqlVars, names) {
|
|
|
5424
6654
|
SELECT sym.name AS name, lf.family AS family, MIN(sym.id) AS id
|
|
5425
6655
|
FROM symbols sym
|
|
5426
6656
|
JOIN lang_family lf ON lf.lang = sym.lang
|
|
5427
|
-
WHERE sym.name IN (${
|
|
6657
|
+
WHERE sym.name IN (${ph})
|
|
5428
6658
|
GROUP BY sym.name, lf.family
|
|
5429
6659
|
UNION ALL
|
|
5430
6660
|
SELECT sym.name AS name, '${LANG_FAMILY_WILDCARD}' AS family, MIN(sym.id) AS id
|
|
5431
6661
|
FROM symbols sym
|
|
5432
|
-
WHERE sym.name IN (${
|
|
6662
|
+
WHERE sym.name IN (${ph})
|
|
5433
6663
|
GROUP BY sym.name
|
|
5434
6664
|
) AS s,
|
|
5435
6665
|
lang_family AS rf
|
|
5436
|
-
WHERE refs.to_name IN (${
|
|
6666
|
+
WHERE refs.to_name IN (${ph})
|
|
5437
6667
|
AND rf.lang = refs.lang
|
|
5438
6668
|
AND s.name = refs.to_name
|
|
5439
6669
|
AND s.family = rf.family`
|
|
@@ -5445,7 +6675,7 @@ function resolveRefsForNamesUnsafe(stmtFn, maxSqlVars, names) {
|
|
|
5445
6675
|
SELECT sym.id FROM symbols sym
|
|
5446
6676
|
WHERE sym.name = refs.to_name AND ${FAMILY_MATCH_SQL}
|
|
5447
6677
|
ORDER BY sym.id LIMIT 1
|
|
5448
|
-
) WHERE refs.to_name IN (${
|
|
6678
|
+
) WHERE refs.to_name IN (${ph})
|
|
5449
6679
|
AND EXISTS (
|
|
5450
6680
|
SELECT 1 FROM symbols sym
|
|
5451
6681
|
WHERE sym.name = refs.to_name AND ${FAMILY_MATCH_SQL}
|
|
@@ -5502,7 +6732,7 @@ function buildWriterSearchWhere(query, filter) {
|
|
|
5502
6732
|
const conditions = [];
|
|
5503
6733
|
const values = [];
|
|
5504
6734
|
let effectiveKind = filter?.kind;
|
|
5505
|
-
if (filter?.lspKind
|
|
6735
|
+
if (filter?.lspKind != null) {
|
|
5506
6736
|
const mapped = lspKindToInternalKind(filter.lspKind);
|
|
5507
6737
|
if (mapped !== null) {
|
|
5508
6738
|
effectiveKind = mapped;
|
|
@@ -5581,7 +6811,7 @@ function searchRankedWithStatement(stmtFn, searchFn, ftsAvailable, vectorsAvaila
|
|
|
5581
6811
|
);
|
|
5582
6812
|
}
|
|
5583
6813
|
let effectiveKind = filter?.kind;
|
|
5584
|
-
if (filter?.lspKind
|
|
6814
|
+
if (filter?.lspKind != null) {
|
|
5585
6815
|
const mapped = lspKindToInternalKind(filter.lspKind);
|
|
5586
6816
|
if (mapped === null) return { results: [], total: 0 };
|
|
5587
6817
|
effectiveKind = mapped;
|
|
@@ -5618,15 +6848,14 @@ function searchRankedWithStatement(stmtFn, searchFn, ftsAvailable, vectorsAvaila
|
|
|
5618
6848
|
values.push(`%${escapeLike(filter.file.replace(/\\/g, "/"))}%`);
|
|
5619
6849
|
}
|
|
5620
6850
|
const where = conditions.join(" AND ");
|
|
5621
|
-
const countRows = stmtFn(
|
|
5622
|
-
`SELECT COUNT(*) AS n FROM symbols_fts JOIN symbols s ON s.id = symbols_fts.rowid WHERE ${where}`
|
|
5623
|
-
).all(...values);
|
|
5624
|
-
const total = countRows[0] ? Number(countRows[0].n) : 0;
|
|
5625
|
-
if (total === 0) return { results: [], total: 0 };
|
|
5626
6851
|
const bm25Rows = stmtFn(
|
|
5627
6852
|
`SELECT s.id, s.lang, s.kind, s.name, s.file, s.line, s.col, s.signature, s.doc_comment,
|
|
5628
6853
|
-bm25(symbols_fts) AS score,
|
|
5629
|
-
snippet(symbols_fts, 0, '', '', '\u2026', 12) AS snippet
|
|
6854
|
+
snippet(symbols_fts, 0, '', '', '\u2026', 12) AS snippet,
|
|
6855
|
+
-- Keep this uncorrelated: referencing outer columns turns it into
|
|
6856
|
+
-- a per-row subquery and defeats the one-count-per-statement win.
|
|
6857
|
+
(SELECT COUNT(*) FROM symbols_fts JOIN symbols s ON s.id = symbols_fts.rowid
|
|
6858
|
+
WHERE ${where}) AS total_count
|
|
5630
6859
|
FROM symbols_fts JOIN symbols s ON s.id = symbols_fts.rowid
|
|
5631
6860
|
WHERE ${where}
|
|
5632
6861
|
ORDER BY
|
|
@@ -5635,13 +6864,15 @@ function searchRankedWithStatement(stmtFn, searchFn, ftsAvailable, vectorsAvaila
|
|
|
5635
6864
|
ELSE 2 END,
|
|
5636
6865
|
bm25(symbols_fts), lower(s.name), s.file, s.line, s.col, s.id
|
|
5637
6866
|
LIMIT ?`
|
|
5638
|
-
).all(...values, query.trim(), `${escapeLike(query.trim())}%`, safeLimit);
|
|
5639
|
-
if (
|
|
6867
|
+
).all(...values, ...values, query.trim(), `${escapeLike(query.trim())}%`, safeLimit);
|
|
6868
|
+
if (bm25Rows.length === 0) return { results: [], total: 0 };
|
|
6869
|
+
const total = Number(bm25Rows[0]?.total_count ?? 0);
|
|
6870
|
+
if (vectorsAvailable) {
|
|
5640
6871
|
const queryVec = embedText(query);
|
|
5641
6872
|
const candidateIds = bm25Rows.map((r) => r.id);
|
|
5642
|
-
const
|
|
6873
|
+
const placeholders2 = candidateIds.map(() => "?").join(",");
|
|
5643
6874
|
const vecRows = stmtFn(
|
|
5644
|
-
`SELECT sv.symbol_id, sv.vector FROM symbol_vectors sv WHERE sv.symbol_id IN (${
|
|
6875
|
+
`SELECT sv.symbol_id, sv.vector FROM symbol_vectors sv WHERE sv.symbol_id IN (${placeholders2})`
|
|
5645
6876
|
).all(...candidateIds);
|
|
5646
6877
|
const vecScores = vecRows.map((r) => ({
|
|
5647
6878
|
id: r.symbol_id,
|
|
@@ -5829,9 +7060,9 @@ var IndexStore = class _IndexStore {
|
|
|
5829
7060
|
}
|
|
5830
7061
|
constructor(projectRoot, opts = {}) {
|
|
5831
7062
|
this.indexDir = resolveIndexDir(projectRoot, opts.indexDir);
|
|
5832
|
-
|
|
7063
|
+
fs9.mkdirSync(this.indexDir, { recursive: true });
|
|
5833
7064
|
const Database = loadDatabaseSync();
|
|
5834
|
-
this.db = new Database(
|
|
7065
|
+
this.db = new Database(path12.join(this.indexDir, DB_FILE2));
|
|
5835
7066
|
applyIndexStorePragmas(this.db);
|
|
5836
7067
|
this.initSchema();
|
|
5837
7068
|
}
|
|
@@ -5960,7 +7191,11 @@ var IndexStore = class _IndexStore {
|
|
|
5960
7191
|
);
|
|
5961
7192
|
if (symbolCount !== ftsCount) {
|
|
5962
7193
|
this.db.exec("DELETE FROM symbols_fts");
|
|
5963
|
-
if (
|
|
7194
|
+
if (vectorEmbeddingEnabled() && this.stmt(
|
|
7195
|
+
"SELECT 1 FROM sqlite_master WHERE type='table' AND name='symbol_vectors'"
|
|
7196
|
+
).get() !== void 0) {
|
|
7197
|
+
this.db.exec("DELETE FROM symbol_vectors");
|
|
7198
|
+
}
|
|
5964
7199
|
const rows = this.stmt(
|
|
5965
7200
|
"SELECT id, name, signature, doc_comment FROM symbols ORDER BY id"
|
|
5966
7201
|
).all();
|
|
@@ -5979,8 +7214,13 @@ var IndexStore = class _IndexStore {
|
|
|
5979
7214
|
this.ftsAvailable = false;
|
|
5980
7215
|
}
|
|
5981
7216
|
try {
|
|
5982
|
-
|
|
5983
|
-
|
|
7217
|
+
if (vectorEmbeddingEnabled()) {
|
|
7218
|
+
this.db.exec(SYMBOL_VECTORS_TABLE_SQL);
|
|
7219
|
+
this.vectorsAvailable = true;
|
|
7220
|
+
} else {
|
|
7221
|
+
this.db.exec("DROP TABLE IF EXISTS symbol_vectors");
|
|
7222
|
+
this.vectorsAvailable = false;
|
|
7223
|
+
}
|
|
5984
7224
|
} catch {
|
|
5985
7225
|
this.vectorsAvailable = false;
|
|
5986
7226
|
}
|
|
@@ -6015,14 +7255,22 @@ var IndexStore = class _IndexStore {
|
|
|
6015
7255
|
}
|
|
6016
7256
|
invalidateIncomingRefsForFiles(files) {
|
|
6017
7257
|
if (files.length === 0) return /* @__PURE__ */ new Set();
|
|
6018
|
-
const
|
|
6019
|
-
|
|
6020
|
-
|
|
6021
|
-
|
|
6022
|
-
|
|
6023
|
-
|
|
6024
|
-
|
|
6025
|
-
|
|
7258
|
+
const names = [];
|
|
7259
|
+
let cursor = 0;
|
|
7260
|
+
for (const take of inListChunks(files.length, _IndexStore.MAX_SQL_VARS)) {
|
|
7261
|
+
const bucket = padToInBucket(files.slice(cursor, cursor + take));
|
|
7262
|
+
cursor += take;
|
|
7263
|
+
const ph = placeholders(bucket.length);
|
|
7264
|
+
for (const row of this.stmt(`SELECT DISTINCT name FROM symbols WHERE file IN (${ph})`).all(
|
|
7265
|
+
...bucket
|
|
7266
|
+
)) {
|
|
7267
|
+
names.push(row.name);
|
|
7268
|
+
}
|
|
7269
|
+
this.stmt(
|
|
7270
|
+
`UPDATE refs SET to_id = NULL
|
|
7271
|
+
WHERE to_id IN (SELECT id FROM symbols WHERE file IN (${ph}))`
|
|
7272
|
+
).run(...bucket);
|
|
7273
|
+
}
|
|
6026
7274
|
return new Set(names);
|
|
6027
7275
|
}
|
|
6028
7276
|
resolveRefsForNamesUnsafe(names) {
|
|
@@ -6056,6 +7304,14 @@ var IndexStore = class _IndexStore {
|
|
|
6056
7304
|
if (this.ftsAvailable) {
|
|
6057
7305
|
ftsRows.push({ id, text: buildIndexableText(s.name, s.signature, s.docComment) });
|
|
6058
7306
|
}
|
|
7307
|
+
if (this.vectorsAvailable) {
|
|
7308
|
+
vectorRows.push({
|
|
7309
|
+
id,
|
|
7310
|
+
vector: encodeVector(
|
|
7311
|
+
embedText(s.text || buildIndexableText(s.name, s.signature, s.docComment))
|
|
7312
|
+
)
|
|
7313
|
+
});
|
|
7314
|
+
}
|
|
6059
7315
|
result.push({ ...s, id });
|
|
6060
7316
|
}
|
|
6061
7317
|
bulkInsertSymbolsWithStatement((sql) => this.stmt(sql), _IndexStore.MAX_SQL_VARS, bulk);
|
|
@@ -6088,15 +7344,15 @@ var IndexStore = class _IndexStore {
|
|
|
6088
7344
|
const affectedNames = this.invalidateIncomingRefsForFiles([file]);
|
|
6089
7345
|
if (this.ftsAvailable) {
|
|
6090
7346
|
this.stmt(
|
|
6091
|
-
"DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE
|
|
7347
|
+
"DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file = ?)"
|
|
6092
7348
|
).run(file);
|
|
6093
7349
|
}
|
|
6094
7350
|
if (this.vectorsAvailable) {
|
|
6095
7351
|
this.stmt(
|
|
6096
|
-
"DELETE FROM symbol_vectors WHERE symbol_id IN (SELECT id FROM symbols WHERE
|
|
7352
|
+
"DELETE FROM symbol_vectors WHERE symbol_id IN (SELECT id FROM symbols WHERE file = ?)"
|
|
6097
7353
|
).run(file);
|
|
6098
7354
|
}
|
|
6099
|
-
this.stmt("DELETE FROM symbols WHERE
|
|
7355
|
+
this.stmt("DELETE FROM symbols WHERE file = ?").run(file);
|
|
6100
7356
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
6101
7357
|
this.commitWriteTransaction(ownsTransaction);
|
|
6102
7358
|
} catch (error) {
|
|
@@ -6113,18 +7369,18 @@ var IndexStore = class _IndexStore {
|
|
|
6113
7369
|
const affectedNames = this.invalidateIncomingRefsForFiles([file]);
|
|
6114
7370
|
if (this.ftsAvailable) {
|
|
6115
7371
|
this.stmt(
|
|
6116
|
-
"DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE
|
|
7372
|
+
"DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file = ?)"
|
|
6117
7373
|
).run(file);
|
|
6118
7374
|
}
|
|
6119
7375
|
if (this.vectorsAvailable) {
|
|
6120
7376
|
this.stmt(
|
|
6121
|
-
"DELETE FROM symbol_vectors WHERE symbol_id IN (SELECT id FROM symbols WHERE
|
|
7377
|
+
"DELETE FROM symbol_vectors WHERE symbol_id IN (SELECT id FROM symbols WHERE file = ?)"
|
|
6122
7378
|
).run(file);
|
|
6123
7379
|
}
|
|
6124
|
-
this.stmt(
|
|
6125
|
-
|
|
6126
|
-
)
|
|
6127
|
-
this.stmt("DELETE FROM symbols WHERE
|
|
7380
|
+
this.stmt("DELETE FROM refs WHERE from_id IN (SELECT id FROM symbols WHERE file = ?)").run(
|
|
7381
|
+
file
|
|
7382
|
+
);
|
|
7383
|
+
this.stmt("DELETE FROM symbols WHERE file = ?").run(file);
|
|
6128
7384
|
this.stmt("DELETE FROM files WHERE file = ?").run(file);
|
|
6129
7385
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
6130
7386
|
this.commitWriteTransaction(ownsTransaction);
|
|
@@ -6228,6 +7484,10 @@ var IndexStore = class _IndexStore {
|
|
|
6228
7484
|
getStats() {
|
|
6229
7485
|
return getStatsWithStatement((sql) => this.stmt(sql), this.indexDir);
|
|
6230
7486
|
}
|
|
7487
|
+
/** P2.5: minimal summary for search-response piggyback (see writer-admin). */
|
|
7488
|
+
getIndexSummary() {
|
|
7489
|
+
return getIndexSummaryWithStatement((sql) => this.stmt(sql));
|
|
7490
|
+
}
|
|
6231
7491
|
setLastIndexed(ts2) {
|
|
6232
7492
|
this.runWithRetry(() => {
|
|
6233
7493
|
this.stmt("INSERT OR REPLACE INTO metadata(key, value) VALUES('last_indexed', ?)").run(
|
|
@@ -6329,18 +7589,18 @@ var IndexStore = class _IndexStore {
|
|
|
6329
7589
|
const affectedNames = this.invalidateIncomingRefsForFiles([meta.file]);
|
|
6330
7590
|
if (this.ftsAvailable) {
|
|
6331
7591
|
this.stmt(
|
|
6332
|
-
"DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE
|
|
7592
|
+
"DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file = ?)"
|
|
6333
7593
|
).run(meta.file);
|
|
6334
7594
|
}
|
|
6335
7595
|
if (this.vectorsAvailable) {
|
|
6336
7596
|
this.stmt(
|
|
6337
|
-
"DELETE FROM symbol_vectors WHERE symbol_id IN (SELECT id FROM symbols WHERE
|
|
7597
|
+
"DELETE FROM symbol_vectors WHERE symbol_id IN (SELECT id FROM symbols WHERE file = ?)"
|
|
6338
7598
|
).run(meta.file);
|
|
6339
7599
|
}
|
|
6340
|
-
this.stmt(
|
|
6341
|
-
|
|
6342
|
-
)
|
|
6343
|
-
this.stmt("DELETE FROM symbols WHERE
|
|
7600
|
+
this.stmt("DELETE FROM refs WHERE from_id IN (SELECT id FROM symbols WHERE file = ?)").run(
|
|
7601
|
+
meta.file
|
|
7602
|
+
);
|
|
7603
|
+
this.stmt("DELETE FROM symbols WHERE file = ?").run(meta.file);
|
|
6344
7604
|
this.stmt(
|
|
6345
7605
|
`INSERT INTO files(file, lang, mtime_ms, content_hash, symbol_count, last_indexed)
|
|
6346
7606
|
VALUES (?, ?, ?, ?, ?, ?)
|
|
@@ -6372,6 +7632,27 @@ var IndexStore = class _IndexStore {
|
|
|
6372
7632
|
} catch {
|
|
6373
7633
|
}
|
|
6374
7634
|
}
|
|
7635
|
+
/**
|
|
7636
|
+
* P4.14: best-effort WAL checkpoint for idle-time maintenance.
|
|
7637
|
+
*
|
|
7638
|
+
* `wal_autocheckpoint` is PASSIVE and only attempts work after a COMMIT —
|
|
7639
|
+
* once writes stop, nothing fires again, so the WAL keeps whatever frames
|
|
7640
|
+
* the last burst left. This probes with PASSIVE first (never blocks; busy=1
|
|
7641
|
+
* means readers still hold WAL snapshots) and only issues the TRUNCATE —
|
|
7642
|
+
* which resets index.db-wal to zero bytes — when the checkpointer can
|
|
7643
|
+
* proceed immediately. Callers run this on the daemon's single thread, so
|
|
7644
|
+
* never wait on readers here: busy means "retry at the next idle window".
|
|
7645
|
+
*/
|
|
7646
|
+
checkpointWal() {
|
|
7647
|
+
try {
|
|
7648
|
+
const probe = this.db.prepare("PRAGMA wal_checkpoint(PASSIVE)").get();
|
|
7649
|
+
if (Number(probe?.busy ?? 1) !== 0) return false;
|
|
7650
|
+
const done = this.db.prepare("PRAGMA wal_checkpoint(TRUNCATE)").get();
|
|
7651
|
+
return Number(done?.busy ?? 1) === 0;
|
|
7652
|
+
} catch {
|
|
7653
|
+
return false;
|
|
7654
|
+
}
|
|
7655
|
+
}
|
|
6375
7656
|
compactIfNeeded(options = {}) {
|
|
6376
7657
|
const minBytes = options.minBytes ?? 256 * 1024 * 1024;
|
|
6377
7658
|
const minFreeRatio = options.minFreeRatio ?? 0.35;
|
|
@@ -6457,7 +7738,9 @@ function resolveParallelBatch() {
|
|
|
6457
7738
|
return indexParallelBatchSize(availableParallelism());
|
|
6458
7739
|
}
|
|
6459
7740
|
function shouldUseParserWorkerPool(candidateFileCount, parseBatchCount) {
|
|
6460
|
-
|
|
7741
|
+
const threshold = resolveWorkerPoolThreshold();
|
|
7742
|
+
if (threshold === 0) return false;
|
|
7743
|
+
return !isFrugalPerf() && candidateFileCount >= threshold && parseBatchCount > 1;
|
|
6461
7744
|
}
|
|
6462
7745
|
function yieldEventLoop() {
|
|
6463
7746
|
return new Promise((resolve2) => setImmediate(resolve2));
|
|
@@ -6480,15 +7763,15 @@ var IndexSourceChangedError = class extends Error {
|
|
|
6480
7763
|
name = "IndexSourceChangedError";
|
|
6481
7764
|
};
|
|
6482
7765
|
function isWithinProject(projectRoot, file) {
|
|
6483
|
-
const rel =
|
|
6484
|
-
return rel !== "" && !rel.startsWith(`..${
|
|
7766
|
+
const rel = path13.relative(projectRoot, file);
|
|
7767
|
+
return rel !== "" && !rel.startsWith(`..${path13.sep}`) && rel !== ".." && !path13.isAbsolute(rel);
|
|
6485
7768
|
}
|
|
6486
7769
|
function isMissingPathError(err) {
|
|
6487
7770
|
const code = err?.code;
|
|
6488
7771
|
return code === "ENOENT" || code === "ENOTDIR";
|
|
6489
7772
|
}
|
|
6490
7773
|
function normalizeComparablePath(value) {
|
|
6491
|
-
const resolved =
|
|
7774
|
+
const resolved = path13.resolve(value);
|
|
6492
7775
|
return process.platform === "win32" ? resolved.toLowerCase() : resolved;
|
|
6493
7776
|
}
|
|
6494
7777
|
function gitOutput(projectRoot, args) {
|
|
@@ -6534,24 +7817,24 @@ async function findGitSourceFiles(projectRoot, ignore, signal) {
|
|
|
6534
7817
|
const record = statusRecords[i];
|
|
6535
7818
|
if (!record) continue;
|
|
6536
7819
|
const status = record.slice(0, 2);
|
|
6537
|
-
const changedPath =
|
|
7820
|
+
const changedPath = path13.resolve(projectRoot, record.slice(3));
|
|
6538
7821
|
dirty.add(changedPath);
|
|
6539
7822
|
if (status.includes("D")) deleted.add(changedPath);
|
|
6540
7823
|
if (status.includes("R") || status.includes("C")) {
|
|
6541
7824
|
const source = statusRecords[++i];
|
|
6542
|
-
if (source) dirty.add(
|
|
7825
|
+
if (source) dirty.add(path13.resolve(projectRoot, source));
|
|
6543
7826
|
}
|
|
6544
7827
|
}
|
|
6545
7828
|
const files = [];
|
|
6546
7829
|
for (const relative2 of output.toString("utf8").split("\0")) {
|
|
6547
7830
|
if (!relative2) continue;
|
|
6548
7831
|
const portable = relative2.replace(/\\/g, "/");
|
|
6549
|
-
if (portable.split("/").some((segment) => ignoreSet.has(segment)) || DEFAULT_IGNORE_FILES.has(
|
|
7832
|
+
if (portable.split("/").some((segment) => ignoreSet.has(segment)) || DEFAULT_IGNORE_FILES.has(path13.posix.basename(portable))) {
|
|
6550
7833
|
continue;
|
|
6551
7834
|
}
|
|
6552
|
-
const full =
|
|
7835
|
+
const full = path13.resolve(projectRoot, relative2);
|
|
6553
7836
|
if (deleted.has(full)) continue;
|
|
6554
|
-
const ext =
|
|
7837
|
+
const ext = path13.extname(relative2).toLowerCase();
|
|
6555
7838
|
if (INDEXABLE_EXTENSION_SET.has(ext) || detectLang(full) !== null) files.push(full);
|
|
6556
7839
|
}
|
|
6557
7840
|
const snapshot = createHash("sha256").update(stagedOutput).update("\0").update(statusOutput);
|
|
@@ -6559,7 +7842,7 @@ async function findGitSourceFiles(projectRoot, ignore, signal) {
|
|
|
6559
7842
|
for (const dirtyFile of [...dirty].sort()) {
|
|
6560
7843
|
if (!indexedFiles.has(dirtyFile) || deleted.has(dirtyFile)) continue;
|
|
6561
7844
|
snapshot.update("\0").update(dirtyFile).update("\0");
|
|
6562
|
-
snapshot.update(xxhash64String(await
|
|
7845
|
+
snapshot.update(xxhash64String(await fs10.readFile(dirtyFile, "utf8")));
|
|
6563
7846
|
}
|
|
6564
7847
|
return {
|
|
6565
7848
|
files,
|
|
@@ -6595,7 +7878,7 @@ async function findSourceFiles(projectRoot, ignore, isGitIgnored, signal) {
|
|
|
6595
7878
|
}
|
|
6596
7879
|
let entries;
|
|
6597
7880
|
try {
|
|
6598
|
-
entries = await
|
|
7881
|
+
entries = await fs10.readdir(dir, { withFileTypes: true });
|
|
6599
7882
|
} catch (err) {
|
|
6600
7883
|
complete = false;
|
|
6601
7884
|
errors.push(`scan error: ${dir}: ${err instanceof Error ? err.message : String(err)}`);
|
|
@@ -6604,14 +7887,14 @@ async function findSourceFiles(projectRoot, ignore, isGitIgnored, signal) {
|
|
|
6604
7887
|
dirCount++;
|
|
6605
7888
|
for (const e of entries) {
|
|
6606
7889
|
if (ignoreSet.has(e.name)) continue;
|
|
6607
|
-
const full =
|
|
6608
|
-
const rel =
|
|
7890
|
+
const full = path13.join(dir, e.name);
|
|
7891
|
+
const rel = path13.relative(projectRoot, full).replace(/\\/g, "/");
|
|
6609
7892
|
if (e.isDirectory()) {
|
|
6610
7893
|
if (isGitIgnored(rel, true)) continue;
|
|
6611
7894
|
await walk(full);
|
|
6612
7895
|
} else if (e.isFile()) {
|
|
6613
7896
|
if (DEFAULT_IGNORE_FILES.has(e.name) || isGitIgnored(rel, false)) continue;
|
|
6614
|
-
const ext =
|
|
7897
|
+
const ext = path13.extname(e.name).toLowerCase();
|
|
6615
7898
|
if (indexableExts.has(ext) || detectLang(full) !== null) {
|
|
6616
7899
|
results.push(full);
|
|
6617
7900
|
}
|
|
@@ -6697,10 +7980,10 @@ async function runIndexerAtomic(store, opts) {
|
|
|
6697
7980
|
let trustedUnchanged;
|
|
6698
7981
|
let discoverySnapshotKey;
|
|
6699
7982
|
if (opts.files && opts.files.length > 0) {
|
|
6700
|
-
files = opts.files.map((f) =>
|
|
7983
|
+
files = opts.files.map((f) => path13.resolve(projectRoot, f)).filter((f) => {
|
|
6701
7984
|
if (!isWithinProject(projectRoot, f)) return false;
|
|
6702
|
-
const rel =
|
|
6703
|
-
return !rel.split("/").some((seg) => DEFAULT_IGNORE.includes(seg)) && !DEFAULT_IGNORE_FILES.has(
|
|
7985
|
+
const rel = path13.relative(projectRoot, f).replace(/\\/g, "/");
|
|
7986
|
+
return !rel.split("/").some((seg) => DEFAULT_IGNORE.includes(seg)) && !DEFAULT_IGNORE_FILES.has(path13.basename(f)) && !isGitIgnored(rel, false);
|
|
6704
7987
|
});
|
|
6705
7988
|
} else {
|
|
6706
7989
|
const discovery = await findSourceFiles(projectRoot, ignore, isGitIgnored, signal);
|
|
@@ -6733,7 +8016,6 @@ async function runIndexerAtomic(store, opts) {
|
|
|
6733
8016
|
if (!meta || !trustedUnchanged.has(file)) return true;
|
|
6734
8017
|
langStats[meta.lang] = (langStats[meta.lang] ?? 0) + meta.symbolCount;
|
|
6735
8018
|
symbolsIndexed += meta.symbolCount;
|
|
6736
|
-
filesIndexed++;
|
|
6737
8019
|
filesSkipped++;
|
|
6738
8020
|
filesPreSkipped++;
|
|
6739
8021
|
return false;
|
|
@@ -6762,7 +8044,7 @@ async function runIndexerAtomic(store, opts) {
|
|
|
6762
8044
|
async (file) => {
|
|
6763
8045
|
let stat2;
|
|
6764
8046
|
try {
|
|
6765
|
-
stat2 = await
|
|
8047
|
+
stat2 = await fs10.stat(file, statOpts);
|
|
6766
8048
|
} catch (e) {
|
|
6767
8049
|
if (isAbortError(e)) throw e;
|
|
6768
8050
|
return {
|
|
@@ -6789,7 +8071,7 @@ async function runIndexerAtomic(store, opts) {
|
|
|
6789
8071
|
const meta = existingMeta.get(file);
|
|
6790
8072
|
let content;
|
|
6791
8073
|
try {
|
|
6792
|
-
content = await
|
|
8074
|
+
content = await fs10.readFile(file, { encoding: "utf8", signal });
|
|
6793
8075
|
} catch (e) {
|
|
6794
8076
|
if (isAbortError(e)) throw e;
|
|
6795
8077
|
return {
|
|
@@ -6854,22 +8136,19 @@ async function runIndexerAtomic(store, opts) {
|
|
|
6854
8136
|
}
|
|
6855
8137
|
}
|
|
6856
8138
|
if (!pool) {
|
|
6857
|
-
await
|
|
6858
|
-
toParse.map(
|
|
6859
|
-
try {
|
|
6860
|
-
const parsed = await parseFileContent(item.file, item.content, item.lang);
|
|
6861
|
-
const settled = statReadParse[item.index];
|
|
6862
|
-
if (settled.status === "fulfilled") {
|
|
6863
|
-
settled.value.parsed = parsed;
|
|
6864
|
-
}
|
|
6865
|
-
} catch (e) {
|
|
6866
|
-
const settled = statReadParse[item.index];
|
|
6867
|
-
if (settled.status === "fulfilled") {
|
|
6868
|
-
settled.value.error = `parse error: ${e instanceof Error ? e.message : String(e)}`;
|
|
6869
|
-
}
|
|
6870
|
-
}
|
|
6871
|
-
})
|
|
8139
|
+
const parsedAll = await parseFilesContent(
|
|
8140
|
+
toParse.map((p) => ({ file: p.file, content: p.content, lang: p.lang }))
|
|
6872
8141
|
);
|
|
8142
|
+
for (let pi2 = 0; pi2 < parsedAll.length && pi2 < toParse.length; pi2++) {
|
|
8143
|
+
const settled = statReadParse[toParse[pi2].index];
|
|
8144
|
+
if (settled.status !== "fulfilled") continue;
|
|
8145
|
+
const slot = parsedAll[pi2];
|
|
8146
|
+
if (slot.result) {
|
|
8147
|
+
settled.value.parsed = slot.result;
|
|
8148
|
+
} else {
|
|
8149
|
+
settled.value.error = `parse error: ${slot.error ?? `no result for ${toParse[pi2].file}`}`;
|
|
8150
|
+
}
|
|
8151
|
+
}
|
|
6873
8152
|
}
|
|
6874
8153
|
}
|
|
6875
8154
|
const batchEntries = [];
|
|
@@ -6895,7 +8174,6 @@ async function runIndexerAtomic(store, opts) {
|
|
|
6895
8174
|
if (result.skippedMeta) {
|
|
6896
8175
|
langStats[lang] = (langStats[lang] ?? 0) + result.skippedMeta.symbolCount;
|
|
6897
8176
|
symbolsIndexed += result.skippedMeta.symbolCount;
|
|
6898
|
-
filesIndexed++;
|
|
6899
8177
|
filesSkipped++;
|
|
6900
8178
|
const stored = existingMeta.get(file);
|
|
6901
8179
|
if (stored && stored.mtimeMs !== result.skippedMeta.mtimeMs) {
|
|
@@ -6920,7 +8198,6 @@ async function runIndexerAtomic(store, opts) {
|
|
|
6920
8198
|
lastIndexed: Date.now(),
|
|
6921
8199
|
contentHash: result.contentHash ?? ""
|
|
6922
8200
|
});
|
|
6923
|
-
filesIndexed++;
|
|
6924
8201
|
filesEmpty++;
|
|
6925
8202
|
}
|
|
6926
8203
|
continue;
|
|
@@ -6934,7 +8211,6 @@ async function runIndexerAtomic(store, opts) {
|
|
|
6934
8211
|
lastIndexed: Date.now(),
|
|
6935
8212
|
contentHash: result.contentHash ?? ""
|
|
6936
8213
|
});
|
|
6937
|
-
filesIndexed++;
|
|
6938
8214
|
filesEmpty++;
|
|
6939
8215
|
continue;
|
|
6940
8216
|
}
|
|
@@ -7064,7 +8340,7 @@ async function indexService(args, hooks = {}) {
|
|
|
7064
8340
|
function searchService(args) {
|
|
7065
8341
|
const store = indexStorePool.acquire(args.projectRoot, { indexDir: args.indexDir });
|
|
7066
8342
|
try {
|
|
7067
|
-
|
|
8343
|
+
const result = store.searchRanked(
|
|
7068
8344
|
args.query,
|
|
7069
8345
|
{
|
|
7070
8346
|
kind: args.kind,
|
|
@@ -7074,6 +8350,10 @@ function searchService(args) {
|
|
|
7074
8350
|
},
|
|
7075
8351
|
args.limit
|
|
7076
8352
|
);
|
|
8353
|
+
if (result.total === 0) {
|
|
8354
|
+
return { ...result, indexSummary: store.getIndexSummary() };
|
|
8355
|
+
}
|
|
8356
|
+
return result;
|
|
7077
8357
|
} finally {
|
|
7078
8358
|
indexStorePool.release(store);
|
|
7079
8359
|
}
|