@wrongstack/tools 0.302.2 → 0.305.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/builtin.d.ts +6 -0
- package/dist/builtin.js +3213 -686
- package/dist/codebase-index/binary-frame.d.ts +43 -0
- package/dist/codebase-index/codebase-incoming-calls-tool.d.ts +1 -0
- package/dist/codebase-index/codebase-index-tool.d.ts +6 -0
- package/dist/codebase-index/codebase-outgoing-calls-tool.d.ts +1 -0
- package/dist/codebase-index/content-hash.d.ts +66 -0
- package/dist/codebase-index/index.js +1597 -153
- package/dist/codebase-index/indexer.d.ts +6 -0
- package/dist/codebase-index/parser-worker-pool.d.ts +63 -0
- package/dist/codebase-index/parser-worker-script.d.ts +42 -0
- package/dist/codebase-index/project-server-protocol.d.ts +2 -0
- package/dist/codebase-index/project-server.js +1483 -104
- package/dist/codebase-index/schema.d.ts +18 -0
- package/dist/codebase-index/tree-sitter/queries.d.ts +48 -0
- package/dist/codebase-index/tree-sitter/util.d.ts +31 -0
- package/dist/codebase-index/tree-sitter/visitor.d.ts +47 -0
- package/dist/codebase-index/tree-sitter-parser.d.ts +58 -0
- package/dist/codebase-index/vector-search.d.ts +62 -0
- package/dist/codebase-index/worker-protocol.d.ts +2 -0
- package/dist/codebase-index/worker.js +1452 -73
- package/dist/codebase-index/writer-bulk-insert.d.ts +5 -0
- package/dist/codebase-index/writer-graph-reader.d.ts +39 -0
- package/dist/codebase-index/writer-schema.d.ts +9 -2
- package/dist/codebase-index/writer.d.ts +36 -0
- package/dist/index.d.ts +4 -3
- package/dist/index.js +3258 -727
- package/dist/kanban-contract-actions.d.ts +7 -0
- package/dist/kanban-task-inputs.d.ts +16 -2
- package/dist/kanban-tool-schema.d.ts +2 -2
- package/dist/kanban-tool-types.d.ts +39 -2
- package/dist/kanban.js +580 -193
- package/dist/pack.js +3212 -686
- package/dist/plan.d.ts +4 -1
- package/dist/plan.js +2701 -18
- package/dist/read.js +1559 -103
- package/dist/session-kanban.d.ts +94 -1
- package/dist/session-kanban.js +308 -44
- package/dist/task.d.ts +5 -4
- package/dist/task.js +2825 -138
- package/dist/todo.d.ts +10 -1
- package/dist/todo.js +2474 -30
- package/dist/tool-tier.js +3212 -686
- package/package.json +8 -4
|
@@ -2127,21 +2127,672 @@ var init_yaml_parser = __esm({
|
|
|
2127
2127
|
}
|
|
2128
2128
|
});
|
|
2129
2129
|
|
|
2130
|
+
// src/codebase-index/tree-sitter/queries.ts
|
|
2131
|
+
function getQueries(lang) {
|
|
2132
|
+
return LANG_QUERIES[lang] ?? DEFAULT_QUERIES;
|
|
2133
|
+
}
|
|
2134
|
+
function readFirstString(node) {
|
|
2135
|
+
if (!node) return null;
|
|
2136
|
+
if (node.type === "string_literal" || node.type === "alias") {
|
|
2137
|
+
return node.text.replace(/^"|"$/g, "");
|
|
2138
|
+
}
|
|
2139
|
+
const child = node.namedChild(0);
|
|
2140
|
+
return child ? readFirstString(child) : null;
|
|
2141
|
+
}
|
|
2142
|
+
var DEFAULT_QUERIES, LANG_QUERIES;
|
|
2143
|
+
var init_queries = __esm({
|
|
2144
|
+
"src/codebase-index/tree-sitter/queries.ts"() {
|
|
2145
|
+
"use strict";
|
|
2146
|
+
DEFAULT_QUERIES = {
|
|
2147
|
+
declKinds: {}
|
|
2148
|
+
};
|
|
2149
|
+
LANG_QUERIES = {
|
|
2150
|
+
// ─── C family ──────────────────────────────────────────────────────────────
|
|
2151
|
+
c: {
|
|
2152
|
+
declKinds: {
|
|
2153
|
+
function_definition: "function",
|
|
2154
|
+
declaration: "function",
|
|
2155
|
+
// K&R-style `int foo(...)` ambiguous w/ local var; the visitor prefers the function branch when the declarator field is present
|
|
2156
|
+
struct_specifier: "struct",
|
|
2157
|
+
union_specifier: "struct",
|
|
2158
|
+
enum_specifier: "enum",
|
|
2159
|
+
type_definition: "type",
|
|
2160
|
+
// `typedef … X;`
|
|
2161
|
+
preproc_def: "const"
|
|
2162
|
+
// `#define NAME …`
|
|
2163
|
+
},
|
|
2164
|
+
nameField: {
|
|
2165
|
+
function_definition: "declarator",
|
|
2166
|
+
declaration: "declarator",
|
|
2167
|
+
struct_specifier: "name",
|
|
2168
|
+
enum_specifier: "name",
|
|
2169
|
+
type_definition: "declarator",
|
|
2170
|
+
preproc_def: "name"
|
|
2171
|
+
},
|
|
2172
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2173
|
+
"translation_unit",
|
|
2174
|
+
"function_definition",
|
|
2175
|
+
"struct_specifier",
|
|
2176
|
+
"union_specifier",
|
|
2177
|
+
"enum_specifier"
|
|
2178
|
+
])
|
|
2179
|
+
},
|
|
2180
|
+
cpp: {
|
|
2181
|
+
declKinds: {
|
|
2182
|
+
function_definition: "function",
|
|
2183
|
+
template_declaration: "function",
|
|
2184
|
+
// `template<typename T> …`
|
|
2185
|
+
class_specifier: "class",
|
|
2186
|
+
struct_specifier: "struct",
|
|
2187
|
+
union_specifier: "struct",
|
|
2188
|
+
enum_specifier: "enum",
|
|
2189
|
+
namespace_definition: "namespace",
|
|
2190
|
+
type_definition: "type"
|
|
2191
|
+
},
|
|
2192
|
+
nameField: {
|
|
2193
|
+
function_definition: "declarator",
|
|
2194
|
+
template_declaration: "name",
|
|
2195
|
+
class_specifier: "name",
|
|
2196
|
+
struct_specifier: "name",
|
|
2197
|
+
enum_specifier: "name",
|
|
2198
|
+
namespace_definition: "name",
|
|
2199
|
+
type_definition: "declarator"
|
|
2200
|
+
},
|
|
2201
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2202
|
+
"translation_unit",
|
|
2203
|
+
"function_definition",
|
|
2204
|
+
"class_specifier",
|
|
2205
|
+
"struct_specifier",
|
|
2206
|
+
"union_specifier",
|
|
2207
|
+
"enum_specifier",
|
|
2208
|
+
"namespace_definition"
|
|
2209
|
+
])
|
|
2210
|
+
},
|
|
2211
|
+
java: {
|
|
2212
|
+
declKinds: {
|
|
2213
|
+
class_declaration: "class",
|
|
2214
|
+
interface_declaration: "interface",
|
|
2215
|
+
enum_declaration: "enum",
|
|
2216
|
+
record_declaration: "class",
|
|
2217
|
+
annotation_type_declaration: "interface",
|
|
2218
|
+
method_declaration: "method",
|
|
2219
|
+
constructor_declaration: "method",
|
|
2220
|
+
field_declaration: "property"
|
|
2221
|
+
},
|
|
2222
|
+
nameField: {
|
|
2223
|
+
class_declaration: "name",
|
|
2224
|
+
interface_declaration: "name",
|
|
2225
|
+
enum_declaration: "name",
|
|
2226
|
+
record_declaration: "name",
|
|
2227
|
+
annotation_type_declaration: "name",
|
|
2228
|
+
method_declaration: "name",
|
|
2229
|
+
constructor_declaration: "name"
|
|
2230
|
+
},
|
|
2231
|
+
// `field_declaration` has no single `name` field — it carries a list of
|
|
2232
|
+
// variable declarators. We emit one Symbol per node using the first
|
|
2233
|
+
// identifier-shaped named child (see `extractName` fallback in
|
|
2234
|
+
// `visitor.ts`). `int a, b, c;` therefore indexes only `a` — splitting
|
|
2235
|
+
// multi-declarator fields into separate Symbols is a separate refactor
|
|
2236
|
+
// that needs the visitor to know it has multiple names per node, and no
|
|
2237
|
+
// current test relies on it.
|
|
2238
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2239
|
+
"program",
|
|
2240
|
+
"class_declaration",
|
|
2241
|
+
"interface_declaration",
|
|
2242
|
+
"enum_declaration",
|
|
2243
|
+
"record_declaration"
|
|
2244
|
+
])
|
|
2245
|
+
},
|
|
2246
|
+
csharp: {
|
|
2247
|
+
// C# 10+ `namespace Foo.Bar;` produces this node type. The legacy block
|
|
2248
|
+
// form `namespace Foo.Bar { ... }` produces `namespace_declaration`. Both
|
|
2249
|
+
// carry a `qualified_name` child whose text already includes the dots.
|
|
2250
|
+
// `using_directive` is intentionally not a declaration. Imports are
|
|
2251
|
+
// extracted separately; indexing a using directive as a namespace makes
|
|
2252
|
+
// the resolver bind it to its own source file before the real declaration.
|
|
2253
|
+
declKinds: {
|
|
2254
|
+
file_scoped_namespace_declaration: "namespace",
|
|
2255
|
+
class_declaration: "class",
|
|
2256
|
+
interface_declaration: "interface",
|
|
2257
|
+
struct_declaration: "struct",
|
|
2258
|
+
enum_declaration: "enum",
|
|
2259
|
+
record_declaration: "class",
|
|
2260
|
+
method_declaration: "method",
|
|
2261
|
+
constructor_declaration: "method",
|
|
2262
|
+
property_declaration: "property",
|
|
2263
|
+
field_declaration: "property",
|
|
2264
|
+
namespace_declaration: "namespace"
|
|
2265
|
+
},
|
|
2266
|
+
// Custom name extractor: take the full dotted name verbatim.
|
|
2267
|
+
nameExtractor: (node) => {
|
|
2268
|
+
const inner = node.namedChild(0);
|
|
2269
|
+
if (inner && (inner.type === "qualified_name" || inner.type === "name")) {
|
|
2270
|
+
return inner.text;
|
|
2271
|
+
}
|
|
2272
|
+
return null;
|
|
2273
|
+
},
|
|
2274
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2275
|
+
"compilation_unit",
|
|
2276
|
+
"namespace_declaration",
|
|
2277
|
+
"class_declaration",
|
|
2278
|
+
"interface_declaration",
|
|
2279
|
+
"struct_declaration",
|
|
2280
|
+
"enum_declaration",
|
|
2281
|
+
"record_declaration"
|
|
2282
|
+
])
|
|
2283
|
+
},
|
|
2284
|
+
php: {
|
|
2285
|
+
declKinds: {
|
|
2286
|
+
function_definition: "function",
|
|
2287
|
+
method_declaration: "method",
|
|
2288
|
+
class_declaration: "class",
|
|
2289
|
+
interface_declaration: "interface",
|
|
2290
|
+
trait_declaration: "class",
|
|
2291
|
+
enum_declaration: "enum",
|
|
2292
|
+
namespace_definition: "namespace"
|
|
2293
|
+
},
|
|
2294
|
+
nameField: {
|
|
2295
|
+
function_definition: "name",
|
|
2296
|
+
method_declaration: "name",
|
|
2297
|
+
class_declaration: "name",
|
|
2298
|
+
interface_declaration: "name",
|
|
2299
|
+
trait_declaration: "name",
|
|
2300
|
+
enum_declaration: "name",
|
|
2301
|
+
namespace_declaration: "name"
|
|
2302
|
+
},
|
|
2303
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2304
|
+
"program",
|
|
2305
|
+
"namespace_definition",
|
|
2306
|
+
"class_declaration",
|
|
2307
|
+
"interface_declaration",
|
|
2308
|
+
"trait_declaration",
|
|
2309
|
+
"enum_declaration"
|
|
2310
|
+
])
|
|
2311
|
+
},
|
|
2312
|
+
// ─── Scripting / mobile ────────────────────────────────────────────────────
|
|
2313
|
+
ruby: {
|
|
2314
|
+
declKinds: {
|
|
2315
|
+
method: "function",
|
|
2316
|
+
singleton_method: "method",
|
|
2317
|
+
class: "class",
|
|
2318
|
+
module: "namespace",
|
|
2319
|
+
constant: "const"
|
|
2320
|
+
},
|
|
2321
|
+
nameField: {
|
|
2322
|
+
method: "name",
|
|
2323
|
+
singleton_method: "name",
|
|
2324
|
+
class: "name",
|
|
2325
|
+
module: "name",
|
|
2326
|
+
constant: "name"
|
|
2327
|
+
},
|
|
2328
|
+
scopeNodes: /* @__PURE__ */ new Set(["program", "class", "module", "singleton_method", "method"])
|
|
2329
|
+
},
|
|
2330
|
+
swift: {
|
|
2331
|
+
declKinds: {
|
|
2332
|
+
function_declaration: "function",
|
|
2333
|
+
class_declaration: "class",
|
|
2334
|
+
struct_declaration: "struct",
|
|
2335
|
+
enum_declaration: "enum",
|
|
2336
|
+
protocol_declaration: "interface",
|
|
2337
|
+
actor_declaration: "class",
|
|
2338
|
+
extension_declaration: "class",
|
|
2339
|
+
initializer: "method",
|
|
2340
|
+
property_declaration: "property"
|
|
2341
|
+
},
|
|
2342
|
+
nameField: {
|
|
2343
|
+
function_declaration: "name",
|
|
2344
|
+
class_declaration: "name",
|
|
2345
|
+
struct_declaration: "name",
|
|
2346
|
+
enum_declaration: "name",
|
|
2347
|
+
protocol_declaration: "name",
|
|
2348
|
+
actor_declaration: "name",
|
|
2349
|
+
extension_declaration: "name",
|
|
2350
|
+
initializer: "name",
|
|
2351
|
+
property_declaration: "name"
|
|
2352
|
+
},
|
|
2353
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2354
|
+
"source_file",
|
|
2355
|
+
"class_declaration",
|
|
2356
|
+
"struct_declaration",
|
|
2357
|
+
"enum_declaration",
|
|
2358
|
+
"protocol_declaration",
|
|
2359
|
+
"actor_declaration",
|
|
2360
|
+
"extension_declaration"
|
|
2361
|
+
])
|
|
2362
|
+
},
|
|
2363
|
+
kotlin: {
|
|
2364
|
+
declKinds: {
|
|
2365
|
+
class_declaration: "class",
|
|
2366
|
+
object_declaration: "class",
|
|
2367
|
+
interface_declaration: "interface",
|
|
2368
|
+
function_declaration: "function",
|
|
2369
|
+
property_declaration: "property",
|
|
2370
|
+
type_alias: "type"
|
|
2371
|
+
},
|
|
2372
|
+
nameField: {
|
|
2373
|
+
class_declaration: "name",
|
|
2374
|
+
object_declaration: "name",
|
|
2375
|
+
interface_declaration: "name",
|
|
2376
|
+
function_declaration: "name",
|
|
2377
|
+
property_declaration: "name",
|
|
2378
|
+
type_alias: "name"
|
|
2379
|
+
},
|
|
2380
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2381
|
+
"source_file",
|
|
2382
|
+
"class_declaration",
|
|
2383
|
+
"object_declaration",
|
|
2384
|
+
"interface_declaration",
|
|
2385
|
+
"function_declaration"
|
|
2386
|
+
])
|
|
2387
|
+
},
|
|
2388
|
+
elixir: {
|
|
2389
|
+
declKinds: {
|
|
2390
|
+
// `def foo`, `defp foo`, `defmacro foo`, `macrop foo` all surface as
|
|
2391
|
+
// `call` nodes in the tree-sitter grammar — there is no
|
|
2392
|
+
// `function_definition`. The `nameExtractor` walks the call's
|
|
2393
|
+
// children to pick the right sibling identifier.
|
|
2394
|
+
call: "function",
|
|
2395
|
+
module: "namespace"
|
|
2396
|
+
},
|
|
2397
|
+
nameExtractor: (node) => {
|
|
2398
|
+
if (node.type === "module") {
|
|
2399
|
+
const aliasNode = node.childForFieldName("alias");
|
|
2400
|
+
return readFirstString(aliasNode) ?? null;
|
|
2401
|
+
}
|
|
2402
|
+
if (node.type !== "call") return null;
|
|
2403
|
+
const first = node.namedChild(0);
|
|
2404
|
+
if (!first) return null;
|
|
2405
|
+
const target = first.text;
|
|
2406
|
+
if (target !== "def" && target !== "defp" && target !== "defmacro" && target !== "defp_macro" && target !== "macrop" && target !== "defprotocol" && target !== "defguard" && target !== "defguardp") {
|
|
2407
|
+
return null;
|
|
2408
|
+
}
|
|
2409
|
+
const nameNode = node.namedChild(1);
|
|
2410
|
+
return nameNode?.text ?? null;
|
|
2411
|
+
},
|
|
2412
|
+
scopeNodes: /* @__PURE__ */ new Set(["source", "module"])
|
|
2413
|
+
},
|
|
2414
|
+
shell: {
|
|
2415
|
+
declKinds: {
|
|
2416
|
+
function_definition: "function"
|
|
2417
|
+
},
|
|
2418
|
+
nameField: { function_definition: "name" },
|
|
2419
|
+
scopeNodes: /* @__PURE__ */ new Set(["program", "function_definition"])
|
|
2420
|
+
}
|
|
2421
|
+
};
|
|
2422
|
+
}
|
|
2423
|
+
});
|
|
2424
|
+
|
|
2425
|
+
// src/codebase-index/tree-sitter/util.ts
|
|
2426
|
+
function lineColAt2(offsets, index) {
|
|
2427
|
+
let low = 0;
|
|
2428
|
+
let high = offsets.length;
|
|
2429
|
+
while (low < high) {
|
|
2430
|
+
const mid = low + high >>> 1;
|
|
2431
|
+
if ((offsets[mid] ?? 0) < index) low = mid + 1;
|
|
2432
|
+
else high = mid;
|
|
2433
|
+
}
|
|
2434
|
+
const lastNl = low > 0 ? offsets[low - 1] ?? -1 : -1;
|
|
2435
|
+
return { line: low + 1, col: index - lastNl };
|
|
2436
|
+
}
|
|
2437
|
+
function newlineOffsets3(content) {
|
|
2438
|
+
const offsets = [];
|
|
2439
|
+
for (let i = 0; i < content.length; i++) {
|
|
2440
|
+
if (content.charCodeAt(i) === 10) offsets.push(i);
|
|
2441
|
+
}
|
|
2442
|
+
return offsets;
|
|
2443
|
+
}
|
|
2444
|
+
var TREE_SITTER_MAX_FILE_CHARS, TREE_SITTER_MAX_SYMBOLS;
|
|
2445
|
+
var init_util = __esm({
|
|
2446
|
+
"src/codebase-index/tree-sitter/util.ts"() {
|
|
2447
|
+
"use strict";
|
|
2448
|
+
TREE_SITTER_MAX_FILE_CHARS = 512 * 1024;
|
|
2449
|
+
TREE_SITTER_MAX_SYMBOLS = 500;
|
|
2450
|
+
}
|
|
2451
|
+
});
|
|
2452
|
+
|
|
2453
|
+
// src/codebase-index/tree-sitter/visitor.ts
|
|
2454
|
+
function visitTree(tree, content, file, lang, queries) {
|
|
2455
|
+
const boundedContent = content.length > TREE_SITTER_MAX_FILE_CHARS ? content.slice(0, TREE_SITTER_MAX_FILE_CHARS) : content;
|
|
2456
|
+
const nlOffsets = newlineOffsets3(boundedContent);
|
|
2457
|
+
const symbols = [];
|
|
2458
|
+
const scopeStack = [];
|
|
2459
|
+
function visit(node, depth) {
|
|
2460
|
+
if (symbols.length >= TREE_SITTER_MAX_SYMBOLS) return;
|
|
2461
|
+
if (node.isMissing || node.isError) {
|
|
2462
|
+
} else {
|
|
2463
|
+
const kind = queries.declKinds[node.type];
|
|
2464
|
+
if (kind) {
|
|
2465
|
+
const emitted = emitSymbol(
|
|
2466
|
+
node,
|
|
2467
|
+
kind,
|
|
2468
|
+
file,
|
|
2469
|
+
lang,
|
|
2470
|
+
scopeStack,
|
|
2471
|
+
boundedContent,
|
|
2472
|
+
nlOffsets,
|
|
2473
|
+
queries
|
|
2474
|
+
);
|
|
2475
|
+
if (emitted) symbols.push(emitted);
|
|
2476
|
+
}
|
|
2477
|
+
}
|
|
2478
|
+
const pushesScope = queries.scopeNodes?.has(node.type) ?? false;
|
|
2479
|
+
const pushIdx = pushesScope ? pushScope(scopeStack, node, queries) : -1;
|
|
2480
|
+
if (queries.skipNamedChildren) {
|
|
2481
|
+
if (pushIdx !== -1) scopeStack.pop();
|
|
2482
|
+
return;
|
|
2483
|
+
}
|
|
2484
|
+
for (const child of node.namedChildren) {
|
|
2485
|
+
visit(child, depth + 1);
|
|
2486
|
+
if (symbols.length >= TREE_SITTER_MAX_SYMBOLS) {
|
|
2487
|
+
if (pushIdx !== -1) scopeStack.pop();
|
|
2488
|
+
return;
|
|
2489
|
+
}
|
|
2490
|
+
}
|
|
2491
|
+
if (pushIdx !== -1) scopeStack.pop();
|
|
2492
|
+
}
|
|
2493
|
+
visit(tree.rootNode, 0);
|
|
2494
|
+
return { symbols };
|
|
2495
|
+
}
|
|
2496
|
+
function pushScope(scopeStack, node, queries) {
|
|
2497
|
+
const name = extractName(node, queries);
|
|
2498
|
+
if (!name) return -1;
|
|
2499
|
+
scopeStack.push(name);
|
|
2500
|
+
return scopeStack.length - 1;
|
|
2501
|
+
}
|
|
2502
|
+
function extractName(node, queries) {
|
|
2503
|
+
if (queries.nameExtractor) {
|
|
2504
|
+
const extracted = queries.nameExtractor(node);
|
|
2505
|
+
if (extracted) return extracted;
|
|
2506
|
+
}
|
|
2507
|
+
const fieldName = queries.nameField?.[node.type] ?? "name";
|
|
2508
|
+
const field = node.childForFieldName(fieldName);
|
|
2509
|
+
if (field) {
|
|
2510
|
+
if (IDENTIFIER_NODE_TYPES.has(field.type)) {
|
|
2511
|
+
return field.text;
|
|
2512
|
+
}
|
|
2513
|
+
const inner = field.childForFieldName("name") ?? field.namedChild(0);
|
|
2514
|
+
if (inner && IDENTIFIER_NODE_TYPES.has(inner.type)) {
|
|
2515
|
+
return inner.text;
|
|
2516
|
+
}
|
|
2517
|
+
}
|
|
2518
|
+
for (let i = 0; i < node.namedChildCount; i++) {
|
|
2519
|
+
const child = node.namedChild(i);
|
|
2520
|
+
if (child && IDENTIFIER_NODE_TYPES.has(child.type)) {
|
|
2521
|
+
return child.text;
|
|
2522
|
+
}
|
|
2523
|
+
}
|
|
2524
|
+
return null;
|
|
2525
|
+
}
|
|
2526
|
+
function emitSymbol(node, kind, file, lang, scopeStack, content, nlOffsets, queries) {
|
|
2527
|
+
const name = extractName(node, queries);
|
|
2528
|
+
if (!name) return null;
|
|
2529
|
+
const pos = node.startIndex;
|
|
2530
|
+
const { line, col } = lineColAt2(nlOffsets, pos);
|
|
2531
|
+
const end = Math.min(node.endIndex, content.length);
|
|
2532
|
+
const signature = content.slice(pos, end).replace(/\s+/g, " ").trim().slice(0, 500);
|
|
2533
|
+
const scope = scopeStack.join(".");
|
|
2534
|
+
const text = [name, signature].filter(Boolean).join(" | ").trim().slice(0, 1e3);
|
|
2535
|
+
return {
|
|
2536
|
+
id: 0,
|
|
2537
|
+
// caller assigns during bulk insertion
|
|
2538
|
+
lang,
|
|
2539
|
+
kind,
|
|
2540
|
+
name: name.slice(0, 200),
|
|
2541
|
+
file,
|
|
2542
|
+
line,
|
|
2543
|
+
col,
|
|
2544
|
+
signature,
|
|
2545
|
+
docComment: "",
|
|
2546
|
+
// doc-comment extraction lands with ref emission on Day 4
|
|
2547
|
+
scope,
|
|
2548
|
+
text
|
|
2549
|
+
};
|
|
2550
|
+
}
|
|
2551
|
+
var IDENTIFIER_NODE_TYPES;
|
|
2552
|
+
var init_visitor = __esm({
|
|
2553
|
+
"src/codebase-index/tree-sitter/visitor.ts"() {
|
|
2554
|
+
"use strict";
|
|
2555
|
+
init_util();
|
|
2556
|
+
IDENTIFIER_NODE_TYPES = /* @__PURE__ */ new Set([
|
|
2557
|
+
"identifier",
|
|
2558
|
+
"simple_identifier",
|
|
2559
|
+
"type_identifier",
|
|
2560
|
+
"field_identifier",
|
|
2561
|
+
"property_identifier",
|
|
2562
|
+
"name",
|
|
2563
|
+
"word",
|
|
2564
|
+
"variable_name",
|
|
2565
|
+
"constant",
|
|
2566
|
+
"sym"
|
|
2567
|
+
]);
|
|
2568
|
+
}
|
|
2569
|
+
});
|
|
2570
|
+
|
|
2571
|
+
// src/codebase-index/tree-sitter-parser.ts
|
|
2572
|
+
var tree_sitter_parser_exports = {};
|
|
2573
|
+
__export(tree_sitter_parser_exports, {
|
|
2574
|
+
__smokeRootType: () => __smokeRootType,
|
|
2575
|
+
getGrammarWasmPath: () => getGrammarWasmPath,
|
|
2576
|
+
isTreeSitterSupported: () => isTreeSitterSupported,
|
|
2577
|
+
loadTreeSitterLanguage: () => loadTreeSitterLanguage,
|
|
2578
|
+
parseSymbols: () => parseSymbols8
|
|
2579
|
+
});
|
|
2580
|
+
import * as path9 from "node:path";
|
|
2581
|
+
import { fileURLToPath } from "node:url";
|
|
2582
|
+
function optInEnabled(env) {
|
|
2583
|
+
return process.env[env] === "1" || process.env[env] === "true";
|
|
2584
|
+
}
|
|
2585
|
+
function getRuntime() {
|
|
2586
|
+
if (!runtimePromise) {
|
|
2587
|
+
runtimePromise = (async () => {
|
|
2588
|
+
const mod = await import("web-tree-sitter");
|
|
2589
|
+
const init = () => mod.Parser.init({ locateFile: () => RUNTIME_WASM });
|
|
2590
|
+
return { Parser: mod.Parser, Language: mod.Language, init };
|
|
2591
|
+
})();
|
|
2592
|
+
}
|
|
2593
|
+
return runtimePromise;
|
|
2594
|
+
}
|
|
2595
|
+
async function loadLanguage(lang) {
|
|
2596
|
+
const existing = languageCache.get(lang);
|
|
2597
|
+
if (existing) return existing;
|
|
2598
|
+
const promise = (async () => {
|
|
2599
|
+
const grammarName = resolveGrammarName(lang);
|
|
2600
|
+
if (!grammarName) {
|
|
2601
|
+
throw new Error(`tree-sitter: no grammar registered for lang "${lang}"`);
|
|
2602
|
+
}
|
|
2603
|
+
const wasmPath = path9.join(WASM_DIR, grammarName, `tree-sitter-${grammarName}.wasm`);
|
|
2604
|
+
const { Language, init } = await getRuntime();
|
|
2605
|
+
await init();
|
|
2606
|
+
const languageObj = await Language.load(wasmPath);
|
|
2607
|
+
return { lang, Language: languageObj };
|
|
2608
|
+
})();
|
|
2609
|
+
languageCache.set(lang, promise);
|
|
2610
|
+
return promise;
|
|
2611
|
+
}
|
|
2612
|
+
function resolveGrammarName(lang) {
|
|
2613
|
+
if (lang === "go" && optInEnabled(GO_OPT_IN)) return "go";
|
|
2614
|
+
if (lang === "py" && optInEnabled(PY_OPT_IN)) return "python";
|
|
2615
|
+
if (lang === "rs" && optInEnabled(RS_OPT_IN)) return "rust";
|
|
2616
|
+
return LANG_TO_GRAMMAR[lang];
|
|
2617
|
+
}
|
|
2618
|
+
function isTreeSitterSupported(lang) {
|
|
2619
|
+
return resolveGrammarName(lang) !== void 0;
|
|
2620
|
+
}
|
|
2621
|
+
function getGrammarWasmPath(lang) {
|
|
2622
|
+
const name = resolveGrammarName(lang);
|
|
2623
|
+
if (!name) return void 0;
|
|
2624
|
+
return path9.join(WASM_DIR, name, `tree-sitter-${name}.wasm`);
|
|
2625
|
+
}
|
|
2626
|
+
async function parseSymbols8(opts) {
|
|
2627
|
+
const { file, content, lang } = opts;
|
|
2628
|
+
if (!isTreeSitterSupported(lang)) {
|
|
2629
|
+
return { file, lang, symbols: [], mtimeMs: Date.now() };
|
|
2630
|
+
}
|
|
2631
|
+
try {
|
|
2632
|
+
const { Parser } = await getRuntime();
|
|
2633
|
+
const cached = await loadLanguage(lang);
|
|
2634
|
+
const parser = new Parser();
|
|
2635
|
+
parser.setLanguage(cached.Language);
|
|
2636
|
+
const tree = parser.parse(content);
|
|
2637
|
+
if (!tree) {
|
|
2638
|
+
return { file, lang, symbols: [], mtimeMs: Date.now() };
|
|
2639
|
+
}
|
|
2640
|
+
const { symbols } = visitTree(tree, content, file, lang, getQueries(lang));
|
|
2641
|
+
parser.delete();
|
|
2642
|
+
tree.delete();
|
|
2643
|
+
return { file, lang, symbols, refs: [], mtimeMs: Date.now() };
|
|
2644
|
+
} catch {
|
|
2645
|
+
return { file, lang, symbols: [], mtimeMs: Date.now() };
|
|
2646
|
+
}
|
|
2647
|
+
}
|
|
2648
|
+
async function loadTreeSitterLanguage(lang) {
|
|
2649
|
+
const cached = await loadLanguage(lang);
|
|
2650
|
+
return cached.Language;
|
|
2651
|
+
}
|
|
2652
|
+
async function __smokeRootType(opts) {
|
|
2653
|
+
if (!isTreeSitterSupported(opts.lang)) {
|
|
2654
|
+
throw new Error(`tree-sitter: no grammar registered for lang "${opts.lang}"`);
|
|
2655
|
+
}
|
|
2656
|
+
const { Parser } = await getRuntime();
|
|
2657
|
+
const cached = await loadLanguage(opts.lang);
|
|
2658
|
+
const parser = new Parser();
|
|
2659
|
+
parser.setLanguage(cached.Language);
|
|
2660
|
+
let tree = null;
|
|
2661
|
+
try {
|
|
2662
|
+
tree = parser.parse(opts.content);
|
|
2663
|
+
if (!tree) throw new Error("tree-sitter: parser.parse returned null");
|
|
2664
|
+
return tree.rootNode.type;
|
|
2665
|
+
} finally {
|
|
2666
|
+
tree?.delete();
|
|
2667
|
+
parser.delete();
|
|
2668
|
+
}
|
|
2669
|
+
}
|
|
2670
|
+
var WASM_DIR, RUNTIME_WASM, LANG_TO_GRAMMAR, GO_OPT_IN, PY_OPT_IN, RS_OPT_IN, runtimePromise, languageCache;
|
|
2671
|
+
var init_tree_sitter_parser = __esm({
|
|
2672
|
+
"src/codebase-index/tree-sitter-parser.ts"() {
|
|
2673
|
+
"use strict";
|
|
2674
|
+
init_queries();
|
|
2675
|
+
init_visitor();
|
|
2676
|
+
WASM_DIR = fileURLToPath(new URL("./wasm/", import.meta.url));
|
|
2677
|
+
RUNTIME_WASM = path9.join(WASM_DIR, "tree-sitter-runtime.wasm");
|
|
2678
|
+
LANG_TO_GRAMMAR = {
|
|
2679
|
+
c: "c",
|
|
2680
|
+
cpp: "cpp",
|
|
2681
|
+
java: "java",
|
|
2682
|
+
csharp: "c_sharp",
|
|
2683
|
+
// tree-sitter directory uses underscore
|
|
2684
|
+
php: "php",
|
|
2685
|
+
ruby: "ruby",
|
|
2686
|
+
swift: "swift",
|
|
2687
|
+
kotlin: "kotlin",
|
|
2688
|
+
shell: "bash",
|
|
2689
|
+
// we treat `.sh` / `.bash` / `.zsh` via the bash grammar
|
|
2690
|
+
// Go/Python/Rust are opt-in only (see goOptIn / pyOptIn / rsOptIn below)
|
|
2691
|
+
elixir: "elixir"
|
|
2692
|
+
};
|
|
2693
|
+
GO_OPT_IN = "WRONGSTACK_USE_TS_GO";
|
|
2694
|
+
PY_OPT_IN = "WRONGSTACK_USE_TS_PY";
|
|
2695
|
+
RS_OPT_IN = "WRONGSTACK_USE_TS_RS";
|
|
2696
|
+
runtimePromise = null;
|
|
2697
|
+
languageCache = /* @__PURE__ */ new Map();
|
|
2698
|
+
}
|
|
2699
|
+
});
|
|
2700
|
+
|
|
2130
2701
|
// src/codebase-index/worker.ts
|
|
2131
2702
|
import { parentPort } from "node:worker_threads";
|
|
2132
2703
|
|
|
2133
2704
|
// src/codebase-index/indexer.ts
|
|
2134
2705
|
import { expectDefined as expectDefined5 } from "@wrongstack/core/utils";
|
|
2135
2706
|
import { execFile } from "node:child_process";
|
|
2136
|
-
import * as
|
|
2707
|
+
import * as fs9 from "node:fs/promises";
|
|
2137
2708
|
import { availableParallelism } from "node:os";
|
|
2138
|
-
import * as
|
|
2709
|
+
import * as path12 from "node:path";
|
|
2139
2710
|
import {
|
|
2140
2711
|
DEFAULT_WALK_IGNORE_DIRS,
|
|
2141
2712
|
indexParallelBatchSize,
|
|
2142
2713
|
isFrugalPerf
|
|
2143
2714
|
} from "@wrongstack/core/utils";
|
|
2144
2715
|
|
|
2716
|
+
// src/codebase-index/content-hash.ts
|
|
2717
|
+
var PRIME64_1 = 0x9e3779b185ebca87n;
|
|
2718
|
+
var PRIME64_2 = 0xc2b2ae3d27d4eb4fn;
|
|
2719
|
+
var PRIME64_3 = 0x165667b19e3779f9n;
|
|
2720
|
+
var PRIME64_4 = 0x85ebca77c2b2ae63n;
|
|
2721
|
+
var PRIME64_5 = 0x27d4eb2f165667c5n;
|
|
2722
|
+
var MASK64 = 0xffffffffffffffffn;
|
|
2723
|
+
function mul64(a, b) {
|
|
2724
|
+
return (a & MASK64) * (b & MASK64) & MASK64;
|
|
2725
|
+
}
|
|
2726
|
+
function rotl64(x, n) {
|
|
2727
|
+
const v = x & MASK64;
|
|
2728
|
+
return (v << BigInt(n) | v >> BigInt(64 - n)) & MASK64;
|
|
2729
|
+
}
|
|
2730
|
+
function readU64LE(buf, off) {
|
|
2731
|
+
let v = 0n;
|
|
2732
|
+
for (let i = 7; i >= 0; i--) {
|
|
2733
|
+
v = v << 8n | BigInt(buf[off + i] ?? 0);
|
|
2734
|
+
}
|
|
2735
|
+
return v & MASK64;
|
|
2736
|
+
}
|
|
2737
|
+
function readU32LE(buf, off) {
|
|
2738
|
+
return (BigInt(buf[off] ?? 0) | BigInt(buf[off + 1] ?? 0) << 8n | BigInt(buf[off + 2] ?? 0) << 16n | BigInt(buf[off + 3] ?? 0) << 24n) & MASK64;
|
|
2739
|
+
}
|
|
2740
|
+
function xxh64Round(acc, lane) {
|
|
2741
|
+
return mul64(rotl64(acc + mul64(lane, PRIME64_2) & MASK64, 31), PRIME64_1);
|
|
2742
|
+
}
|
|
2743
|
+
function xxh64MergeRound(acc, val) {
|
|
2744
|
+
return mul64(acc ^ xxh64Round(0n, val), PRIME64_1) + PRIME64_4 & MASK64;
|
|
2745
|
+
}
|
|
2746
|
+
function xxhash64Hex(buf, explicitLen) {
|
|
2747
|
+
const length = explicitLen ?? buf.length;
|
|
2748
|
+
let h;
|
|
2749
|
+
let off = 0;
|
|
2750
|
+
if (length >= 32) {
|
|
2751
|
+
let v1 = PRIME64_1 + PRIME64_2 & MASK64;
|
|
2752
|
+
let v2 = PRIME64_2;
|
|
2753
|
+
let v3 = 0n;
|
|
2754
|
+
let v4 = 0n - PRIME64_1 & MASK64;
|
|
2755
|
+
const end32 = length - 32;
|
|
2756
|
+
while (off <= end32) {
|
|
2757
|
+
v1 = xxh64Round(v1, readU64LE(buf, off));
|
|
2758
|
+
v2 = xxh64Round(v2, readU64LE(buf, off + 8));
|
|
2759
|
+
v3 = xxh64Round(v3, readU64LE(buf, off + 16));
|
|
2760
|
+
v4 = xxh64Round(v4, readU64LE(buf, off + 24));
|
|
2761
|
+
off += 32;
|
|
2762
|
+
}
|
|
2763
|
+
h = rotl64(v1, 1) + rotl64(v2, 7) + rotl64(v3, 12) + rotl64(v4, 18) & MASK64;
|
|
2764
|
+
h = xxh64MergeRound(h, v1);
|
|
2765
|
+
h = xxh64MergeRound(h, v2);
|
|
2766
|
+
h = xxh64MergeRound(h, v3);
|
|
2767
|
+
h = xxh64MergeRound(h, v4);
|
|
2768
|
+
} else {
|
|
2769
|
+
h = PRIME64_5;
|
|
2770
|
+
}
|
|
2771
|
+
h = h + BigInt(length) & MASK64;
|
|
2772
|
+
while (off + 8 <= length) {
|
|
2773
|
+
const k1 = xxh64Round(0n, readU64LE(buf, off));
|
|
2774
|
+
h = mul64(rotl64(h ^ k1, 27), PRIME64_1) + PRIME64_4 & MASK64;
|
|
2775
|
+
off += 8;
|
|
2776
|
+
}
|
|
2777
|
+
if (off + 4 <= length) {
|
|
2778
|
+
h = mul64(rotl64(h ^ mul64(readU32LE(buf, off), PRIME64_1), 23), PRIME64_2) + PRIME64_3 & MASK64;
|
|
2779
|
+
off += 4;
|
|
2780
|
+
}
|
|
2781
|
+
while (off < length) {
|
|
2782
|
+
h = mul64(rotl64(h ^ mul64(BigInt(buf[off] ?? 0), PRIME64_5), 11), PRIME64_1) & MASK64;
|
|
2783
|
+
off += 1;
|
|
2784
|
+
}
|
|
2785
|
+
h = (h ^ h >> 33n) & MASK64;
|
|
2786
|
+
h = mul64(h, PRIME64_2);
|
|
2787
|
+
h = (h ^ h >> 29n) & MASK64;
|
|
2788
|
+
h = mul64(h, PRIME64_3);
|
|
2789
|
+
h = (h ^ h >> 32n) & MASK64;
|
|
2790
|
+
return h.toString(16).padStart(16, "0");
|
|
2791
|
+
}
|
|
2792
|
+
function xxhash64String(content) {
|
|
2793
|
+
return xxhash64Hex(new TextEncoder().encode(content));
|
|
2794
|
+
}
|
|
2795
|
+
|
|
2145
2796
|
// src/codebase-index/gitignore.ts
|
|
2146
2797
|
import * as fs from "node:fs/promises";
|
|
2147
2798
|
import * as path from "node:path";
|
|
@@ -2937,32 +3588,56 @@ async function dispatch(file, content, lang) {
|
|
|
2937
3588
|
case "tsx":
|
|
2938
3589
|
case "js":
|
|
2939
3590
|
case "jsx": {
|
|
2940
|
-
const { parseSymbols:
|
|
2941
|
-
return
|
|
3591
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_ts_parser(), ts_parser_exports));
|
|
3592
|
+
return parseSymbols9({ file, content, lang });
|
|
2942
3593
|
}
|
|
2943
3594
|
case "go": {
|
|
2944
|
-
const { parseSymbols:
|
|
2945
|
-
return
|
|
3595
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_go_parser(), go_parser_exports));
|
|
3596
|
+
return parseSymbols9({ file, content, lang: "go" });
|
|
2946
3597
|
}
|
|
2947
3598
|
case "py": {
|
|
2948
|
-
const { parseSymbols:
|
|
2949
|
-
return
|
|
3599
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_py_parser(), py_parser_exports));
|
|
3600
|
+
return parseSymbols9({ file, content, lang: "py" });
|
|
2950
3601
|
}
|
|
2951
3602
|
case "rs": {
|
|
2952
|
-
const { parseSymbols:
|
|
2953
|
-
return
|
|
3603
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_rs_parser(), rs_parser_exports));
|
|
3604
|
+
return parseSymbols9({ file, content, lang: "rs" });
|
|
2954
3605
|
}
|
|
2955
3606
|
case "json": {
|
|
2956
|
-
const { parseSymbols:
|
|
2957
|
-
return
|
|
3607
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_json_parser(), json_parser_exports));
|
|
3608
|
+
return parseSymbols9({ file, content, lang: "json" });
|
|
2958
3609
|
}
|
|
2959
3610
|
case "yaml": {
|
|
2960
|
-
const { parseSymbols:
|
|
2961
|
-
return
|
|
3611
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_yaml_parser(), yaml_parser_exports));
|
|
3612
|
+
return parseSymbols9({ file, content, lang: "yaml" });
|
|
3613
|
+
}
|
|
3614
|
+
// Phase 1: ten languages now route through the Tree-Sitter WASM
|
|
3615
|
+
// universal parser (`tree-sitter-parser.ts`). The dispatch falls back to
|
|
3616
|
+
// the regex extractor in `generic-parser.ts` whenever WASM loading fails
|
|
3617
|
+
// or the parser returns zero symbols — preserving the indexable-file
|
|
3618
|
+
// contract that "missing a parser must never mean skipping the file".
|
|
3619
|
+
case "c":
|
|
3620
|
+
case "cpp":
|
|
3621
|
+
case "java":
|
|
3622
|
+
case "csharp":
|
|
3623
|
+
case "php":
|
|
3624
|
+
case "ruby":
|
|
3625
|
+
case "swift":
|
|
3626
|
+
case "kotlin":
|
|
3627
|
+
case "shell":
|
|
3628
|
+
case "elixir": {
|
|
3629
|
+
try {
|
|
3630
|
+
const { parseSymbols: parseSymbols10 } = await Promise.resolve().then(() => (init_tree_sitter_parser(), tree_sitter_parser_exports));
|
|
3631
|
+
const parsed = await parseSymbols10({ file, content, lang });
|
|
3632
|
+
if (parsed.symbols.length > 0) return parsed;
|
|
3633
|
+
} catch {
|
|
3634
|
+
}
|
|
3635
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_generic_parser(), generic_parser_exports));
|
|
3636
|
+
return parseSymbols9({ file, content, lang });
|
|
2962
3637
|
}
|
|
2963
3638
|
default: {
|
|
2964
|
-
const { parseSymbols:
|
|
2965
|
-
return
|
|
3639
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_generic_parser(), generic_parser_exports));
|
|
3640
|
+
return parseSymbols9({ file, content, lang });
|
|
2966
3641
|
}
|
|
2967
3642
|
}
|
|
2968
3643
|
}
|
|
@@ -2974,10 +3649,185 @@ function withRelations(parsed, content, lang) {
|
|
|
2974
3649
|
return { ...parsed, refs: refs.map((ref) => ref.lang ? ref : { ...ref, lang }) };
|
|
2975
3650
|
}
|
|
2976
3651
|
|
|
3652
|
+
// src/codebase-index/parser-worker-pool.ts
|
|
3653
|
+
import { Worker } from "node:worker_threads";
|
|
3654
|
+
import { fileURLToPath as fileURLToPath2 } from "node:url";
|
|
3655
|
+
import * as fs6 from "node:fs";
|
|
3656
|
+
var WORKER_POOL_THRESHOLD = 500;
|
|
3657
|
+
var ParserWorkerPool = class {
|
|
3658
|
+
constructor(maxWorkers = defaultWorkerCount()) {
|
|
3659
|
+
this.maxWorkers = maxWorkers;
|
|
3660
|
+
}
|
|
3661
|
+
maxWorkers;
|
|
3662
|
+
workers = [];
|
|
3663
|
+
nextBatchId = 1;
|
|
3664
|
+
pending = /* @__PURE__ */ new Map();
|
|
3665
|
+
creating = false;
|
|
3666
|
+
unavailable = false;
|
|
3667
|
+
/**
|
|
3668
|
+
* True if the pool is available for use. Returns false when:
|
|
3669
|
+
* - Worker threads aren't supported (sandbox, exotic runtime)
|
|
3670
|
+
* - The built worker script can't be found
|
|
3671
|
+
* - Pool creation was attempted and failed
|
|
3672
|
+
*/
|
|
3673
|
+
isAvailable() {
|
|
3674
|
+
return !this.unavailable && this.workers.length > 0;
|
|
3675
|
+
}
|
|
3676
|
+
/**
|
|
3677
|
+
* Lazily create the worker pool. Returns true if the pool is ready, false
|
|
3678
|
+
* if it's unavailable (caller should fall back to inline parsing).
|
|
3679
|
+
*/
|
|
3680
|
+
async ensureReady() {
|
|
3681
|
+
if (this.isAvailable()) return true;
|
|
3682
|
+
if (this.unavailable) return false;
|
|
3683
|
+
if (this.creating) {
|
|
3684
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
3685
|
+
return this.isAvailable();
|
|
3686
|
+
}
|
|
3687
|
+
this.creating = true;
|
|
3688
|
+
try {
|
|
3689
|
+
const url = resolveWorkerScriptUrl();
|
|
3690
|
+
if (!url) {
|
|
3691
|
+
this.unavailable = true;
|
|
3692
|
+
return false;
|
|
3693
|
+
}
|
|
3694
|
+
for (let i = 0; i < this.maxWorkers; i++) {
|
|
3695
|
+
try {
|
|
3696
|
+
const w = new Worker(url, { name: `wstack-parser-${i}` });
|
|
3697
|
+
w.unref();
|
|
3698
|
+
w.on("message", (msg) => this.handleMessage(msg));
|
|
3699
|
+
w.on("error", (err) => this.handleError(err, w));
|
|
3700
|
+
this.workers.push({ worker: w, busy: false });
|
|
3701
|
+
} catch {
|
|
3702
|
+
if (this.workers.length === 0) {
|
|
3703
|
+
this.unavailable = true;
|
|
3704
|
+
return false;
|
|
3705
|
+
}
|
|
3706
|
+
break;
|
|
3707
|
+
}
|
|
3708
|
+
}
|
|
3709
|
+
return this.workers.length > 0;
|
|
3710
|
+
} finally {
|
|
3711
|
+
this.creating = false;
|
|
3712
|
+
}
|
|
3713
|
+
}
|
|
3714
|
+
/**
|
|
3715
|
+
* Parse files in parallel across the worker pool. Returns a flat
|
|
3716
|
+
* `FileSymbols[]` in completion order (caller sorts if needed).
|
|
3717
|
+
*
|
|
3718
|
+
* Content is pre-read by the main thread (for the content-hash check)
|
|
3719
|
+
* and passed to workers to avoid a second disk read. Files are
|
|
3720
|
+
* distributed round-robin across workers.
|
|
3721
|
+
*/
|
|
3722
|
+
async parseFiles(files) {
|
|
3723
|
+
if (!this.isAvailable()) {
|
|
3724
|
+
throw new Error("ParserWorkerPool.parseFiles called before ensureReady() succeeded");
|
|
3725
|
+
}
|
|
3726
|
+
if (files.length === 0) return [];
|
|
3727
|
+
const batchId = this.nextBatchId++;
|
|
3728
|
+
const workerCount = Math.min(this.workers.length, files.length);
|
|
3729
|
+
const chunks = Array.from(
|
|
3730
|
+
{ length: workerCount },
|
|
3731
|
+
() => []
|
|
3732
|
+
);
|
|
3733
|
+
for (let i = 0; i < files.length; i++) {
|
|
3734
|
+
chunks[i % workerCount].push(files[i]);
|
|
3735
|
+
}
|
|
3736
|
+
return new Promise((resolve2, reject) => {
|
|
3737
|
+
this.pending.set(batchId, {
|
|
3738
|
+
resolve: resolve2,
|
|
3739
|
+
reject,
|
|
3740
|
+
accumulated: [],
|
|
3741
|
+
expectedWorkers: workerCount,
|
|
3742
|
+
completedWorkers: 0
|
|
3743
|
+
});
|
|
3744
|
+
for (let i = 0; i < workerCount; i++) {
|
|
3745
|
+
const pw = this.workers[i];
|
|
3746
|
+
pw.busy = true;
|
|
3747
|
+
pw.worker.postMessage({
|
|
3748
|
+
type: "parse",
|
|
3749
|
+
id: batchId,
|
|
3750
|
+
files: chunks[i]
|
|
3751
|
+
});
|
|
3752
|
+
}
|
|
3753
|
+
});
|
|
3754
|
+
}
|
|
3755
|
+
/** Shut down all workers. Safe to call multiple times. */
|
|
3756
|
+
async shutdown() {
|
|
3757
|
+
const workers = this.workers.map((w) => w.worker);
|
|
3758
|
+
this.workers = [];
|
|
3759
|
+
this.unavailable = false;
|
|
3760
|
+
for (const w of workers) {
|
|
3761
|
+
try {
|
|
3762
|
+
w.postMessage({ type: "shutdown" });
|
|
3763
|
+
} catch {
|
|
3764
|
+
}
|
|
3765
|
+
}
|
|
3766
|
+
await Promise.allSettled(
|
|
3767
|
+
workers.map(
|
|
3768
|
+
(w) => Promise.race([
|
|
3769
|
+
new Promise((resolve2) => {
|
|
3770
|
+
w.once("exit", () => resolve2());
|
|
3771
|
+
}),
|
|
3772
|
+
new Promise((resolve2) => setTimeout(() => resolve2(), 2e3))
|
|
3773
|
+
]).then(() => {
|
|
3774
|
+
if (!w.threadId) return;
|
|
3775
|
+
return w.terminate().catch(() => {
|
|
3776
|
+
});
|
|
3777
|
+
})
|
|
3778
|
+
)
|
|
3779
|
+
);
|
|
3780
|
+
for (const [, p] of this.pending) p.reject(new Error("ParserWorkerPool shut down"));
|
|
3781
|
+
this.pending.clear();
|
|
3782
|
+
}
|
|
3783
|
+
handleMessage(msg) {
|
|
3784
|
+
const batch = this.pending.get(msg.id);
|
|
3785
|
+
if (!batch) return;
|
|
3786
|
+
batch.accumulated.push(...msg.results);
|
|
3787
|
+
batch.completedWorkers++;
|
|
3788
|
+
const freeWorker = this.workers.find((w) => w.busy);
|
|
3789
|
+
if (freeWorker) freeWorker.busy = false;
|
|
3790
|
+
if (batch.completedWorkers >= batch.expectedWorkers) {
|
|
3791
|
+
this.pending.delete(msg.id);
|
|
3792
|
+
batch.resolve(batch.accumulated);
|
|
3793
|
+
}
|
|
3794
|
+
}
|
|
3795
|
+
handleError(err, source) {
|
|
3796
|
+
this.workers = this.workers.filter((w) => w.worker !== source);
|
|
3797
|
+
if (this.workers.length === 0) {
|
|
3798
|
+
for (const [, p] of this.pending) p.reject(err);
|
|
3799
|
+
this.pending.clear();
|
|
3800
|
+
this.unavailable = true;
|
|
3801
|
+
}
|
|
3802
|
+
}
|
|
3803
|
+
};
|
|
3804
|
+
function defaultWorkerCount() {
|
|
3805
|
+
const cores = globalThis.navigator?.hardwareConcurrency ?? 4;
|
|
3806
|
+
return Math.max(1, Math.min(4, cores - 1));
|
|
3807
|
+
}
|
|
3808
|
+
function resolveWorkerScriptUrl() {
|
|
3809
|
+
for (const rel of [
|
|
3810
|
+
"./parser-worker-script.js",
|
|
3811
|
+
"./codebase-index/parser-worker-script.js"
|
|
3812
|
+
]) {
|
|
3813
|
+
try {
|
|
3814
|
+
const url = new URL(rel, import.meta.url);
|
|
3815
|
+
if (url.protocol === "file:" && fs6.existsSync(fileURLToPath2(url))) return url;
|
|
3816
|
+
} catch {
|
|
3817
|
+
}
|
|
3818
|
+
}
|
|
3819
|
+
return null;
|
|
3820
|
+
}
|
|
3821
|
+
var _pool = null;
|
|
3822
|
+
function getParserPool() {
|
|
3823
|
+
_pool ??= new ParserWorkerPool();
|
|
3824
|
+
return _pool;
|
|
3825
|
+
}
|
|
3826
|
+
|
|
2977
3827
|
// src/codebase-index/writer.ts
|
|
2978
3828
|
import { expectDefined as expectDefined4 } from "@wrongstack/core/utils";
|
|
2979
|
-
import * as
|
|
2980
|
-
import * as
|
|
3829
|
+
import * as fs8 from "node:fs";
|
|
3830
|
+
import * as path11 from "node:path";
|
|
2981
3831
|
|
|
2982
3832
|
// src/codebase-index/bm25.ts
|
|
2983
3833
|
var K1 = 1.5;
|
|
@@ -3251,8 +4101,8 @@ function runSqliteWithRetry(fn) {
|
|
|
3251
4101
|
}
|
|
3252
4102
|
|
|
3253
4103
|
// src/codebase-index/writer-admin.ts
|
|
3254
|
-
import * as
|
|
3255
|
-
import * as
|
|
4104
|
+
import * as fs7 from "node:fs";
|
|
4105
|
+
import * as path10 from "node:path";
|
|
3256
4106
|
var DB_FILE = "index.db";
|
|
3257
4107
|
function getAllIndexableWithStatement(stmt) {
|
|
3258
4108
|
return stmt("SELECT id, text FROM symbols").all().map(({ id, text }) => ({ id, text }));
|
|
@@ -3288,7 +4138,7 @@ function getMetadataWithStatement(stmt, key) {
|
|
|
3288
4138
|
}
|
|
3289
4139
|
function getFileMetaWithStatement(stmt, file) {
|
|
3290
4140
|
const rows = stmt(
|
|
3291
|
-
"SELECT file, lang, mtime_ms, symbol_count, last_indexed FROM files WHERE file = ?"
|
|
4141
|
+
"SELECT file, lang, mtime_ms, symbol_count, last_indexed, content_hash FROM files WHERE file = ?"
|
|
3292
4142
|
).all(file);
|
|
3293
4143
|
const r = rows[0];
|
|
3294
4144
|
if (!r) return null;
|
|
@@ -3297,21 +4147,25 @@ function getFileMetaWithStatement(stmt, file) {
|
|
|
3297
4147
|
lang: r.lang,
|
|
3298
4148
|
mtimeMs: r.mtime_ms,
|
|
3299
4149
|
symbolCount: r.symbol_count,
|
|
3300
|
-
lastIndexed: r.last_indexed
|
|
4150
|
+
lastIndexed: r.last_indexed,
|
|
4151
|
+
contentHash: r.content_hash
|
|
3301
4152
|
};
|
|
3302
4153
|
}
|
|
3303
4154
|
function getAllFileMetasWithStatement(stmt) {
|
|
3304
|
-
return stmt(
|
|
4155
|
+
return stmt(
|
|
4156
|
+
"SELECT file, lang, mtime_ms, symbol_count, last_indexed, content_hash FROM files"
|
|
4157
|
+
).all().map((r) => ({
|
|
3305
4158
|
file: r.file,
|
|
3306
4159
|
lang: r.lang,
|
|
3307
4160
|
mtimeMs: r.mtime_ms,
|
|
3308
4161
|
symbolCount: r.symbol_count,
|
|
3309
|
-
lastIndexed: r.last_indexed
|
|
4162
|
+
lastIndexed: r.last_indexed,
|
|
4163
|
+
contentHash: r.content_hash
|
|
3310
4164
|
}));
|
|
3311
4165
|
}
|
|
3312
4166
|
function getIndexDbSizeBytes(indexDir) {
|
|
3313
4167
|
try {
|
|
3314
|
-
return
|
|
4168
|
+
return fs7.statSync(path10.join(indexDir, DB_FILE)).size;
|
|
3315
4169
|
} catch {
|
|
3316
4170
|
return 0;
|
|
3317
4171
|
}
|
|
@@ -3360,6 +4214,18 @@ function bulkInsertFtsWithStatement(stmt, maxSqlVars, ftsAvailable, rows) {
|
|
|
3360
4214
|
insert.run(...binds);
|
|
3361
4215
|
}
|
|
3362
4216
|
}
|
|
4217
|
+
function bulkInsertVectorsWithStatement(stmt, maxSqlVars, rows) {
|
|
4218
|
+
if (rows.length === 0) return;
|
|
4219
|
+
const chunkSize = Math.max(1, Math.floor(maxSqlVars / 2));
|
|
4220
|
+
for (let i = 0; i < rows.length; i += chunkSize) {
|
|
4221
|
+
const chunk = rows.slice(i, i + chunkSize);
|
|
4222
|
+
const placeholders = chunk.map(() => "(?, ?)").join(", ");
|
|
4223
|
+
const insert = stmt(`INSERT INTO symbol_vectors(symbol_id, vector) VALUES ${placeholders}`);
|
|
4224
|
+
const binds = [];
|
|
4225
|
+
for (const r of chunk) binds.push(r.id, r.vector);
|
|
4226
|
+
insert.run(...binds);
|
|
4227
|
+
}
|
|
4228
|
+
}
|
|
3363
4229
|
function bulkInsertRefsWithStatement(stmt, maxSqlVars, refs) {
|
|
3364
4230
|
if (refs.length === 0) return;
|
|
3365
4231
|
const chunkSize = Math.max(1, Math.floor(maxSqlVars / 8));
|
|
@@ -3664,6 +4530,171 @@ function findOutgoingCallsByName(stmt, symbolName, file, limit) {
|
|
|
3664
4530
|
const calls = rows.map(mapCallSiteRow).slice(0, limit);
|
|
3665
4531
|
return { calls, symbolFound: true, unresolvedCount, totalMatches: rows.length };
|
|
3666
4532
|
}
|
|
4533
|
+
function runCteWithSeeds(stmt, seedIds, buildSql) {
|
|
4534
|
+
if (seedIds.length <= 900) {
|
|
4535
|
+
const ph = seedIds.map(() => "?").join(",");
|
|
4536
|
+
return stmt(buildSql(ph)).all(...seedIds);
|
|
4537
|
+
}
|
|
4538
|
+
stmt("DROP TABLE IF EXISTS _cte_seeds").run();
|
|
4539
|
+
try {
|
|
4540
|
+
stmt("CREATE TEMP TABLE _cte_seeds (id INTEGER PRIMARY KEY)").run();
|
|
4541
|
+
for (let i = 0; i < seedIds.length; i += 500) {
|
|
4542
|
+
const chunk = seedIds.slice(i, i + 500);
|
|
4543
|
+
const ph = chunk.map(() => "(?)").join(",");
|
|
4544
|
+
stmt(`INSERT OR IGNORE INTO _cte_seeds (id) VALUES ${ph}`).run(...chunk);
|
|
4545
|
+
}
|
|
4546
|
+
return stmt(buildSql("SELECT id FROM _cte_seeds")).all();
|
|
4547
|
+
} finally {
|
|
4548
|
+
stmt("DROP TABLE IF EXISTS _cte_seeds").run();
|
|
4549
|
+
}
|
|
4550
|
+
}
|
|
4551
|
+
function findTransitiveIncomingCallsByName(stmt, symbolName, file, limit) {
|
|
4552
|
+
const targetIds = resolveSymbolIds(stmt, symbolName, file);
|
|
4553
|
+
if (targetIds.length === 0)
|
|
4554
|
+
return { calls: [], symbolFound: false, ambiguous: false, totalMatches: 0 };
|
|
4555
|
+
let matchIds = targetIds;
|
|
4556
|
+
let ambiguous = false;
|
|
4557
|
+
if (file !== void 0) {
|
|
4558
|
+
const allNamedIds = resolveSymbolIds(stmt, symbolName, void 0);
|
|
4559
|
+
if (allNamedIds.length > targetIds.length) {
|
|
4560
|
+
matchIds = allNamedIds;
|
|
4561
|
+
ambiguous = true;
|
|
4562
|
+
}
|
|
4563
|
+
}
|
|
4564
|
+
const cteSql = (seedSource) => `WITH RECURSIVE incoming_tree(from_id) AS (
|
|
4565
|
+
SELECT r.from_id
|
|
4566
|
+
FROM refs r
|
|
4567
|
+
WHERE r.to_id IN (${seedSource})
|
|
4568
|
+
|
|
4569
|
+
UNION
|
|
4570
|
+
|
|
4571
|
+
SELECT r.from_id
|
|
4572
|
+
FROM refs r
|
|
4573
|
+
JOIN incoming_tree it ON r.to_id = it.from_id
|
|
4574
|
+
)
|
|
4575
|
+
SELECT
|
|
4576
|
+
s.id AS sym_id,
|
|
4577
|
+
s.name AS sym_name,
|
|
4578
|
+
s.kind AS sym_kind,
|
|
4579
|
+
s.lang AS sym_lang,
|
|
4580
|
+
s.file AS sym_file,
|
|
4581
|
+
s.line AS sym_line,
|
|
4582
|
+
s.signature AS sym_signature,
|
|
4583
|
+
'' AS call_type,
|
|
4584
|
+
0 AS ref_line
|
|
4585
|
+
FROM incoming_tree it
|
|
4586
|
+
JOIN symbols s ON s.id = it.from_id
|
|
4587
|
+
GROUP BY s.id
|
|
4588
|
+
ORDER BY s.file, s.line`;
|
|
4589
|
+
const rows = runCteWithSeeds(stmt, matchIds, cteSql);
|
|
4590
|
+
if (!file) {
|
|
4591
|
+
const fallbackRows = stmt(
|
|
4592
|
+
`SELECT
|
|
4593
|
+
s.id AS sym_id,
|
|
4594
|
+
s.name AS sym_name,
|
|
4595
|
+
s.kind AS sym_kind,
|
|
4596
|
+
s.lang AS sym_lang,
|
|
4597
|
+
s.file AS sym_file,
|
|
4598
|
+
s.line AS sym_line,
|
|
4599
|
+
s.signature AS sym_signature,
|
|
4600
|
+
r.call_type,
|
|
4601
|
+
r.line AS ref_line
|
|
4602
|
+
FROM refs r
|
|
4603
|
+
JOIN symbols s ON s.id = r.from_id
|
|
4604
|
+
WHERE r.to_id IS NULL AND r.to_name = ?
|
|
4605
|
+
ORDER BY r.line, r.id`
|
|
4606
|
+
).all(symbolName);
|
|
4607
|
+
rows.push(...fallbackRows);
|
|
4608
|
+
}
|
|
4609
|
+
rows.sort((a, b) => a.ref_line - b.ref_line || a.sym_id - b.sym_id);
|
|
4610
|
+
const allCalls = rows.map(mapCallSiteRow);
|
|
4611
|
+
return {
|
|
4612
|
+
calls: allCalls.slice(0, limit),
|
|
4613
|
+
symbolFound: true,
|
|
4614
|
+
ambiguous,
|
|
4615
|
+
totalMatches: allCalls.length
|
|
4616
|
+
};
|
|
4617
|
+
}
|
|
4618
|
+
function findTransitiveOutgoingCallsByName(stmt, symbolName, file, limit) {
|
|
4619
|
+
const sourceIds = resolveSymbolIds(stmt, symbolName, file);
|
|
4620
|
+
if (sourceIds.length === 0)
|
|
4621
|
+
return { calls: [], symbolFound: false, unresolvedCount: 0, totalMatches: 0 };
|
|
4622
|
+
const unresolvedCount = chunkedIdScalar(
|
|
4623
|
+
stmt,
|
|
4624
|
+
sourceIds,
|
|
4625
|
+
(ph) => `SELECT COUNT(*) AS n FROM refs WHERE from_id IN (${ph}) AND to_id IS NULL`
|
|
4626
|
+
);
|
|
4627
|
+
const cteSql = (seedSource) => `WITH RECURSIVE outgoing_tree(to_id) AS (
|
|
4628
|
+
SELECT r.to_id
|
|
4629
|
+
FROM refs r
|
|
4630
|
+
WHERE r.from_id IN (${seedSource}) AND r.to_id IS NOT NULL
|
|
4631
|
+
|
|
4632
|
+
UNION
|
|
4633
|
+
|
|
4634
|
+
SELECT r.to_id
|
|
4635
|
+
FROM refs r
|
|
4636
|
+
JOIN outgoing_tree ot ON r.from_id = ot.to_id
|
|
4637
|
+
WHERE r.to_id IS NOT NULL
|
|
4638
|
+
)
|
|
4639
|
+
SELECT
|
|
4640
|
+
s.id AS sym_id,
|
|
4641
|
+
s.name AS sym_name,
|
|
4642
|
+
s.kind AS sym_kind,
|
|
4643
|
+
s.lang AS sym_lang,
|
|
4644
|
+
s.file AS sym_file,
|
|
4645
|
+
s.line AS sym_line,
|
|
4646
|
+
s.signature AS sym_signature,
|
|
4647
|
+
'' AS call_type,
|
|
4648
|
+
0 AS ref_line
|
|
4649
|
+
FROM outgoing_tree ot
|
|
4650
|
+
JOIN symbols s ON s.id = ot.to_id
|
|
4651
|
+
GROUP BY s.id
|
|
4652
|
+
ORDER BY s.file, s.line`;
|
|
4653
|
+
const rows = runCteWithSeeds(stmt, sourceIds, cteSql);
|
|
4654
|
+
const calls = rows.map(mapCallSiteRow).slice(0, limit);
|
|
4655
|
+
return { calls, symbolFound: true, unresolvedCount, totalMatches: rows.length };
|
|
4656
|
+
}
|
|
4657
|
+
function findReachableSymbolIds(stmt, seedIds) {
|
|
4658
|
+
if (seedIds.length === 0) return /* @__PURE__ */ new Set();
|
|
4659
|
+
if (seedIds.length > 900) {
|
|
4660
|
+
stmt("DROP TABLE IF EXISTS _seeds").run();
|
|
4661
|
+
try {
|
|
4662
|
+
stmt("CREATE TEMP TABLE _seeds (id INTEGER PRIMARY KEY)").run();
|
|
4663
|
+
for (let i = 0; i < seedIds.length; i += 500) {
|
|
4664
|
+
const chunk = seedIds.slice(i, i + 500);
|
|
4665
|
+
const ph2 = chunk.map(() => "(?)").join(",");
|
|
4666
|
+
stmt(`INSERT OR IGNORE INTO _seeds (id) VALUES ${ph2}`).run(...chunk);
|
|
4667
|
+
}
|
|
4668
|
+
const rows2 = stmt(
|
|
4669
|
+
`WITH RECURSIVE reachable(id) AS (
|
|
4670
|
+
SELECT id FROM _seeds
|
|
4671
|
+
UNION
|
|
4672
|
+
SELECT r.to_id
|
|
4673
|
+
FROM refs r
|
|
4674
|
+
JOIN reachable ON r.from_id = reachable.id
|
|
4675
|
+
WHERE r.to_id IS NOT NULL
|
|
4676
|
+
)
|
|
4677
|
+
SELECT DISTINCT id FROM reachable`
|
|
4678
|
+
).all();
|
|
4679
|
+
return new Set(rows2.map((r) => r.id));
|
|
4680
|
+
} finally {
|
|
4681
|
+
stmt("DROP TABLE IF EXISTS _seeds").run();
|
|
4682
|
+
}
|
|
4683
|
+
}
|
|
4684
|
+
const ph = seedIds.map(() => "?").join(",");
|
|
4685
|
+
const rows = stmt(
|
|
4686
|
+
`WITH RECURSIVE reachable(id) AS (
|
|
4687
|
+
SELECT id FROM symbols WHERE id IN (${ph})
|
|
4688
|
+
UNION
|
|
4689
|
+
SELECT r.to_id
|
|
4690
|
+
FROM refs r
|
|
4691
|
+
JOIN reachable ON r.from_id = reachable.id
|
|
4692
|
+
WHERE r.to_id IS NOT NULL
|
|
4693
|
+
)
|
|
4694
|
+
SELECT DISTINCT id FROM reachable`
|
|
4695
|
+
).all(...seedIds);
|
|
4696
|
+
return new Set(rows.map((r) => r.id));
|
|
4697
|
+
}
|
|
3667
4698
|
function findRefsToWithStatement(stmt, symbolId) {
|
|
3668
4699
|
return stmt(
|
|
3669
4700
|
"SELECT id, from_id, to_name, to_id, call_type, line FROM refs WHERE to_id = ? OR to_name = (SELECT name FROM symbols WHERE id = ?)"
|
|
@@ -3903,6 +4934,12 @@ var CORE_TABLES_SQL = `
|
|
|
3903
4934
|
file TEXT PRIMARY KEY,
|
|
3904
4935
|
lang TEXT NOT NULL,
|
|
3905
4936
|
mtime_ms INTEGER NOT NULL,
|
|
4937
|
+
-- Phase 2: xxHash64 of the file's UTF-8 bytes. Empty string when the
|
|
4938
|
+
-- indexer hasn't populated it yet (legacy rows, schema repaired by
|
|
4939
|
+
-- repairMissingColumns). Compared on incremental re-index so that a
|
|
4940
|
+
-- touch or branch-switch that leaves content byte-identical skips the
|
|
4941
|
+
-- expensive parse phase entirely (refactoring proposal Phase 2).
|
|
4942
|
+
content_hash TEXT NOT NULL DEFAULT '',
|
|
3906
4943
|
symbol_count INTEGER NOT NULL DEFAULT 0,
|
|
3907
4944
|
last_indexed INTEGER NOT NULL,
|
|
3908
4945
|
-- Code Atlas grouping label, computed at index time from the ecosystem's
|
|
@@ -3970,7 +5007,14 @@ var LANG_FAMILY_TABLE_SQL = `
|
|
|
3970
5007
|
);
|
|
3971
5008
|
`;
|
|
3972
5009
|
var LANG_FAMILY_WILDCARD = "*";
|
|
3973
|
-
var SYMBOLS_FTS_SQL = "CREATE VIRTUAL TABLE IF NOT EXISTS symbols_fts USING fts5(text, tokenize = '
|
|
5010
|
+
var SYMBOLS_FTS_SQL = "CREATE VIRTUAL TABLE IF NOT EXISTS symbols_fts USING fts5(text, tokenize = 'trigram')";
|
|
5011
|
+
var SYMBOL_VECTORS_TABLE_SQL = `
|
|
5012
|
+
CREATE TABLE IF NOT EXISTS symbol_vectors (
|
|
5013
|
+
symbol_id INTEGER PRIMARY KEY,
|
|
5014
|
+
vector BLOB NOT NULL,
|
|
5015
|
+
FOREIGN KEY (symbol_id) REFERENCES symbols(id) ON DELETE CASCADE
|
|
5016
|
+
);
|
|
5017
|
+
`;
|
|
3974
5018
|
|
|
3975
5019
|
// src/codebase-index/writer-search-helpers.ts
|
|
3976
5020
|
var SEARCH_CANDIDATE_SCAN_CAP = 5e3;
|
|
@@ -4111,6 +5155,85 @@ var StorePool = class {
|
|
|
4111
5155
|
}
|
|
4112
5156
|
};
|
|
4113
5157
|
|
|
5158
|
+
// src/codebase-index/vector-search.ts
|
|
5159
|
+
var RRF_K = 60;
|
|
5160
|
+
var VECTOR_DIMENSIONS = 384;
|
|
5161
|
+
var NGRAM_SIZE = 3;
|
|
5162
|
+
function embedText(text) {
|
|
5163
|
+
const vec = new Float32Array(VECTOR_DIMENSIONS);
|
|
5164
|
+
const normalized = text.toLowerCase().trim();
|
|
5165
|
+
if (normalized.length < NGRAM_SIZE) {
|
|
5166
|
+
const padded = ` ${normalized} `.slice(0, Math.max(NGRAM_SIZE, normalized.length + 2));
|
|
5167
|
+
for (let i = 0; i <= padded.length - NGRAM_SIZE; i++) {
|
|
5168
|
+
const ngram = padded.slice(i, i + NGRAM_SIZE);
|
|
5169
|
+
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
5170
|
+
vec[bucket] += 1;
|
|
5171
|
+
}
|
|
5172
|
+
} else {
|
|
5173
|
+
for (let i = 0; i <= normalized.length - NGRAM_SIZE; i++) {
|
|
5174
|
+
const ngram = normalized.slice(i, i + NGRAM_SIZE);
|
|
5175
|
+
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
5176
|
+
vec[bucket] += 1;
|
|
5177
|
+
}
|
|
5178
|
+
}
|
|
5179
|
+
let norm = 0;
|
|
5180
|
+
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
5181
|
+
norm += vec[i] * vec[i];
|
|
5182
|
+
}
|
|
5183
|
+
norm = Math.sqrt(norm);
|
|
5184
|
+
if (norm > 0) {
|
|
5185
|
+
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
5186
|
+
vec[i] /= norm;
|
|
5187
|
+
}
|
|
5188
|
+
}
|
|
5189
|
+
return vec;
|
|
5190
|
+
}
|
|
5191
|
+
function hashNgram(str) {
|
|
5192
|
+
let hash = 2166136261;
|
|
5193
|
+
for (let i = 0; i < str.length; i++) {
|
|
5194
|
+
hash ^= str.charCodeAt(i);
|
|
5195
|
+
hash = Math.imul(hash, 16777619);
|
|
5196
|
+
}
|
|
5197
|
+
return hash >>> 0;
|
|
5198
|
+
}
|
|
5199
|
+
function cosineSimilarity(a, b) {
|
|
5200
|
+
let dot = 0;
|
|
5201
|
+
const len = Math.min(a.length, b.length);
|
|
5202
|
+
for (let i = 0; i < len; i++) {
|
|
5203
|
+
dot += a[i] * b[i];
|
|
5204
|
+
}
|
|
5205
|
+
return dot;
|
|
5206
|
+
}
|
|
5207
|
+
function encodeVector(vec) {
|
|
5208
|
+
return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
|
|
5209
|
+
}
|
|
5210
|
+
function decodeVector(buf) {
|
|
5211
|
+
const view = new DataView(
|
|
5212
|
+
buf.buffer,
|
|
5213
|
+
buf.byteOffset,
|
|
5214
|
+
buf.byteLength
|
|
5215
|
+
);
|
|
5216
|
+
const copy = new Float32Array(buf.byteLength / 4);
|
|
5217
|
+
for (let i = 0; i < copy.length; i++) {
|
|
5218
|
+
copy[i] = view.getFloat32(i * 4, true);
|
|
5219
|
+
}
|
|
5220
|
+
return copy;
|
|
5221
|
+
}
|
|
5222
|
+
function reciprocalRankFusion(bm25Ranks, vectorRanks, k = RRF_K) {
|
|
5223
|
+
const allIds = /* @__PURE__ */ new Set([...bm25Ranks.keys(), ...vectorRanks.keys()]);
|
|
5224
|
+
const scored = [];
|
|
5225
|
+
for (const id of allIds) {
|
|
5226
|
+
const bm25Rank = bm25Ranks.get(id);
|
|
5227
|
+
const vecRank = vectorRanks.get(id);
|
|
5228
|
+
let score = 0;
|
|
5229
|
+
if (bm25Rank !== void 0) score += 1 / (k + bm25Rank);
|
|
5230
|
+
if (vecRank !== void 0) score += 1 / (k + vecRank);
|
|
5231
|
+
scored.push([id, score]);
|
|
5232
|
+
}
|
|
5233
|
+
scored.sort((a, b) => b[1] - a[1]);
|
|
5234
|
+
return scored;
|
|
5235
|
+
}
|
|
5236
|
+
|
|
4114
5237
|
// src/codebase-index/writer.ts
|
|
4115
5238
|
var DB_FILE2 = "index.db";
|
|
4116
5239
|
var MAX_STATEMENT_CACHE = 128;
|
|
@@ -4123,6 +5246,12 @@ var IndexStore = class _IndexStore {
|
|
|
4123
5246
|
* When false, ranked search falls back to the LIKE + in-process BM25 path.
|
|
4124
5247
|
*/
|
|
4125
5248
|
ftsAvailable = false;
|
|
5249
|
+
/**
|
|
5250
|
+
* Phase 3: true when the `symbol_vectors` table was created successfully.
|
|
5251
|
+
* When false, hybrid search skips the vector pass and falls back to FTS5
|
|
5252
|
+
* (or LIKE) only.
|
|
5253
|
+
*/
|
|
5254
|
+
vectorsAvailable = false;
|
|
4126
5255
|
/**
|
|
4127
5256
|
* Cache of prepared statements keyed by their SQL text. `DatabaseSync`
|
|
4128
5257
|
* compiles SQL on every `.prepare()` call; for the fixed-SQL methods
|
|
@@ -4181,9 +5310,9 @@ var IndexStore = class _IndexStore {
|
|
|
4181
5310
|
}
|
|
4182
5311
|
constructor(projectRoot, opts = {}) {
|
|
4183
5312
|
this.indexDir = resolveIndexDir(projectRoot, opts.indexDir);
|
|
4184
|
-
|
|
5313
|
+
fs8.mkdirSync(this.indexDir, { recursive: true });
|
|
4185
5314
|
const Database = loadDatabaseSync();
|
|
4186
|
-
this.db = new Database(
|
|
5315
|
+
this.db = new Database(path11.join(this.indexDir, DB_FILE2));
|
|
4187
5316
|
applyIndexStorePragmas(this.db);
|
|
4188
5317
|
this.initSchema();
|
|
4189
5318
|
}
|
|
@@ -4221,7 +5350,13 @@ var IndexStore = class _IndexStore {
|
|
|
4221
5350
|
*/
|
|
4222
5351
|
repairMissingColumns() {
|
|
4223
5352
|
const expected = [
|
|
4224
|
-
{
|
|
5353
|
+
{
|
|
5354
|
+
table: "files",
|
|
5355
|
+
columns: [
|
|
5356
|
+
["package", "TEXT NOT NULL DEFAULT ''"],
|
|
5357
|
+
["content_hash", "TEXT NOT NULL DEFAULT ''"]
|
|
5358
|
+
]
|
|
5359
|
+
},
|
|
4225
5360
|
{
|
|
4226
5361
|
table: "refs",
|
|
4227
5362
|
columns: [
|
|
@@ -4253,6 +5388,7 @@ var IndexStore = class _IndexStore {
|
|
|
4253
5388
|
DROP TABLE IF EXISTS symbols;
|
|
4254
5389
|
DROP TABLE IF EXISTS files;
|
|
4255
5390
|
DROP TABLE IF EXISTS refs;
|
|
5391
|
+
DROP TABLE IF EXISTS symbol_vectors;
|
|
4256
5392
|
`);
|
|
4257
5393
|
this.db.exec("DROP TABLE IF EXISTS symbols_fts");
|
|
4258
5394
|
this.stmt("UPDATE metadata SET value = ? WHERE key = ?").run(
|
|
@@ -4274,6 +5410,12 @@ var IndexStore = class _IndexStore {
|
|
|
4274
5410
|
this.db.exec(LANG_FAMILY_TABLE_SQL);
|
|
4275
5411
|
this.seedLangFamilies();
|
|
4276
5412
|
try {
|
|
5413
|
+
const ftsSchema = this.stmt(
|
|
5414
|
+
"SELECT sql FROM sqlite_master WHERE type='table' AND name='symbols_fts'"
|
|
5415
|
+
).get();
|
|
5416
|
+
if (ftsSchema?.sql?.includes("unicode61")) {
|
|
5417
|
+
this.db.exec("DROP TABLE IF EXISTS symbols_fts");
|
|
5418
|
+
}
|
|
4277
5419
|
this.db.exec(SYMBOLS_FTS_SQL);
|
|
4278
5420
|
this.ftsAvailable = true;
|
|
4279
5421
|
const symbolCount = Number(
|
|
@@ -4284,6 +5426,7 @@ var IndexStore = class _IndexStore {
|
|
|
4284
5426
|
);
|
|
4285
5427
|
if (symbolCount !== ftsCount) {
|
|
4286
5428
|
this.db.exec("DELETE FROM symbols_fts");
|
|
5429
|
+
if (this.vectorsAvailable) this.db.exec("DELETE FROM symbol_vectors");
|
|
4287
5430
|
const rows = this.stmt(
|
|
4288
5431
|
"SELECT id, name, signature, doc_comment FROM symbols ORDER BY id"
|
|
4289
5432
|
).all();
|
|
@@ -4301,6 +5444,12 @@ var IndexStore = class _IndexStore {
|
|
|
4301
5444
|
} catch {
|
|
4302
5445
|
this.ftsAvailable = false;
|
|
4303
5446
|
}
|
|
5447
|
+
try {
|
|
5448
|
+
this.db.exec(SYMBOL_VECTORS_TABLE_SQL);
|
|
5449
|
+
this.vectorsAvailable = true;
|
|
5450
|
+
} catch {
|
|
5451
|
+
this.vectorsAvailable = false;
|
|
5452
|
+
}
|
|
4304
5453
|
this.ensureNextSymbolIdSeeded();
|
|
4305
5454
|
}
|
|
4306
5455
|
// ─── ID allocation & bulk helpers ────────────────────────────────────────────
|
|
@@ -4410,6 +5559,7 @@ var IndexStore = class _IndexStore {
|
|
|
4410
5559
|
const result = [];
|
|
4411
5560
|
const bulk = [];
|
|
4412
5561
|
const ftsRows = [];
|
|
5562
|
+
const vectorRows = [];
|
|
4413
5563
|
for (const s of symbols) {
|
|
4414
5564
|
const id = nextId++;
|
|
4415
5565
|
bulk.push({
|
|
@@ -4428,6 +5578,10 @@ var IndexStore = class _IndexStore {
|
|
|
4428
5578
|
if (this.ftsAvailable) {
|
|
4429
5579
|
ftsRows.push({ id, text: buildIndexableText(s.name, s.signature, s.docComment) });
|
|
4430
5580
|
}
|
|
5581
|
+
vectorRows.push({
|
|
5582
|
+
id,
|
|
5583
|
+
vector: encodeVector(embedText(s.text || buildIndexableText(s.name, s.signature, s.docComment)))
|
|
5584
|
+
});
|
|
4431
5585
|
result.push({ ...s, id });
|
|
4432
5586
|
}
|
|
4433
5587
|
bulkInsertSymbolsWithStatement((sql) => this.stmt(sql), _IndexStore.MAX_SQL_VARS, bulk);
|
|
@@ -4437,6 +5591,13 @@ var IndexStore = class _IndexStore {
|
|
|
4437
5591
|
this.ftsAvailable,
|
|
4438
5592
|
ftsRows
|
|
4439
5593
|
);
|
|
5594
|
+
if (this.vectorsAvailable) {
|
|
5595
|
+
bulkInsertVectorsWithStatement(
|
|
5596
|
+
(sql) => this.stmt(sql),
|
|
5597
|
+
_IndexStore.MAX_SQL_VARS,
|
|
5598
|
+
vectorRows
|
|
5599
|
+
);
|
|
5600
|
+
}
|
|
4440
5601
|
this.db.exec("COMMIT");
|
|
4441
5602
|
return result;
|
|
4442
5603
|
} catch (err) {
|
|
@@ -4456,6 +5617,11 @@ var IndexStore = class _IndexStore {
|
|
|
4456
5617
|
"DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
4457
5618
|
).run(file);
|
|
4458
5619
|
}
|
|
5620
|
+
if (this.vectorsAvailable) {
|
|
5621
|
+
this.stmt(
|
|
5622
|
+
"DELETE FROM symbol_vectors WHERE symbol_id IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
5623
|
+
).run(file);
|
|
5624
|
+
}
|
|
4459
5625
|
this.stmt("DELETE FROM symbols WHERE file_fk = ?").run(file);
|
|
4460
5626
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
4461
5627
|
this.db.exec("COMMIT");
|
|
@@ -4481,6 +5647,11 @@ var IndexStore = class _IndexStore {
|
|
|
4481
5647
|
"DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
4482
5648
|
).run(file);
|
|
4483
5649
|
}
|
|
5650
|
+
if (this.vectorsAvailable) {
|
|
5651
|
+
this.stmt(
|
|
5652
|
+
"DELETE FROM symbol_vectors WHERE symbol_id IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
5653
|
+
).run(file);
|
|
5654
|
+
}
|
|
4484
5655
|
this.stmt(
|
|
4485
5656
|
"DELETE FROM refs WHERE from_id IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
4486
5657
|
).run(file);
|
|
@@ -4498,14 +5669,22 @@ var IndexStore = class _IndexStore {
|
|
|
4498
5669
|
upsertFile(meta) {
|
|
4499
5670
|
this.runWithRetry(() => {
|
|
4500
5671
|
this.stmt(
|
|
4501
|
-
`INSERT INTO files(file, lang, mtime_ms, symbol_count, last_indexed)
|
|
4502
|
-
VALUES (?, ?, ?, ?, ?)
|
|
5672
|
+
`INSERT INTO files(file, lang, mtime_ms, content_hash, symbol_count, last_indexed)
|
|
5673
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
4503
5674
|
ON CONFLICT(file) DO UPDATE SET
|
|
4504
5675
|
lang = excluded.lang,
|
|
4505
5676
|
mtime_ms = excluded.mtime_ms,
|
|
5677
|
+
content_hash = excluded.content_hash,
|
|
4506
5678
|
symbol_count = excluded.symbol_count,
|
|
4507
5679
|
last_indexed = excluded.last_indexed`
|
|
4508
|
-
).run(
|
|
5680
|
+
).run(
|
|
5681
|
+
meta.file,
|
|
5682
|
+
meta.lang,
|
|
5683
|
+
meta.mtimeMs,
|
|
5684
|
+
meta.contentHash ?? "",
|
|
5685
|
+
meta.symbolCount,
|
|
5686
|
+
meta.lastIndexed
|
|
5687
|
+
);
|
|
4509
5688
|
});
|
|
4510
5689
|
}
|
|
4511
5690
|
getFileMeta(file) {
|
|
@@ -4672,9 +5851,18 @@ var IndexStore = class _IndexStore {
|
|
|
4672
5851
|
if (mapped === null) return { results: [], total: 0 };
|
|
4673
5852
|
effectiveKind = mapped;
|
|
4674
5853
|
}
|
|
4675
|
-
const
|
|
5854
|
+
const longTokens = tokens.filter((t) => t.length >= 3);
|
|
5855
|
+
const shortTokens = tokens.filter((t) => t.length < 3);
|
|
5856
|
+
if (longTokens.length === 0) {
|
|
5857
|
+
return this.searchRankedFallback(query, filter, safeLimit);
|
|
5858
|
+
}
|
|
5859
|
+
const match = longTokens.map((t) => `"${t.replaceAll('"', "")}"`).join(" OR ");
|
|
4676
5860
|
const conditions = ["symbols_fts MATCH ?"];
|
|
4677
5861
|
const values = [match];
|
|
5862
|
+
for (const shortTok of shortTokens) {
|
|
5863
|
+
conditions.push("s.text LIKE ? ESCAPE '\\'");
|
|
5864
|
+
values.push(`%${escapeLike(shortTok)}%`);
|
|
5865
|
+
}
|
|
4678
5866
|
if (effectiveKind) {
|
|
4679
5867
|
conditions.push("s.kind = ?");
|
|
4680
5868
|
values.push(effectiveKind);
|
|
@@ -4693,7 +5881,7 @@ var IndexStore = class _IndexStore {
|
|
|
4693
5881
|
).all(...values);
|
|
4694
5882
|
const total = countRows[0] ? Number(countRows[0].n) : 0;
|
|
4695
5883
|
if (total === 0) return { results: [], total: 0 };
|
|
4696
|
-
const
|
|
5884
|
+
const bm25Rows = this.stmt(
|
|
4697
5885
|
`SELECT s.id, s.lang, s.kind, s.name, s.file, s.line, s.col, s.signature, s.doc_comment,
|
|
4698
5886
|
-bm25(symbols_fts) AS score,
|
|
4699
5887
|
snippet(symbols_fts, 0, '', '', '\u2026', 12) AS snippet
|
|
@@ -4706,8 +5894,39 @@ var IndexStore = class _IndexStore {
|
|
|
4706
5894
|
bm25(symbols_fts), lower(s.name), s.file, s.line, s.col, s.id
|
|
4707
5895
|
LIMIT ?`
|
|
4708
5896
|
).all(...values, query.trim(), `${escapeLike(query.trim())}%`, safeLimit);
|
|
5897
|
+
if (this.vectorsAvailable && bm25Rows.length > 0) {
|
|
5898
|
+
const queryVec = embedText(query);
|
|
5899
|
+
const candidateIds = bm25Rows.map((r) => r.id);
|
|
5900
|
+
const placeholders = candidateIds.map(() => "?").join(",");
|
|
5901
|
+
const vecRows = this.stmt(
|
|
5902
|
+
`SELECT sv.symbol_id, sv.vector FROM symbol_vectors sv WHERE sv.symbol_id IN (${placeholders})`
|
|
5903
|
+
).all(...candidateIds);
|
|
5904
|
+
const vecScores = vecRows.map((r) => ({
|
|
5905
|
+
id: r.symbol_id,
|
|
5906
|
+
sim: cosineSimilarity(queryVec, decodeVector(r.vector))
|
|
5907
|
+
})).sort((a, b) => b.sim - a.sim);
|
|
5908
|
+
const bm25Rank = /* @__PURE__ */ new Map();
|
|
5909
|
+
bm25Rows.forEach((r, i) => {
|
|
5910
|
+
bm25Rank.set(r.id, i);
|
|
5911
|
+
});
|
|
5912
|
+
const vecRank = /* @__PURE__ */ new Map();
|
|
5913
|
+
vecScores.forEach((r, i) => {
|
|
5914
|
+
vecRank.set(r.id, i);
|
|
5915
|
+
});
|
|
5916
|
+
const fused = reciprocalRankFusion(bm25Rank, vecRank, 60);
|
|
5917
|
+
const fusedScore = new Map(fused);
|
|
5918
|
+
const sorted = [...bm25Rows].sort(
|
|
5919
|
+
(a, b) => (fusedScore.get(b.id) ?? 0) - (fusedScore.get(a.id) ?? 0)
|
|
5920
|
+
);
|
|
5921
|
+
return {
|
|
5922
|
+
results: sorted.map(
|
|
5923
|
+
(row) => mapWriterSearchRow(row, filter?.lspKind, Math.max(1e-4, row.score), row.snippet)
|
|
5924
|
+
),
|
|
5925
|
+
total
|
|
5926
|
+
};
|
|
5927
|
+
}
|
|
4709
5928
|
return {
|
|
4710
|
-
results:
|
|
5929
|
+
results: bm25Rows.map(
|
|
4711
5930
|
(row) => mapWriterSearchRow(row, filter?.lspKind, Math.max(1e-4, row.score), row.snippet)
|
|
4712
5931
|
),
|
|
4713
5932
|
total
|
|
@@ -4835,6 +6054,7 @@ var IndexStore = class _IndexStore {
|
|
|
4835
6054
|
this.db.exec("DROP TABLE IF EXISTS files");
|
|
4836
6055
|
this.db.exec("DROP TABLE IF EXISTS metadata");
|
|
4837
6056
|
if (this.ftsAvailable) this.db.exec("DROP TABLE IF EXISTS symbols_fts");
|
|
6057
|
+
this.db.exec("DROP TABLE IF EXISTS symbol_vectors");
|
|
4838
6058
|
this.db.exec("COMMIT");
|
|
4839
6059
|
this.stmtCache.clear();
|
|
4840
6060
|
this.initSchema();
|
|
@@ -4922,6 +6142,11 @@ var IndexStore = class _IndexStore {
|
|
|
4922
6142
|
`DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file IN (${placeholders}))`
|
|
4923
6143
|
).run(...options.deleteForFiles);
|
|
4924
6144
|
}
|
|
6145
|
+
if (this.vectorsAvailable) {
|
|
6146
|
+
this.stmt(
|
|
6147
|
+
`DELETE FROM symbol_vectors WHERE symbol_id IN (SELECT id FROM symbols WHERE file IN (${placeholders}))`
|
|
6148
|
+
).run(...options.deleteForFiles);
|
|
6149
|
+
}
|
|
4925
6150
|
this.stmt(
|
|
4926
6151
|
`DELETE FROM refs WHERE from_id IN (SELECT id FROM symbols WHERE file IN (${placeholders}))`
|
|
4927
6152
|
).run(...options.deleteForFiles);
|
|
@@ -4935,6 +6160,7 @@ var IndexStore = class _IndexStore {
|
|
|
4935
6160
|
const refsToInsert = [];
|
|
4936
6161
|
const bulkSyms = [];
|
|
4937
6162
|
const ftsRows = [];
|
|
6163
|
+
const vectorRows = [];
|
|
4938
6164
|
for (const entry of entries) {
|
|
4939
6165
|
const insertedForEntry = [];
|
|
4940
6166
|
for (const s of entry.symbols) {
|
|
@@ -4958,6 +6184,10 @@ var IndexStore = class _IndexStore {
|
|
|
4958
6184
|
text: buildIndexableText(s.name, s.signature, s.docComment)
|
|
4959
6185
|
});
|
|
4960
6186
|
}
|
|
6187
|
+
vectorRows.push({
|
|
6188
|
+
id,
|
|
6189
|
+
vector: encodeVector(embedText(s.text || buildIndexableText(s.name, s.signature, s.docComment)))
|
|
6190
|
+
});
|
|
4961
6191
|
const inserted = { ...s, id };
|
|
4962
6192
|
allInserted.push(inserted);
|
|
4963
6193
|
insertedForEntry.push(inserted);
|
|
@@ -4971,19 +6201,34 @@ var IndexStore = class _IndexStore {
|
|
|
4971
6201
|
this.ftsAvailable,
|
|
4972
6202
|
ftsRows
|
|
4973
6203
|
);
|
|
6204
|
+
if (this.vectorsAvailable) {
|
|
6205
|
+
bulkInsertVectorsWithStatement(
|
|
6206
|
+
(sql) => this.stmt(sql),
|
|
6207
|
+
_IndexStore.MAX_SQL_VARS,
|
|
6208
|
+
vectorRows
|
|
6209
|
+
);
|
|
6210
|
+
}
|
|
4974
6211
|
bulkInsertRefsWithStatement((sql) => this.stmt(sql), _IndexStore.MAX_SQL_VARS, refsToInsert);
|
|
4975
6212
|
const upsertStmt = this.stmt(
|
|
4976
|
-
`INSERT INTO files(file, lang, mtime_ms, symbol_count, last_indexed)
|
|
4977
|
-
VALUES (?, ?, ?, ?, ?)
|
|
6213
|
+
`INSERT INTO files(file, lang, mtime_ms, content_hash, symbol_count, last_indexed)
|
|
6214
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
4978
6215
|
ON CONFLICT(file) DO UPDATE SET
|
|
4979
6216
|
lang = excluded.lang,
|
|
4980
6217
|
mtime_ms = excluded.mtime_ms,
|
|
6218
|
+
content_hash = excluded.content_hash,
|
|
4981
6219
|
symbol_count = excluded.symbol_count,
|
|
4982
6220
|
last_indexed = excluded.last_indexed`
|
|
4983
6221
|
);
|
|
4984
6222
|
const now = Date.now();
|
|
4985
6223
|
for (const entry of entries) {
|
|
4986
|
-
upsertStmt.run(
|
|
6224
|
+
upsertStmt.run(
|
|
6225
|
+
entry.file,
|
|
6226
|
+
entry.lang,
|
|
6227
|
+
entry.mtimeMs,
|
|
6228
|
+
entry.contentHash ?? "",
|
|
6229
|
+
entry.symbolCount,
|
|
6230
|
+
now
|
|
6231
|
+
);
|
|
4987
6232
|
}
|
|
4988
6233
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
4989
6234
|
this.db.exec("COMMIT");
|
|
@@ -5074,19 +6319,32 @@ var IndexStore = class _IndexStore {
|
|
|
5074
6319
|
"DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
5075
6320
|
).run(meta.file);
|
|
5076
6321
|
}
|
|
6322
|
+
if (this.vectorsAvailable) {
|
|
6323
|
+
this.stmt(
|
|
6324
|
+
"DELETE FROM symbol_vectors WHERE symbol_id IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
6325
|
+
).run(meta.file);
|
|
6326
|
+
}
|
|
5077
6327
|
this.stmt(
|
|
5078
6328
|
"DELETE FROM refs WHERE from_id IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
5079
6329
|
).run(meta.file);
|
|
5080
6330
|
this.stmt("DELETE FROM symbols WHERE file_fk = ?").run(meta.file);
|
|
5081
6331
|
this.stmt(
|
|
5082
|
-
`INSERT INTO files(file, lang, mtime_ms, symbol_count, last_indexed)
|
|
5083
|
-
VALUES (?, ?, ?, ?, ?)
|
|
6332
|
+
`INSERT INTO files(file, lang, mtime_ms, content_hash, symbol_count, last_indexed)
|
|
6333
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
5084
6334
|
ON CONFLICT(file) DO UPDATE SET
|
|
5085
6335
|
lang = excluded.lang,
|
|
5086
6336
|
mtime_ms = excluded.mtime_ms,
|
|
6337
|
+
content_hash = excluded.content_hash,
|
|
5087
6338
|
symbol_count = excluded.symbol_count,
|
|
5088
6339
|
last_indexed = excluded.last_indexed`
|
|
5089
|
-
).run(
|
|
6340
|
+
).run(
|
|
6341
|
+
meta.file,
|
|
6342
|
+
meta.lang,
|
|
6343
|
+
meta.mtimeMs,
|
|
6344
|
+
meta.contentHash ?? "",
|
|
6345
|
+
meta.symbolCount,
|
|
6346
|
+
meta.lastIndexed
|
|
6347
|
+
);
|
|
5090
6348
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
5091
6349
|
this.db.exec("COMMIT");
|
|
5092
6350
|
} catch (err) {
|
|
@@ -5149,6 +6407,31 @@ var IndexStore = class _IndexStore {
|
|
|
5149
6407
|
findOutgoingCallsByName(symbolName, file, limit = 100) {
|
|
5150
6408
|
return findOutgoingCallsByName((sql) => this.stmt(sql), symbolName, file, limit);
|
|
5151
6409
|
}
|
|
6410
|
+
/**
|
|
6411
|
+
* Transitive incoming-call tree: all symbols that transitively call the
|
|
6412
|
+
* target, to an unbounded depth (cycle-safe via SQL UNION deduplication).
|
|
6413
|
+
* Used by `codebase-incoming-calls` when the caller wants the full call
|
|
6414
|
+
* chain rather than just direct callers.
|
|
6415
|
+
*/
|
|
6416
|
+
findTransitiveIncomingCallsByName(symbolName, file, limit = 200) {
|
|
6417
|
+
return findTransitiveIncomingCallsByName((sql) => this.stmt(sql), symbolName, file, limit);
|
|
6418
|
+
}
|
|
6419
|
+
/**
|
|
6420
|
+
* Transitive outgoing-call tree: all symbols the target transitively calls.
|
|
6421
|
+
* Used by `codebase-outgoing-calls` when the caller wants the full
|
|
6422
|
+
* dependency chain rather than just direct callees.
|
|
6423
|
+
*/
|
|
6424
|
+
findTransitiveOutgoingCallsByName(symbolName, file, limit = 200) {
|
|
6425
|
+
return findTransitiveOutgoingCallsByName((sql) => this.stmt(sql), symbolName, file, limit);
|
|
6426
|
+
}
|
|
6427
|
+
/**
|
|
6428
|
+
* Compute the set of symbol IDs reachable from the given seed IDs using a
|
|
6429
|
+
* native SQLite recursive CTE. Used by dead-code detection to replace the
|
|
6430
|
+
* in-memory BFS.
|
|
6431
|
+
*/
|
|
6432
|
+
findReachableSymbolIds(seedIds) {
|
|
6433
|
+
return findReachableSymbolIds((sql) => this.stmt(sql), seedIds);
|
|
6434
|
+
}
|
|
5152
6435
|
/**
|
|
5153
6436
|
* Find all references TO a given symbol (who calls / uses this symbol?).
|
|
5154
6437
|
*/
|
|
@@ -5239,6 +6522,9 @@ var YIELD_EVERY_N = 50;
|
|
|
5239
6522
|
function resolveParallelBatch() {
|
|
5240
6523
|
return indexParallelBatchSize(availableParallelism());
|
|
5241
6524
|
}
|
|
6525
|
+
function shouldUseParserWorkerPool(candidateFileCount, parseBatchCount) {
|
|
6526
|
+
return !isFrugalPerf() && candidateFileCount >= WORKER_POOL_THRESHOLD && parseBatchCount > 1;
|
|
6527
|
+
}
|
|
5242
6528
|
function yieldEventLoop() {
|
|
5243
6529
|
return new Promise((resolve2) => setImmediate(resolve2));
|
|
5244
6530
|
}
|
|
@@ -5256,15 +6542,15 @@ var INDEXABLE_EXTENSION_SET = new Set(INDEXABLE_EXTENSIONS);
|
|
|
5256
6542
|
var MAX_INDEX_FILE_BYTES = 5 * 1024 * 1024;
|
|
5257
6543
|
var MAX_GIT_FILE_LIST_BYTES = 64 * 1024 * 1024;
|
|
5258
6544
|
function isWithinProject(projectRoot, file) {
|
|
5259
|
-
const rel =
|
|
5260
|
-
return rel !== "" && !rel.startsWith(`..${
|
|
6545
|
+
const rel = path12.relative(projectRoot, file);
|
|
6546
|
+
return rel !== "" && !rel.startsWith(`..${path12.sep}`) && rel !== ".." && !path12.isAbsolute(rel);
|
|
5261
6547
|
}
|
|
5262
6548
|
function isMissingPathError(err) {
|
|
5263
6549
|
const code = err?.code;
|
|
5264
6550
|
return code === "ENOENT" || code === "ENOTDIR";
|
|
5265
6551
|
}
|
|
5266
6552
|
function normalizeComparablePath(value) {
|
|
5267
|
-
const resolved =
|
|
6553
|
+
const resolved = path12.resolve(value);
|
|
5268
6554
|
return process.platform === "win32" ? resolved.toLowerCase() : resolved;
|
|
5269
6555
|
}
|
|
5270
6556
|
function gitOutput(projectRoot, args) {
|
|
@@ -5309,24 +6595,24 @@ async function findGitSourceFiles(projectRoot, ignore, signal) {
|
|
|
5309
6595
|
const record = statusRecords[i];
|
|
5310
6596
|
if (!record) continue;
|
|
5311
6597
|
const status = record.slice(0, 2);
|
|
5312
|
-
const changedPath =
|
|
6598
|
+
const changedPath = path12.resolve(projectRoot, record.slice(3));
|
|
5313
6599
|
dirty.add(changedPath);
|
|
5314
6600
|
if (status.includes("D")) deleted.add(changedPath);
|
|
5315
6601
|
if (status.includes("R") || status.includes("C")) {
|
|
5316
6602
|
const source = statusRecords[++i];
|
|
5317
|
-
if (source) dirty.add(
|
|
6603
|
+
if (source) dirty.add(path12.resolve(projectRoot, source));
|
|
5318
6604
|
}
|
|
5319
6605
|
}
|
|
5320
6606
|
const files = [];
|
|
5321
6607
|
for (const relative2 of output.toString("utf8").split("\0")) {
|
|
5322
6608
|
if (!relative2) continue;
|
|
5323
6609
|
const portable = relative2.replace(/\\/g, "/");
|
|
5324
|
-
if (portable.split("/").some((segment) => ignoreSet.has(segment)) || DEFAULT_IGNORE_FILES.has(
|
|
6610
|
+
if (portable.split("/").some((segment) => ignoreSet.has(segment)) || DEFAULT_IGNORE_FILES.has(path12.posix.basename(portable))) {
|
|
5325
6611
|
continue;
|
|
5326
6612
|
}
|
|
5327
|
-
const full =
|
|
6613
|
+
const full = path12.resolve(projectRoot, relative2);
|
|
5328
6614
|
if (deleted.has(full)) continue;
|
|
5329
|
-
const ext =
|
|
6615
|
+
const ext = path12.extname(relative2).toLowerCase();
|
|
5330
6616
|
if (INDEXABLE_EXTENSION_SET.has(ext) || detectLang(full) !== null) files.push(full);
|
|
5331
6617
|
}
|
|
5332
6618
|
return {
|
|
@@ -5361,7 +6647,7 @@ async function findSourceFiles(projectRoot, ignore, isGitIgnored, signal) {
|
|
|
5361
6647
|
}
|
|
5362
6648
|
let entries;
|
|
5363
6649
|
try {
|
|
5364
|
-
entries = await
|
|
6650
|
+
entries = await fs9.readdir(dir, { withFileTypes: true });
|
|
5365
6651
|
} catch (err) {
|
|
5366
6652
|
complete = false;
|
|
5367
6653
|
errors.push(`scan error: ${dir}: ${err instanceof Error ? err.message : String(err)}`);
|
|
@@ -5370,14 +6656,14 @@ async function findSourceFiles(projectRoot, ignore, isGitIgnored, signal) {
|
|
|
5370
6656
|
dirCount++;
|
|
5371
6657
|
for (const e of entries) {
|
|
5372
6658
|
if (ignoreSet.has(e.name)) continue;
|
|
5373
|
-
const full =
|
|
5374
|
-
const rel =
|
|
6659
|
+
const full = path12.join(dir, e.name);
|
|
6660
|
+
const rel = path12.relative(projectRoot, full).replace(/\\/g, "/");
|
|
5375
6661
|
if (e.isDirectory()) {
|
|
5376
6662
|
if (isGitIgnored(rel, true)) continue;
|
|
5377
6663
|
await walk(full);
|
|
5378
6664
|
} else if (e.isFile()) {
|
|
5379
6665
|
if (DEFAULT_IGNORE_FILES.has(e.name) || isGitIgnored(rel, false)) continue;
|
|
5380
|
-
const ext =
|
|
6666
|
+
const ext = path12.extname(e.name).toLowerCase();
|
|
5381
6667
|
if (indexableExts.has(ext) || detectLang(full) !== null) {
|
|
5382
6668
|
results.push(full);
|
|
5383
6669
|
}
|
|
@@ -5415,11 +6701,7 @@ async function resolveProjectRelations(store, projectRoot, opts) {
|
|
|
5415
6701
|
const structure = await detectModuleRoots(projectRoot, indexedFiles);
|
|
5416
6702
|
if (opts.signal?.aborted) return;
|
|
5417
6703
|
store.setFilePackages(assignPackageLabels(structure, indexedFiles));
|
|
5418
|
-
const resolver = new ModuleResolver(
|
|
5419
|
-
structure,
|
|
5420
|
-
indexedFiles,
|
|
5421
|
-
store.getNamespaceDeclarations()
|
|
5422
|
-
);
|
|
6704
|
+
const resolver = new ModuleResolver(structure, indexedFiles, store.getNamespaceDeclarations());
|
|
5423
6705
|
const pending = store.getUnresolvedImports(opts.onlyFiles);
|
|
5424
6706
|
const resolutions = [];
|
|
5425
6707
|
for (const entry of pending) {
|
|
@@ -5444,6 +6726,10 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5444
6726
|
const errors = [];
|
|
5445
6727
|
const langStats = {};
|
|
5446
6728
|
let filesIndexed = 0;
|
|
6729
|
+
let filesParsed = 0;
|
|
6730
|
+
let filesSkipped = 0;
|
|
6731
|
+
let filesEmpty = 0;
|
|
6732
|
+
let filesFailed = 0;
|
|
5447
6733
|
let symbolsIndexed = 0;
|
|
5448
6734
|
const isGitIgnored = await loadGitignoreMatcher(projectRoot);
|
|
5449
6735
|
let files;
|
|
@@ -5451,10 +6737,10 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5451
6737
|
let discoveryComplete = true;
|
|
5452
6738
|
let trustedUnchanged;
|
|
5453
6739
|
if (opts.files && opts.files.length > 0) {
|
|
5454
|
-
files = opts.files.map((f) =>
|
|
6740
|
+
files = opts.files.map((f) => path12.resolve(projectRoot, f)).filter((f) => {
|
|
5455
6741
|
if (!isWithinProject(projectRoot, f)) return false;
|
|
5456
|
-
const rel =
|
|
5457
|
-
return !rel.split("/").some((seg) => DEFAULT_IGNORE.includes(seg)) && !DEFAULT_IGNORE_FILES.has(
|
|
6742
|
+
const rel = path12.relative(projectRoot, f).replace(/\\/g, "/");
|
|
6743
|
+
return !rel.split("/").some((seg) => DEFAULT_IGNORE.includes(seg)) && !DEFAULT_IGNORE_FILES.has(path12.basename(f)) && !isGitIgnored(rel, false);
|
|
5458
6744
|
});
|
|
5459
6745
|
} else {
|
|
5460
6746
|
const discovery = await findSourceFiles(projectRoot, ignore, isGitIgnored, signal);
|
|
@@ -5485,12 +6771,14 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5485
6771
|
langStats[meta.lang] = (langStats[meta.lang] ?? 0) + meta.symbolCount;
|
|
5486
6772
|
symbolsIndexed += meta.symbolCount;
|
|
5487
6773
|
filesIndexed++;
|
|
6774
|
+
filesSkipped++;
|
|
5488
6775
|
filesPreSkipped++;
|
|
5489
6776
|
return false;
|
|
5490
6777
|
});
|
|
5491
6778
|
if (filesPreSkipped > 0) opts.onProgress?.(filesPreSkipped, totalFilesForProgress);
|
|
5492
6779
|
}
|
|
5493
6780
|
const parallelBatch = resolveParallelBatch();
|
|
6781
|
+
const parserPoolCandidateCount = files.length;
|
|
5494
6782
|
let filesSinceLastYield = 0;
|
|
5495
6783
|
for (let batchStart = 0; batchStart < files.length; batchStart += parallelBatch) {
|
|
5496
6784
|
const batchEnd = Math.min(batchStart + parallelBatch, files.length);
|
|
@@ -5511,7 +6799,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5511
6799
|
async (file) => {
|
|
5512
6800
|
let stat2;
|
|
5513
6801
|
try {
|
|
5514
|
-
stat2 = await
|
|
6802
|
+
stat2 = await fs9.stat(file, statOpts);
|
|
5515
6803
|
} catch (e) {
|
|
5516
6804
|
if (isAbortError(e)) throw e;
|
|
5517
6805
|
return {
|
|
@@ -5541,7 +6829,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5541
6829
|
}
|
|
5542
6830
|
let content;
|
|
5543
6831
|
try {
|
|
5544
|
-
content = await
|
|
6832
|
+
content = await fs9.readFile(file, { encoding: "utf8", signal });
|
|
5545
6833
|
} catch (e) {
|
|
5546
6834
|
if (isAbortError(e)) throw e;
|
|
5547
6835
|
return {
|
|
@@ -5552,22 +6840,78 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5552
6840
|
error: `read error: ${e instanceof Error ? e.message : String(e)}`
|
|
5553
6841
|
};
|
|
5554
6842
|
}
|
|
5555
|
-
|
|
5556
|
-
|
|
5557
|
-
parsed = await parseFileContent(file, content, lang);
|
|
5558
|
-
} catch (e) {
|
|
6843
|
+
const contentHash = xxhash64String(content);
|
|
6844
|
+
if (!force && meta && meta.contentHash && contentHash === meta.contentHash) {
|
|
5559
6845
|
return {
|
|
5560
6846
|
file,
|
|
5561
6847
|
stat: stat2,
|
|
5562
6848
|
lang,
|
|
5563
6849
|
parsed: null,
|
|
5564
|
-
|
|
6850
|
+
content,
|
|
6851
|
+
contentHash,
|
|
6852
|
+
skippedMeta: { ...meta, mtimeMs: Math.floor(stat2.mtimeMs) }
|
|
5565
6853
|
};
|
|
5566
6854
|
}
|
|
5567
|
-
return { file, stat: stat2, lang, parsed, content };
|
|
6855
|
+
return { file, stat: stat2, lang, parsed: null, content, contentHash };
|
|
5568
6856
|
}
|
|
5569
6857
|
)
|
|
5570
6858
|
);
|
|
6859
|
+
const toParse = [];
|
|
6860
|
+
for (let pi = 0; pi < statReadParse.length; pi++) {
|
|
6861
|
+
const s = statReadParse[pi];
|
|
6862
|
+
if (s.status !== "fulfilled") continue;
|
|
6863
|
+
const r = s.value;
|
|
6864
|
+
if (r.error || r.skippedMeta || !r.lang || r.parsed) continue;
|
|
6865
|
+
if (r.content === void 0) continue;
|
|
6866
|
+
toParse.push({
|
|
6867
|
+
index: pi,
|
|
6868
|
+
file: batchFiles[pi],
|
|
6869
|
+
content: r.content,
|
|
6870
|
+
lang: r.lang
|
|
6871
|
+
});
|
|
6872
|
+
}
|
|
6873
|
+
if (toParse.length > 0) {
|
|
6874
|
+
let pool = shouldUseParserWorkerPool(parserPoolCandidateCount, toParse.length) ? getParserPool() : null;
|
|
6875
|
+
if (pool) {
|
|
6876
|
+
try {
|
|
6877
|
+
await pool.ensureReady();
|
|
6878
|
+
const parsedResults = await pool.parseFiles(
|
|
6879
|
+
toParse.map((p) => ({ file: p.file, content: p.content, lang: p.lang }))
|
|
6880
|
+
);
|
|
6881
|
+
const byFile = new Map(parsedResults.map((r) => [r.file, r]));
|
|
6882
|
+
for (const item of toParse) {
|
|
6883
|
+
const parsed = byFile.get(item.file);
|
|
6884
|
+
const settled = statReadParse[item.index];
|
|
6885
|
+
if (settled.status !== "fulfilled") continue;
|
|
6886
|
+
if (parsed) {
|
|
6887
|
+
settled.value.parsed = parsed;
|
|
6888
|
+
} else {
|
|
6889
|
+
settled.value.error = `parse error: worker returned no result for ${item.file}`;
|
|
6890
|
+
}
|
|
6891
|
+
}
|
|
6892
|
+
} catch {
|
|
6893
|
+
pool = null;
|
|
6894
|
+
}
|
|
6895
|
+
}
|
|
6896
|
+
if (!pool) {
|
|
6897
|
+
await Promise.all(
|
|
6898
|
+
toParse.map(async (item) => {
|
|
6899
|
+
try {
|
|
6900
|
+
const parsed = await parseFileContent(item.file, item.content, item.lang);
|
|
6901
|
+
const settled = statReadParse[item.index];
|
|
6902
|
+
if (settled.status === "fulfilled") {
|
|
6903
|
+
settled.value.parsed = parsed;
|
|
6904
|
+
}
|
|
6905
|
+
} catch (e) {
|
|
6906
|
+
const settled = statReadParse[item.index];
|
|
6907
|
+
if (settled.status === "fulfilled") {
|
|
6908
|
+
settled.value.error = `parse error: ${e instanceof Error ? e.message : String(e)}`;
|
|
6909
|
+
}
|
|
6910
|
+
}
|
|
6911
|
+
})
|
|
6912
|
+
);
|
|
6913
|
+
}
|
|
6914
|
+
}
|
|
5571
6915
|
const batchEntries = [];
|
|
5572
6916
|
const deleteForFiles = [];
|
|
5573
6917
|
for (let fi = 0; fi < statReadParse.length; fi++) {
|
|
@@ -5577,12 +6921,14 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5577
6921
|
const err = settled.reason;
|
|
5578
6922
|
if (err instanceof Error && isAbortError(err)) throw err;
|
|
5579
6923
|
errors.push(`batch error: ${file}: ${err instanceof Error ? err.message : String(err)}`);
|
|
6924
|
+
filesFailed++;
|
|
5580
6925
|
continue;
|
|
5581
6926
|
}
|
|
5582
6927
|
const result = settled.value;
|
|
5583
6928
|
if (result.error) {
|
|
5584
6929
|
if (result.missing) store.deleteFile(file);
|
|
5585
6930
|
errors.push(`${file}: ${result.error}`);
|
|
6931
|
+
filesFailed++;
|
|
5586
6932
|
continue;
|
|
5587
6933
|
}
|
|
5588
6934
|
const { stat: stat2, lang, parsed } = result;
|
|
@@ -5590,6 +6936,18 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5590
6936
|
langStats[lang] = (langStats[lang] ?? 0) + result.skippedMeta.symbolCount;
|
|
5591
6937
|
symbolsIndexed += result.skippedMeta.symbolCount;
|
|
5592
6938
|
filesIndexed++;
|
|
6939
|
+
filesSkipped++;
|
|
6940
|
+
const stored = existingMeta.get(file);
|
|
6941
|
+
if (stored && stored.mtimeMs !== result.skippedMeta.mtimeMs) {
|
|
6942
|
+
store.upsertFile({
|
|
6943
|
+
file,
|
|
6944
|
+
lang,
|
|
6945
|
+
mtimeMs: result.skippedMeta.mtimeMs,
|
|
6946
|
+
symbolCount: result.skippedMeta.symbolCount,
|
|
6947
|
+
lastIndexed: Date.now(),
|
|
6948
|
+
contentHash: result.skippedMeta.contentHash
|
|
6949
|
+
});
|
|
6950
|
+
}
|
|
5593
6951
|
continue;
|
|
5594
6952
|
}
|
|
5595
6953
|
if (!lang || !parsed) {
|
|
@@ -5599,9 +6957,11 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5599
6957
|
lang,
|
|
5600
6958
|
mtimeMs: Math.floor(stat2.mtimeMs),
|
|
5601
6959
|
symbolCount: 0,
|
|
5602
|
-
lastIndexed: Date.now()
|
|
6960
|
+
lastIndexed: Date.now(),
|
|
6961
|
+
contentHash: result.contentHash ?? ""
|
|
5603
6962
|
});
|
|
5604
6963
|
filesIndexed++;
|
|
6964
|
+
filesEmpty++;
|
|
5605
6965
|
}
|
|
5606
6966
|
continue;
|
|
5607
6967
|
}
|
|
@@ -5611,9 +6971,11 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5611
6971
|
lang,
|
|
5612
6972
|
mtimeMs: Math.floor(stat2.mtimeMs),
|
|
5613
6973
|
symbolCount: 0,
|
|
5614
|
-
lastIndexed: Date.now()
|
|
6974
|
+
lastIndexed: Date.now(),
|
|
6975
|
+
contentHash: result.contentHash ?? ""
|
|
5615
6976
|
});
|
|
5616
6977
|
filesIndexed++;
|
|
6978
|
+
filesEmpty++;
|
|
5617
6979
|
continue;
|
|
5618
6980
|
}
|
|
5619
6981
|
batchEntries.push({
|
|
@@ -5622,7 +6984,8 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5622
6984
|
symbols: parsed.symbols,
|
|
5623
6985
|
refs: parsed.refs ?? [],
|
|
5624
6986
|
mtimeMs: Math.floor(stat2.mtimeMs),
|
|
5625
|
-
symbolCount: parsed.symbols.length
|
|
6987
|
+
symbolCount: parsed.symbols.length,
|
|
6988
|
+
contentHash: result.contentHash ?? ""
|
|
5626
6989
|
});
|
|
5627
6990
|
deleteForFiles.push(file);
|
|
5628
6991
|
}
|
|
@@ -5634,6 +6997,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5634
6997
|
symbolsIndexed += count;
|
|
5635
6998
|
langStats[entry.lang] = (langStats[entry.lang] ?? 0) + count;
|
|
5636
6999
|
filesIndexed++;
|
|
7000
|
+
filesParsed++;
|
|
5637
7001
|
}
|
|
5638
7002
|
} catch (err) {
|
|
5639
7003
|
const message = err instanceof Error ? err.message : String(err);
|
|
@@ -5646,6 +7010,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5646
7010
|
symbolsIndexed += symbolsWithIds.length;
|
|
5647
7011
|
langStats[entry.lang] = (langStats[entry.lang] ?? 0) + symbolsWithIds.length;
|
|
5648
7012
|
filesIndexed++;
|
|
7013
|
+
filesParsed++;
|
|
5649
7014
|
if (entry.refs.length > 0 && symbolsWithIds.length > 0) {
|
|
5650
7015
|
const fallbackBatch = assignRefsToSymbols2(entry.refs, symbolsWithIds);
|
|
5651
7016
|
if (fallbackBatch.length > 0) store.insertRefsBatch(fallbackBatch);
|
|
@@ -5659,9 +7024,11 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5659
7024
|
lang: entry.lang,
|
|
5660
7025
|
mtimeMs: entry.mtimeMs,
|
|
5661
7026
|
symbolCount: entry.symbolCount,
|
|
5662
|
-
lastIndexed: Date.now()
|
|
7027
|
+
lastIndexed: Date.now(),
|
|
7028
|
+
contentHash: entry.contentHash
|
|
5663
7029
|
});
|
|
5664
7030
|
} catch (innerErr) {
|
|
7031
|
+
filesFailed++;
|
|
5665
7032
|
errors.push(
|
|
5666
7033
|
`fallback write failed: ${entry.file}: ${innerErr instanceof Error ? innerErr.message : String(innerErr)}`
|
|
5667
7034
|
);
|
|
@@ -5694,6 +7061,12 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5694
7061
|
const durationMs = Date.now() - startMs;
|
|
5695
7062
|
return {
|
|
5696
7063
|
filesIndexed,
|
|
7064
|
+
fileOutcomes: {
|
|
7065
|
+
parsed: filesParsed,
|
|
7066
|
+
skipped: filesSkipped,
|
|
7067
|
+
empty: filesEmpty,
|
|
7068
|
+
failed: filesFailed
|
|
7069
|
+
},
|
|
5697
7070
|
symbolsIndexed,
|
|
5698
7071
|
langStats,
|
|
5699
7072
|
durationMs,
|
|
@@ -5771,6 +7144,9 @@ function symbolGraphService(args) {
|
|
|
5771
7144
|
function incomingCallsService(args) {
|
|
5772
7145
|
const store = indexStorePool.acquire(args.projectRoot, { indexDir: args.indexDir });
|
|
5773
7146
|
try {
|
|
7147
|
+
if (args.transitive) {
|
|
7148
|
+
return store.findTransitiveIncomingCallsByName(args.symbol, args.file, args.limit ?? 100);
|
|
7149
|
+
}
|
|
5774
7150
|
return store.findIncomingCallsByName(args.symbol, args.file, args.limit ?? 100);
|
|
5775
7151
|
} finally {
|
|
5776
7152
|
indexStorePool.release(store);
|
|
@@ -5779,6 +7155,9 @@ function incomingCallsService(args) {
|
|
|
5779
7155
|
function outgoingCallsService(args) {
|
|
5780
7156
|
const store = indexStorePool.acquire(args.projectRoot, { indexDir: args.indexDir });
|
|
5781
7157
|
try {
|
|
7158
|
+
if (args.transitive) {
|
|
7159
|
+
return store.findTransitiveOutgoingCallsByName(args.symbol, args.file, args.limit ?? 100);
|
|
7160
|
+
}
|
|
5782
7161
|
return store.findOutgoingCallsByName(args.symbol, args.file, args.limit ?? 100);
|
|
5783
7162
|
} finally {
|
|
5784
7163
|
indexStorePool.release(store);
|