@wrongstack/tools 0.302.2 → 0.305.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/builtin.d.ts +6 -0
- package/dist/builtin.js +3213 -686
- package/dist/codebase-index/binary-frame.d.ts +43 -0
- package/dist/codebase-index/codebase-incoming-calls-tool.d.ts +1 -0
- package/dist/codebase-index/codebase-index-tool.d.ts +6 -0
- package/dist/codebase-index/codebase-outgoing-calls-tool.d.ts +1 -0
- package/dist/codebase-index/content-hash.d.ts +66 -0
- package/dist/codebase-index/index.js +1597 -153
- package/dist/codebase-index/indexer.d.ts +6 -0
- package/dist/codebase-index/parser-worker-pool.d.ts +63 -0
- package/dist/codebase-index/parser-worker-script.d.ts +42 -0
- package/dist/codebase-index/project-server-protocol.d.ts +2 -0
- package/dist/codebase-index/project-server.js +1483 -104
- package/dist/codebase-index/schema.d.ts +18 -0
- package/dist/codebase-index/tree-sitter/queries.d.ts +48 -0
- package/dist/codebase-index/tree-sitter/util.d.ts +31 -0
- package/dist/codebase-index/tree-sitter/visitor.d.ts +47 -0
- package/dist/codebase-index/tree-sitter-parser.d.ts +58 -0
- package/dist/codebase-index/vector-search.d.ts +62 -0
- package/dist/codebase-index/worker-protocol.d.ts +2 -0
- package/dist/codebase-index/worker.js +1452 -73
- package/dist/codebase-index/writer-bulk-insert.d.ts +5 -0
- package/dist/codebase-index/writer-graph-reader.d.ts +39 -0
- package/dist/codebase-index/writer-schema.d.ts +9 -2
- package/dist/codebase-index/writer.d.ts +36 -0
- package/dist/index.d.ts +4 -3
- package/dist/index.js +3258 -727
- package/dist/kanban-contract-actions.d.ts +7 -0
- package/dist/kanban-task-inputs.d.ts +16 -2
- package/dist/kanban-tool-schema.d.ts +2 -2
- package/dist/kanban-tool-types.d.ts +39 -2
- package/dist/kanban.js +580 -193
- package/dist/pack.js +3212 -686
- package/dist/plan.d.ts +4 -1
- package/dist/plan.js +2701 -18
- package/dist/read.js +1559 -103
- package/dist/session-kanban.d.ts +94 -1
- package/dist/session-kanban.js +308 -44
- package/dist/task.d.ts +5 -4
- package/dist/task.js +2825 -138
- package/dist/todo.d.ts +10 -1
- package/dist/todo.js +2474 -30
- package/dist/tool-tier.js +3212 -686
- package/package.json +8 -4
|
@@ -2131,11 +2131,582 @@ var init_yaml_parser = __esm({
|
|
|
2131
2131
|
}
|
|
2132
2132
|
});
|
|
2133
2133
|
|
|
2134
|
+
// src/codebase-index/tree-sitter/queries.ts
|
|
2135
|
+
function getQueries(lang) {
|
|
2136
|
+
return LANG_QUERIES[lang] ?? DEFAULT_QUERIES;
|
|
2137
|
+
}
|
|
2138
|
+
function readFirstString(node) {
|
|
2139
|
+
if (!node) return null;
|
|
2140
|
+
if (node.type === "string_literal" || node.type === "alias") {
|
|
2141
|
+
return node.text.replace(/^"|"$/g, "");
|
|
2142
|
+
}
|
|
2143
|
+
const child = node.namedChild(0);
|
|
2144
|
+
return child ? readFirstString(child) : null;
|
|
2145
|
+
}
|
|
2146
|
+
var DEFAULT_QUERIES, LANG_QUERIES;
|
|
2147
|
+
var init_queries = __esm({
|
|
2148
|
+
"src/codebase-index/tree-sitter/queries.ts"() {
|
|
2149
|
+
"use strict";
|
|
2150
|
+
DEFAULT_QUERIES = {
|
|
2151
|
+
declKinds: {}
|
|
2152
|
+
};
|
|
2153
|
+
LANG_QUERIES = {
|
|
2154
|
+
// ─── C family ──────────────────────────────────────────────────────────────
|
|
2155
|
+
c: {
|
|
2156
|
+
declKinds: {
|
|
2157
|
+
function_definition: "function",
|
|
2158
|
+
declaration: "function",
|
|
2159
|
+
// K&R-style `int foo(...)` ambiguous w/ local var; the visitor prefers the function branch when the declarator field is present
|
|
2160
|
+
struct_specifier: "struct",
|
|
2161
|
+
union_specifier: "struct",
|
|
2162
|
+
enum_specifier: "enum",
|
|
2163
|
+
type_definition: "type",
|
|
2164
|
+
// `typedef … X;`
|
|
2165
|
+
preproc_def: "const"
|
|
2166
|
+
// `#define NAME …`
|
|
2167
|
+
},
|
|
2168
|
+
nameField: {
|
|
2169
|
+
function_definition: "declarator",
|
|
2170
|
+
declaration: "declarator",
|
|
2171
|
+
struct_specifier: "name",
|
|
2172
|
+
enum_specifier: "name",
|
|
2173
|
+
type_definition: "declarator",
|
|
2174
|
+
preproc_def: "name"
|
|
2175
|
+
},
|
|
2176
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2177
|
+
"translation_unit",
|
|
2178
|
+
"function_definition",
|
|
2179
|
+
"struct_specifier",
|
|
2180
|
+
"union_specifier",
|
|
2181
|
+
"enum_specifier"
|
|
2182
|
+
])
|
|
2183
|
+
},
|
|
2184
|
+
cpp: {
|
|
2185
|
+
declKinds: {
|
|
2186
|
+
function_definition: "function",
|
|
2187
|
+
template_declaration: "function",
|
|
2188
|
+
// `template<typename T> …`
|
|
2189
|
+
class_specifier: "class",
|
|
2190
|
+
struct_specifier: "struct",
|
|
2191
|
+
union_specifier: "struct",
|
|
2192
|
+
enum_specifier: "enum",
|
|
2193
|
+
namespace_definition: "namespace",
|
|
2194
|
+
type_definition: "type"
|
|
2195
|
+
},
|
|
2196
|
+
nameField: {
|
|
2197
|
+
function_definition: "declarator",
|
|
2198
|
+
template_declaration: "name",
|
|
2199
|
+
class_specifier: "name",
|
|
2200
|
+
struct_specifier: "name",
|
|
2201
|
+
enum_specifier: "name",
|
|
2202
|
+
namespace_definition: "name",
|
|
2203
|
+
type_definition: "declarator"
|
|
2204
|
+
},
|
|
2205
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2206
|
+
"translation_unit",
|
|
2207
|
+
"function_definition",
|
|
2208
|
+
"class_specifier",
|
|
2209
|
+
"struct_specifier",
|
|
2210
|
+
"union_specifier",
|
|
2211
|
+
"enum_specifier",
|
|
2212
|
+
"namespace_definition"
|
|
2213
|
+
])
|
|
2214
|
+
},
|
|
2215
|
+
java: {
|
|
2216
|
+
declKinds: {
|
|
2217
|
+
class_declaration: "class",
|
|
2218
|
+
interface_declaration: "interface",
|
|
2219
|
+
enum_declaration: "enum",
|
|
2220
|
+
record_declaration: "class",
|
|
2221
|
+
annotation_type_declaration: "interface",
|
|
2222
|
+
method_declaration: "method",
|
|
2223
|
+
constructor_declaration: "method",
|
|
2224
|
+
field_declaration: "property"
|
|
2225
|
+
},
|
|
2226
|
+
nameField: {
|
|
2227
|
+
class_declaration: "name",
|
|
2228
|
+
interface_declaration: "name",
|
|
2229
|
+
enum_declaration: "name",
|
|
2230
|
+
record_declaration: "name",
|
|
2231
|
+
annotation_type_declaration: "name",
|
|
2232
|
+
method_declaration: "name",
|
|
2233
|
+
constructor_declaration: "name"
|
|
2234
|
+
},
|
|
2235
|
+
// `field_declaration` has no single `name` field — it carries a list of
|
|
2236
|
+
// variable declarators. We emit one Symbol per node using the first
|
|
2237
|
+
// identifier-shaped named child (see `extractName` fallback in
|
|
2238
|
+
// `visitor.ts`). `int a, b, c;` therefore indexes only `a` — splitting
|
|
2239
|
+
// multi-declarator fields into separate Symbols is a separate refactor
|
|
2240
|
+
// that needs the visitor to know it has multiple names per node, and no
|
|
2241
|
+
// current test relies on it.
|
|
2242
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2243
|
+
"program",
|
|
2244
|
+
"class_declaration",
|
|
2245
|
+
"interface_declaration",
|
|
2246
|
+
"enum_declaration",
|
|
2247
|
+
"record_declaration"
|
|
2248
|
+
])
|
|
2249
|
+
},
|
|
2250
|
+
csharp: {
|
|
2251
|
+
// C# 10+ `namespace Foo.Bar;` produces this node type. The legacy block
|
|
2252
|
+
// form `namespace Foo.Bar { ... }` produces `namespace_declaration`. Both
|
|
2253
|
+
// carry a `qualified_name` child whose text already includes the dots.
|
|
2254
|
+
// `using_directive` is intentionally not a declaration. Imports are
|
|
2255
|
+
// extracted separately; indexing a using directive as a namespace makes
|
|
2256
|
+
// the resolver bind it to its own source file before the real declaration.
|
|
2257
|
+
declKinds: {
|
|
2258
|
+
file_scoped_namespace_declaration: "namespace",
|
|
2259
|
+
class_declaration: "class",
|
|
2260
|
+
interface_declaration: "interface",
|
|
2261
|
+
struct_declaration: "struct",
|
|
2262
|
+
enum_declaration: "enum",
|
|
2263
|
+
record_declaration: "class",
|
|
2264
|
+
method_declaration: "method",
|
|
2265
|
+
constructor_declaration: "method",
|
|
2266
|
+
property_declaration: "property",
|
|
2267
|
+
field_declaration: "property",
|
|
2268
|
+
namespace_declaration: "namespace"
|
|
2269
|
+
},
|
|
2270
|
+
// Custom name extractor: take the full dotted name verbatim.
|
|
2271
|
+
nameExtractor: (node) => {
|
|
2272
|
+
const inner = node.namedChild(0);
|
|
2273
|
+
if (inner && (inner.type === "qualified_name" || inner.type === "name")) {
|
|
2274
|
+
return inner.text;
|
|
2275
|
+
}
|
|
2276
|
+
return null;
|
|
2277
|
+
},
|
|
2278
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2279
|
+
"compilation_unit",
|
|
2280
|
+
"namespace_declaration",
|
|
2281
|
+
"class_declaration",
|
|
2282
|
+
"interface_declaration",
|
|
2283
|
+
"struct_declaration",
|
|
2284
|
+
"enum_declaration",
|
|
2285
|
+
"record_declaration"
|
|
2286
|
+
])
|
|
2287
|
+
},
|
|
2288
|
+
php: {
|
|
2289
|
+
declKinds: {
|
|
2290
|
+
function_definition: "function",
|
|
2291
|
+
method_declaration: "method",
|
|
2292
|
+
class_declaration: "class",
|
|
2293
|
+
interface_declaration: "interface",
|
|
2294
|
+
trait_declaration: "class",
|
|
2295
|
+
enum_declaration: "enum",
|
|
2296
|
+
namespace_definition: "namespace"
|
|
2297
|
+
},
|
|
2298
|
+
nameField: {
|
|
2299
|
+
function_definition: "name",
|
|
2300
|
+
method_declaration: "name",
|
|
2301
|
+
class_declaration: "name",
|
|
2302
|
+
interface_declaration: "name",
|
|
2303
|
+
trait_declaration: "name",
|
|
2304
|
+
enum_declaration: "name",
|
|
2305
|
+
namespace_declaration: "name"
|
|
2306
|
+
},
|
|
2307
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2308
|
+
"program",
|
|
2309
|
+
"namespace_definition",
|
|
2310
|
+
"class_declaration",
|
|
2311
|
+
"interface_declaration",
|
|
2312
|
+
"trait_declaration",
|
|
2313
|
+
"enum_declaration"
|
|
2314
|
+
])
|
|
2315
|
+
},
|
|
2316
|
+
// ─── Scripting / mobile ────────────────────────────────────────────────────
|
|
2317
|
+
ruby: {
|
|
2318
|
+
declKinds: {
|
|
2319
|
+
method: "function",
|
|
2320
|
+
singleton_method: "method",
|
|
2321
|
+
class: "class",
|
|
2322
|
+
module: "namespace",
|
|
2323
|
+
constant: "const"
|
|
2324
|
+
},
|
|
2325
|
+
nameField: {
|
|
2326
|
+
method: "name",
|
|
2327
|
+
singleton_method: "name",
|
|
2328
|
+
class: "name",
|
|
2329
|
+
module: "name",
|
|
2330
|
+
constant: "name"
|
|
2331
|
+
},
|
|
2332
|
+
scopeNodes: /* @__PURE__ */ new Set(["program", "class", "module", "singleton_method", "method"])
|
|
2333
|
+
},
|
|
2334
|
+
swift: {
|
|
2335
|
+
declKinds: {
|
|
2336
|
+
function_declaration: "function",
|
|
2337
|
+
class_declaration: "class",
|
|
2338
|
+
struct_declaration: "struct",
|
|
2339
|
+
enum_declaration: "enum",
|
|
2340
|
+
protocol_declaration: "interface",
|
|
2341
|
+
actor_declaration: "class",
|
|
2342
|
+
extension_declaration: "class",
|
|
2343
|
+
initializer: "method",
|
|
2344
|
+
property_declaration: "property"
|
|
2345
|
+
},
|
|
2346
|
+
nameField: {
|
|
2347
|
+
function_declaration: "name",
|
|
2348
|
+
class_declaration: "name",
|
|
2349
|
+
struct_declaration: "name",
|
|
2350
|
+
enum_declaration: "name",
|
|
2351
|
+
protocol_declaration: "name",
|
|
2352
|
+
actor_declaration: "name",
|
|
2353
|
+
extension_declaration: "name",
|
|
2354
|
+
initializer: "name",
|
|
2355
|
+
property_declaration: "name"
|
|
2356
|
+
},
|
|
2357
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2358
|
+
"source_file",
|
|
2359
|
+
"class_declaration",
|
|
2360
|
+
"struct_declaration",
|
|
2361
|
+
"enum_declaration",
|
|
2362
|
+
"protocol_declaration",
|
|
2363
|
+
"actor_declaration",
|
|
2364
|
+
"extension_declaration"
|
|
2365
|
+
])
|
|
2366
|
+
},
|
|
2367
|
+
kotlin: {
|
|
2368
|
+
declKinds: {
|
|
2369
|
+
class_declaration: "class",
|
|
2370
|
+
object_declaration: "class",
|
|
2371
|
+
interface_declaration: "interface",
|
|
2372
|
+
function_declaration: "function",
|
|
2373
|
+
property_declaration: "property",
|
|
2374
|
+
type_alias: "type"
|
|
2375
|
+
},
|
|
2376
|
+
nameField: {
|
|
2377
|
+
class_declaration: "name",
|
|
2378
|
+
object_declaration: "name",
|
|
2379
|
+
interface_declaration: "name",
|
|
2380
|
+
function_declaration: "name",
|
|
2381
|
+
property_declaration: "name",
|
|
2382
|
+
type_alias: "name"
|
|
2383
|
+
},
|
|
2384
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2385
|
+
"source_file",
|
|
2386
|
+
"class_declaration",
|
|
2387
|
+
"object_declaration",
|
|
2388
|
+
"interface_declaration",
|
|
2389
|
+
"function_declaration"
|
|
2390
|
+
])
|
|
2391
|
+
},
|
|
2392
|
+
elixir: {
|
|
2393
|
+
declKinds: {
|
|
2394
|
+
// `def foo`, `defp foo`, `defmacro foo`, `macrop foo` all surface as
|
|
2395
|
+
// `call` nodes in the tree-sitter grammar — there is no
|
|
2396
|
+
// `function_definition`. The `nameExtractor` walks the call's
|
|
2397
|
+
// children to pick the right sibling identifier.
|
|
2398
|
+
call: "function",
|
|
2399
|
+
module: "namespace"
|
|
2400
|
+
},
|
|
2401
|
+
nameExtractor: (node) => {
|
|
2402
|
+
if (node.type === "module") {
|
|
2403
|
+
const aliasNode = node.childForFieldName("alias");
|
|
2404
|
+
return readFirstString(aliasNode) ?? null;
|
|
2405
|
+
}
|
|
2406
|
+
if (node.type !== "call") return null;
|
|
2407
|
+
const first = node.namedChild(0);
|
|
2408
|
+
if (!first) return null;
|
|
2409
|
+
const target = first.text;
|
|
2410
|
+
if (target !== "def" && target !== "defp" && target !== "defmacro" && target !== "defp_macro" && target !== "macrop" && target !== "defprotocol" && target !== "defguard" && target !== "defguardp") {
|
|
2411
|
+
return null;
|
|
2412
|
+
}
|
|
2413
|
+
const nameNode = node.namedChild(1);
|
|
2414
|
+
return nameNode?.text ?? null;
|
|
2415
|
+
},
|
|
2416
|
+
scopeNodes: /* @__PURE__ */ new Set(["source", "module"])
|
|
2417
|
+
},
|
|
2418
|
+
shell: {
|
|
2419
|
+
declKinds: {
|
|
2420
|
+
function_definition: "function"
|
|
2421
|
+
},
|
|
2422
|
+
nameField: { function_definition: "name" },
|
|
2423
|
+
scopeNodes: /* @__PURE__ */ new Set(["program", "function_definition"])
|
|
2424
|
+
}
|
|
2425
|
+
};
|
|
2426
|
+
}
|
|
2427
|
+
});
|
|
2428
|
+
|
|
2429
|
+
// src/codebase-index/tree-sitter/util.ts
|
|
2430
|
+
function lineColAt2(offsets, index) {
|
|
2431
|
+
let low = 0;
|
|
2432
|
+
let high = offsets.length;
|
|
2433
|
+
while (low < high) {
|
|
2434
|
+
const mid = low + high >>> 1;
|
|
2435
|
+
if ((offsets[mid] ?? 0) < index) low = mid + 1;
|
|
2436
|
+
else high = mid;
|
|
2437
|
+
}
|
|
2438
|
+
const lastNl = low > 0 ? offsets[low - 1] ?? -1 : -1;
|
|
2439
|
+
return { line: low + 1, col: index - lastNl };
|
|
2440
|
+
}
|
|
2441
|
+
function newlineOffsets3(content) {
|
|
2442
|
+
const offsets = [];
|
|
2443
|
+
for (let i = 0; i < content.length; i++) {
|
|
2444
|
+
if (content.charCodeAt(i) === 10) offsets.push(i);
|
|
2445
|
+
}
|
|
2446
|
+
return offsets;
|
|
2447
|
+
}
|
|
2448
|
+
var TREE_SITTER_MAX_FILE_CHARS, TREE_SITTER_MAX_SYMBOLS;
|
|
2449
|
+
var init_util = __esm({
|
|
2450
|
+
"src/codebase-index/tree-sitter/util.ts"() {
|
|
2451
|
+
"use strict";
|
|
2452
|
+
TREE_SITTER_MAX_FILE_CHARS = 512 * 1024;
|
|
2453
|
+
TREE_SITTER_MAX_SYMBOLS = 500;
|
|
2454
|
+
}
|
|
2455
|
+
});
|
|
2456
|
+
|
|
2457
|
+
// src/codebase-index/tree-sitter/visitor.ts
|
|
2458
|
+
function visitTree(tree, content, file, lang, queries) {
|
|
2459
|
+
const boundedContent = content.length > TREE_SITTER_MAX_FILE_CHARS ? content.slice(0, TREE_SITTER_MAX_FILE_CHARS) : content;
|
|
2460
|
+
const nlOffsets = newlineOffsets3(boundedContent);
|
|
2461
|
+
const symbols = [];
|
|
2462
|
+
const scopeStack = [];
|
|
2463
|
+
function visit(node, depth) {
|
|
2464
|
+
if (symbols.length >= TREE_SITTER_MAX_SYMBOLS) return;
|
|
2465
|
+
if (node.isMissing || node.isError) {
|
|
2466
|
+
} else {
|
|
2467
|
+
const kind = queries.declKinds[node.type];
|
|
2468
|
+
if (kind) {
|
|
2469
|
+
const emitted = emitSymbol(
|
|
2470
|
+
node,
|
|
2471
|
+
kind,
|
|
2472
|
+
file,
|
|
2473
|
+
lang,
|
|
2474
|
+
scopeStack,
|
|
2475
|
+
boundedContent,
|
|
2476
|
+
nlOffsets,
|
|
2477
|
+
queries
|
|
2478
|
+
);
|
|
2479
|
+
if (emitted) symbols.push(emitted);
|
|
2480
|
+
}
|
|
2481
|
+
}
|
|
2482
|
+
const pushesScope = queries.scopeNodes?.has(node.type) ?? false;
|
|
2483
|
+
const pushIdx = pushesScope ? pushScope(scopeStack, node, queries) : -1;
|
|
2484
|
+
if (queries.skipNamedChildren) {
|
|
2485
|
+
if (pushIdx !== -1) scopeStack.pop();
|
|
2486
|
+
return;
|
|
2487
|
+
}
|
|
2488
|
+
for (const child of node.namedChildren) {
|
|
2489
|
+
visit(child, depth + 1);
|
|
2490
|
+
if (symbols.length >= TREE_SITTER_MAX_SYMBOLS) {
|
|
2491
|
+
if (pushIdx !== -1) scopeStack.pop();
|
|
2492
|
+
return;
|
|
2493
|
+
}
|
|
2494
|
+
}
|
|
2495
|
+
if (pushIdx !== -1) scopeStack.pop();
|
|
2496
|
+
}
|
|
2497
|
+
visit(tree.rootNode, 0);
|
|
2498
|
+
return { symbols };
|
|
2499
|
+
}
|
|
2500
|
+
function pushScope(scopeStack, node, queries) {
|
|
2501
|
+
const name = extractName(node, queries);
|
|
2502
|
+
if (!name) return -1;
|
|
2503
|
+
scopeStack.push(name);
|
|
2504
|
+
return scopeStack.length - 1;
|
|
2505
|
+
}
|
|
2506
|
+
function extractName(node, queries) {
|
|
2507
|
+
if (queries.nameExtractor) {
|
|
2508
|
+
const extracted = queries.nameExtractor(node);
|
|
2509
|
+
if (extracted) return extracted;
|
|
2510
|
+
}
|
|
2511
|
+
const fieldName = queries.nameField?.[node.type] ?? "name";
|
|
2512
|
+
const field = node.childForFieldName(fieldName);
|
|
2513
|
+
if (field) {
|
|
2514
|
+
if (IDENTIFIER_NODE_TYPES.has(field.type)) {
|
|
2515
|
+
return field.text;
|
|
2516
|
+
}
|
|
2517
|
+
const inner = field.childForFieldName("name") ?? field.namedChild(0);
|
|
2518
|
+
if (inner && IDENTIFIER_NODE_TYPES.has(inner.type)) {
|
|
2519
|
+
return inner.text;
|
|
2520
|
+
}
|
|
2521
|
+
}
|
|
2522
|
+
for (let i = 0; i < node.namedChildCount; i++) {
|
|
2523
|
+
const child = node.namedChild(i);
|
|
2524
|
+
if (child && IDENTIFIER_NODE_TYPES.has(child.type)) {
|
|
2525
|
+
return child.text;
|
|
2526
|
+
}
|
|
2527
|
+
}
|
|
2528
|
+
return null;
|
|
2529
|
+
}
|
|
2530
|
+
function emitSymbol(node, kind, file, lang, scopeStack, content, nlOffsets, queries) {
|
|
2531
|
+
const name = extractName(node, queries);
|
|
2532
|
+
if (!name) return null;
|
|
2533
|
+
const pos = node.startIndex;
|
|
2534
|
+
const { line, col } = lineColAt2(nlOffsets, pos);
|
|
2535
|
+
const end = Math.min(node.endIndex, content.length);
|
|
2536
|
+
const signature = content.slice(pos, end).replace(/\s+/g, " ").trim().slice(0, 500);
|
|
2537
|
+
const scope = scopeStack.join(".");
|
|
2538
|
+
const text = [name, signature].filter(Boolean).join(" | ").trim().slice(0, 1e3);
|
|
2539
|
+
return {
|
|
2540
|
+
id: 0,
|
|
2541
|
+
// caller assigns during bulk insertion
|
|
2542
|
+
lang,
|
|
2543
|
+
kind,
|
|
2544
|
+
name: name.slice(0, 200),
|
|
2545
|
+
file,
|
|
2546
|
+
line,
|
|
2547
|
+
col,
|
|
2548
|
+
signature,
|
|
2549
|
+
docComment: "",
|
|
2550
|
+
// doc-comment extraction lands with ref emission on Day 4
|
|
2551
|
+
scope,
|
|
2552
|
+
text
|
|
2553
|
+
};
|
|
2554
|
+
}
|
|
2555
|
+
var IDENTIFIER_NODE_TYPES;
|
|
2556
|
+
var init_visitor = __esm({
|
|
2557
|
+
"src/codebase-index/tree-sitter/visitor.ts"() {
|
|
2558
|
+
"use strict";
|
|
2559
|
+
init_util();
|
|
2560
|
+
IDENTIFIER_NODE_TYPES = /* @__PURE__ */ new Set([
|
|
2561
|
+
"identifier",
|
|
2562
|
+
"simple_identifier",
|
|
2563
|
+
"type_identifier",
|
|
2564
|
+
"field_identifier",
|
|
2565
|
+
"property_identifier",
|
|
2566
|
+
"name",
|
|
2567
|
+
"word",
|
|
2568
|
+
"variable_name",
|
|
2569
|
+
"constant",
|
|
2570
|
+
"sym"
|
|
2571
|
+
]);
|
|
2572
|
+
}
|
|
2573
|
+
});
|
|
2574
|
+
|
|
2575
|
+
// src/codebase-index/tree-sitter-parser.ts
|
|
2576
|
+
var tree_sitter_parser_exports = {};
|
|
2577
|
+
__export(tree_sitter_parser_exports, {
|
|
2578
|
+
__smokeRootType: () => __smokeRootType,
|
|
2579
|
+
getGrammarWasmPath: () => getGrammarWasmPath,
|
|
2580
|
+
isTreeSitterSupported: () => isTreeSitterSupported,
|
|
2581
|
+
loadTreeSitterLanguage: () => loadTreeSitterLanguage,
|
|
2582
|
+
parseSymbols: () => parseSymbols8
|
|
2583
|
+
});
|
|
2584
|
+
import * as path9 from "node:path";
|
|
2585
|
+
import { fileURLToPath } from "node:url";
|
|
2586
|
+
function optInEnabled(env) {
|
|
2587
|
+
return process.env[env] === "1" || process.env[env] === "true";
|
|
2588
|
+
}
|
|
2589
|
+
function getRuntime() {
|
|
2590
|
+
if (!runtimePromise) {
|
|
2591
|
+
runtimePromise = (async () => {
|
|
2592
|
+
const mod = await import("web-tree-sitter");
|
|
2593
|
+
const init = () => mod.Parser.init({ locateFile: () => RUNTIME_WASM });
|
|
2594
|
+
return { Parser: mod.Parser, Language: mod.Language, init };
|
|
2595
|
+
})();
|
|
2596
|
+
}
|
|
2597
|
+
return runtimePromise;
|
|
2598
|
+
}
|
|
2599
|
+
async function loadLanguage(lang) {
|
|
2600
|
+
const existing = languageCache.get(lang);
|
|
2601
|
+
if (existing) return existing;
|
|
2602
|
+
const promise = (async () => {
|
|
2603
|
+
const grammarName = resolveGrammarName(lang);
|
|
2604
|
+
if (!grammarName) {
|
|
2605
|
+
throw new Error(`tree-sitter: no grammar registered for lang "${lang}"`);
|
|
2606
|
+
}
|
|
2607
|
+
const wasmPath = path9.join(WASM_DIR, grammarName, `tree-sitter-${grammarName}.wasm`);
|
|
2608
|
+
const { Language, init } = await getRuntime();
|
|
2609
|
+
await init();
|
|
2610
|
+
const languageObj = await Language.load(wasmPath);
|
|
2611
|
+
return { lang, Language: languageObj };
|
|
2612
|
+
})();
|
|
2613
|
+
languageCache.set(lang, promise);
|
|
2614
|
+
return promise;
|
|
2615
|
+
}
|
|
2616
|
+
function resolveGrammarName(lang) {
|
|
2617
|
+
if (lang === "go" && optInEnabled(GO_OPT_IN)) return "go";
|
|
2618
|
+
if (lang === "py" && optInEnabled(PY_OPT_IN)) return "python";
|
|
2619
|
+
if (lang === "rs" && optInEnabled(RS_OPT_IN)) return "rust";
|
|
2620
|
+
return LANG_TO_GRAMMAR[lang];
|
|
2621
|
+
}
|
|
2622
|
+
function isTreeSitterSupported(lang) {
|
|
2623
|
+
return resolveGrammarName(lang) !== void 0;
|
|
2624
|
+
}
|
|
2625
|
+
function getGrammarWasmPath(lang) {
|
|
2626
|
+
const name = resolveGrammarName(lang);
|
|
2627
|
+
if (!name) return void 0;
|
|
2628
|
+
return path9.join(WASM_DIR, name, `tree-sitter-${name}.wasm`);
|
|
2629
|
+
}
|
|
2630
|
+
async function parseSymbols8(opts) {
|
|
2631
|
+
const { file, content, lang } = opts;
|
|
2632
|
+
if (!isTreeSitterSupported(lang)) {
|
|
2633
|
+
return { file, lang, symbols: [], mtimeMs: Date.now() };
|
|
2634
|
+
}
|
|
2635
|
+
try {
|
|
2636
|
+
const { Parser } = await getRuntime();
|
|
2637
|
+
const cached = await loadLanguage(lang);
|
|
2638
|
+
const parser = new Parser();
|
|
2639
|
+
parser.setLanguage(cached.Language);
|
|
2640
|
+
const tree = parser.parse(content);
|
|
2641
|
+
if (!tree) {
|
|
2642
|
+
return { file, lang, symbols: [], mtimeMs: Date.now() };
|
|
2643
|
+
}
|
|
2644
|
+
const { symbols } = visitTree(tree, content, file, lang, getQueries(lang));
|
|
2645
|
+
parser.delete();
|
|
2646
|
+
tree.delete();
|
|
2647
|
+
return { file, lang, symbols, refs: [], mtimeMs: Date.now() };
|
|
2648
|
+
} catch {
|
|
2649
|
+
return { file, lang, symbols: [], mtimeMs: Date.now() };
|
|
2650
|
+
}
|
|
2651
|
+
}
|
|
2652
|
+
async function loadTreeSitterLanguage(lang) {
|
|
2653
|
+
const cached = await loadLanguage(lang);
|
|
2654
|
+
return cached.Language;
|
|
2655
|
+
}
|
|
2656
|
+
async function __smokeRootType(opts) {
|
|
2657
|
+
if (!isTreeSitterSupported(opts.lang)) {
|
|
2658
|
+
throw new Error(`tree-sitter: no grammar registered for lang "${opts.lang}"`);
|
|
2659
|
+
}
|
|
2660
|
+
const { Parser } = await getRuntime();
|
|
2661
|
+
const cached = await loadLanguage(opts.lang);
|
|
2662
|
+
const parser = new Parser();
|
|
2663
|
+
parser.setLanguage(cached.Language);
|
|
2664
|
+
let tree = null;
|
|
2665
|
+
try {
|
|
2666
|
+
tree = parser.parse(opts.content);
|
|
2667
|
+
if (!tree) throw new Error("tree-sitter: parser.parse returned null");
|
|
2668
|
+
return tree.rootNode.type;
|
|
2669
|
+
} finally {
|
|
2670
|
+
tree?.delete();
|
|
2671
|
+
parser.delete();
|
|
2672
|
+
}
|
|
2673
|
+
}
|
|
2674
|
+
var WASM_DIR, RUNTIME_WASM, LANG_TO_GRAMMAR, GO_OPT_IN, PY_OPT_IN, RS_OPT_IN, runtimePromise, languageCache;
|
|
2675
|
+
var init_tree_sitter_parser = __esm({
|
|
2676
|
+
"src/codebase-index/tree-sitter-parser.ts"() {
|
|
2677
|
+
"use strict";
|
|
2678
|
+
init_queries();
|
|
2679
|
+
init_visitor();
|
|
2680
|
+
WASM_DIR = fileURLToPath(new URL("./wasm/", import.meta.url));
|
|
2681
|
+
RUNTIME_WASM = path9.join(WASM_DIR, "tree-sitter-runtime.wasm");
|
|
2682
|
+
LANG_TO_GRAMMAR = {
|
|
2683
|
+
c: "c",
|
|
2684
|
+
cpp: "cpp",
|
|
2685
|
+
java: "java",
|
|
2686
|
+
csharp: "c_sharp",
|
|
2687
|
+
// tree-sitter directory uses underscore
|
|
2688
|
+
php: "php",
|
|
2689
|
+
ruby: "ruby",
|
|
2690
|
+
swift: "swift",
|
|
2691
|
+
kotlin: "kotlin",
|
|
2692
|
+
shell: "bash",
|
|
2693
|
+
// we treat `.sh` / `.bash` / `.zsh` via the bash grammar
|
|
2694
|
+
// Go/Python/Rust are opt-in only (see goOptIn / pyOptIn / rsOptIn below)
|
|
2695
|
+
elixir: "elixir"
|
|
2696
|
+
};
|
|
2697
|
+
GO_OPT_IN = "WRONGSTACK_USE_TS_GO";
|
|
2698
|
+
PY_OPT_IN = "WRONGSTACK_USE_TS_PY";
|
|
2699
|
+
RS_OPT_IN = "WRONGSTACK_USE_TS_RS";
|
|
2700
|
+
runtimePromise = null;
|
|
2701
|
+
languageCache = /* @__PURE__ */ new Map();
|
|
2702
|
+
}
|
|
2703
|
+
});
|
|
2704
|
+
|
|
2134
2705
|
// src/codebase-index/project-server.ts
|
|
2135
2706
|
import { randomBytes } from "node:crypto";
|
|
2136
|
-
import * as
|
|
2707
|
+
import * as fs11 from "node:fs";
|
|
2137
2708
|
import * as net from "node:net";
|
|
2138
|
-
import * as
|
|
2709
|
+
import * as path14 from "node:path";
|
|
2139
2710
|
import {
|
|
2140
2711
|
DEFAULT_WALK_IGNORE_SET,
|
|
2141
2712
|
startSharedHeapWatchdog,
|
|
@@ -2146,15 +2717,95 @@ import {
|
|
|
2146
2717
|
// src/codebase-index/indexer.ts
|
|
2147
2718
|
import { expectDefined as expectDefined5 } from "@wrongstack/core/utils";
|
|
2148
2719
|
import { execFile } from "node:child_process";
|
|
2149
|
-
import * as
|
|
2720
|
+
import * as fs9 from "node:fs/promises";
|
|
2150
2721
|
import { availableParallelism } from "node:os";
|
|
2151
|
-
import * as
|
|
2722
|
+
import * as path12 from "node:path";
|
|
2152
2723
|
import {
|
|
2153
2724
|
DEFAULT_WALK_IGNORE_DIRS,
|
|
2154
2725
|
indexParallelBatchSize,
|
|
2155
2726
|
isFrugalPerf
|
|
2156
2727
|
} from "@wrongstack/core/utils";
|
|
2157
2728
|
|
|
2729
|
+
// src/codebase-index/content-hash.ts
|
|
2730
|
+
var PRIME64_1 = 0x9e3779b185ebca87n;
|
|
2731
|
+
var PRIME64_2 = 0xc2b2ae3d27d4eb4fn;
|
|
2732
|
+
var PRIME64_3 = 0x165667b19e3779f9n;
|
|
2733
|
+
var PRIME64_4 = 0x85ebca77c2b2ae63n;
|
|
2734
|
+
var PRIME64_5 = 0x27d4eb2f165667c5n;
|
|
2735
|
+
var MASK64 = 0xffffffffffffffffn;
|
|
2736
|
+
function mul64(a, b) {
|
|
2737
|
+
return (a & MASK64) * (b & MASK64) & MASK64;
|
|
2738
|
+
}
|
|
2739
|
+
function rotl64(x, n) {
|
|
2740
|
+
const v = x & MASK64;
|
|
2741
|
+
return (v << BigInt(n) | v >> BigInt(64 - n)) & MASK64;
|
|
2742
|
+
}
|
|
2743
|
+
function readU64LE(buf, off) {
|
|
2744
|
+
let v = 0n;
|
|
2745
|
+
for (let i = 7; i >= 0; i--) {
|
|
2746
|
+
v = v << 8n | BigInt(buf[off + i] ?? 0);
|
|
2747
|
+
}
|
|
2748
|
+
return v & MASK64;
|
|
2749
|
+
}
|
|
2750
|
+
function readU32LE(buf, off) {
|
|
2751
|
+
return (BigInt(buf[off] ?? 0) | BigInt(buf[off + 1] ?? 0) << 8n | BigInt(buf[off + 2] ?? 0) << 16n | BigInt(buf[off + 3] ?? 0) << 24n) & MASK64;
|
|
2752
|
+
}
|
|
2753
|
+
function xxh64Round(acc, lane) {
|
|
2754
|
+
return mul64(rotl64(acc + mul64(lane, PRIME64_2) & MASK64, 31), PRIME64_1);
|
|
2755
|
+
}
|
|
2756
|
+
function xxh64MergeRound(acc, val) {
|
|
2757
|
+
return mul64(acc ^ xxh64Round(0n, val), PRIME64_1) + PRIME64_4 & MASK64;
|
|
2758
|
+
}
|
|
2759
|
+
function xxhash64Hex(buf, explicitLen) {
|
|
2760
|
+
const length = explicitLen ?? buf.length;
|
|
2761
|
+
let h;
|
|
2762
|
+
let off = 0;
|
|
2763
|
+
if (length >= 32) {
|
|
2764
|
+
let v1 = PRIME64_1 + PRIME64_2 & MASK64;
|
|
2765
|
+
let v2 = PRIME64_2;
|
|
2766
|
+
let v3 = 0n;
|
|
2767
|
+
let v4 = 0n - PRIME64_1 & MASK64;
|
|
2768
|
+
const end32 = length - 32;
|
|
2769
|
+
while (off <= end32) {
|
|
2770
|
+
v1 = xxh64Round(v1, readU64LE(buf, off));
|
|
2771
|
+
v2 = xxh64Round(v2, readU64LE(buf, off + 8));
|
|
2772
|
+
v3 = xxh64Round(v3, readU64LE(buf, off + 16));
|
|
2773
|
+
v4 = xxh64Round(v4, readU64LE(buf, off + 24));
|
|
2774
|
+
off += 32;
|
|
2775
|
+
}
|
|
2776
|
+
h = rotl64(v1, 1) + rotl64(v2, 7) + rotl64(v3, 12) + rotl64(v4, 18) & MASK64;
|
|
2777
|
+
h = xxh64MergeRound(h, v1);
|
|
2778
|
+
h = xxh64MergeRound(h, v2);
|
|
2779
|
+
h = xxh64MergeRound(h, v3);
|
|
2780
|
+
h = xxh64MergeRound(h, v4);
|
|
2781
|
+
} else {
|
|
2782
|
+
h = PRIME64_5;
|
|
2783
|
+
}
|
|
2784
|
+
h = h + BigInt(length) & MASK64;
|
|
2785
|
+
while (off + 8 <= length) {
|
|
2786
|
+
const k1 = xxh64Round(0n, readU64LE(buf, off));
|
|
2787
|
+
h = mul64(rotl64(h ^ k1, 27), PRIME64_1) + PRIME64_4 & MASK64;
|
|
2788
|
+
off += 8;
|
|
2789
|
+
}
|
|
2790
|
+
if (off + 4 <= length) {
|
|
2791
|
+
h = mul64(rotl64(h ^ mul64(readU32LE(buf, off), PRIME64_1), 23), PRIME64_2) + PRIME64_3 & MASK64;
|
|
2792
|
+
off += 4;
|
|
2793
|
+
}
|
|
2794
|
+
while (off < length) {
|
|
2795
|
+
h = mul64(rotl64(h ^ mul64(BigInt(buf[off] ?? 0), PRIME64_5), 11), PRIME64_1) & MASK64;
|
|
2796
|
+
off += 1;
|
|
2797
|
+
}
|
|
2798
|
+
h = (h ^ h >> 33n) & MASK64;
|
|
2799
|
+
h = mul64(h, PRIME64_2);
|
|
2800
|
+
h = (h ^ h >> 29n) & MASK64;
|
|
2801
|
+
h = mul64(h, PRIME64_3);
|
|
2802
|
+
h = (h ^ h >> 32n) & MASK64;
|
|
2803
|
+
return h.toString(16).padStart(16, "0");
|
|
2804
|
+
}
|
|
2805
|
+
function xxhash64String(content) {
|
|
2806
|
+
return xxhash64Hex(new TextEncoder().encode(content));
|
|
2807
|
+
}
|
|
2808
|
+
|
|
2158
2809
|
// src/codebase-index/gitignore.ts
|
|
2159
2810
|
import * as fs from "node:fs/promises";
|
|
2160
2811
|
import * as path from "node:path";
|
|
@@ -2950,32 +3601,56 @@ async function dispatch(file, content, lang) {
|
|
|
2950
3601
|
case "tsx":
|
|
2951
3602
|
case "js":
|
|
2952
3603
|
case "jsx": {
|
|
2953
|
-
const { parseSymbols:
|
|
2954
|
-
return
|
|
3604
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_ts_parser(), ts_parser_exports));
|
|
3605
|
+
return parseSymbols9({ file, content, lang });
|
|
2955
3606
|
}
|
|
2956
3607
|
case "go": {
|
|
2957
|
-
const { parseSymbols:
|
|
2958
|
-
return
|
|
3608
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_go_parser(), go_parser_exports));
|
|
3609
|
+
return parseSymbols9({ file, content, lang: "go" });
|
|
2959
3610
|
}
|
|
2960
3611
|
case "py": {
|
|
2961
|
-
const { parseSymbols:
|
|
2962
|
-
return
|
|
3612
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_py_parser(), py_parser_exports));
|
|
3613
|
+
return parseSymbols9({ file, content, lang: "py" });
|
|
2963
3614
|
}
|
|
2964
3615
|
case "rs": {
|
|
2965
|
-
const { parseSymbols:
|
|
2966
|
-
return
|
|
3616
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_rs_parser(), rs_parser_exports));
|
|
3617
|
+
return parseSymbols9({ file, content, lang: "rs" });
|
|
2967
3618
|
}
|
|
2968
3619
|
case "json": {
|
|
2969
|
-
const { parseSymbols:
|
|
2970
|
-
return
|
|
3620
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_json_parser(), json_parser_exports));
|
|
3621
|
+
return parseSymbols9({ file, content, lang: "json" });
|
|
2971
3622
|
}
|
|
2972
3623
|
case "yaml": {
|
|
2973
|
-
const { parseSymbols:
|
|
2974
|
-
return
|
|
3624
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_yaml_parser(), yaml_parser_exports));
|
|
3625
|
+
return parseSymbols9({ file, content, lang: "yaml" });
|
|
3626
|
+
}
|
|
3627
|
+
// Phase 1: ten languages now route through the Tree-Sitter WASM
|
|
3628
|
+
// universal parser (`tree-sitter-parser.ts`). The dispatch falls back to
|
|
3629
|
+
// the regex extractor in `generic-parser.ts` whenever WASM loading fails
|
|
3630
|
+
// or the parser returns zero symbols — preserving the indexable-file
|
|
3631
|
+
// contract that "missing a parser must never mean skipping the file".
|
|
3632
|
+
case "c":
|
|
3633
|
+
case "cpp":
|
|
3634
|
+
case "java":
|
|
3635
|
+
case "csharp":
|
|
3636
|
+
case "php":
|
|
3637
|
+
case "ruby":
|
|
3638
|
+
case "swift":
|
|
3639
|
+
case "kotlin":
|
|
3640
|
+
case "shell":
|
|
3641
|
+
case "elixir": {
|
|
3642
|
+
try {
|
|
3643
|
+
const { parseSymbols: parseSymbols10 } = await Promise.resolve().then(() => (init_tree_sitter_parser(), tree_sitter_parser_exports));
|
|
3644
|
+
const parsed2 = await parseSymbols10({ file, content, lang });
|
|
3645
|
+
if (parsed2.symbols.length > 0) return parsed2;
|
|
3646
|
+
} catch {
|
|
3647
|
+
}
|
|
3648
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_generic_parser(), generic_parser_exports));
|
|
3649
|
+
return parseSymbols9({ file, content, lang });
|
|
2975
3650
|
}
|
|
2976
3651
|
default: {
|
|
2977
|
-
const { parseSymbols:
|
|
2978
|
-
return
|
|
3652
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_generic_parser(), generic_parser_exports));
|
|
3653
|
+
return parseSymbols9({ file, content, lang });
|
|
2979
3654
|
}
|
|
2980
3655
|
}
|
|
2981
3656
|
}
|
|
@@ -2987,10 +3662,185 @@ function withRelations(parsed2, content, lang) {
|
|
|
2987
3662
|
return { ...parsed2, refs: refs.map((ref) => ref.lang ? ref : { ...ref, lang }) };
|
|
2988
3663
|
}
|
|
2989
3664
|
|
|
3665
|
+
// src/codebase-index/parser-worker-pool.ts
|
|
3666
|
+
import { Worker } from "node:worker_threads";
|
|
3667
|
+
import { fileURLToPath as fileURLToPath2 } from "node:url";
|
|
3668
|
+
import * as fs6 from "node:fs";
|
|
3669
|
+
var WORKER_POOL_THRESHOLD = 500;
|
|
3670
|
+
var ParserWorkerPool = class {
|
|
3671
|
+
constructor(maxWorkers = defaultWorkerCount()) {
|
|
3672
|
+
this.maxWorkers = maxWorkers;
|
|
3673
|
+
}
|
|
3674
|
+
maxWorkers;
|
|
3675
|
+
workers = [];
|
|
3676
|
+
nextBatchId = 1;
|
|
3677
|
+
pending = /* @__PURE__ */ new Map();
|
|
3678
|
+
creating = false;
|
|
3679
|
+
unavailable = false;
|
|
3680
|
+
/**
|
|
3681
|
+
* True if the pool is available for use. Returns false when:
|
|
3682
|
+
* - Worker threads aren't supported (sandbox, exotic runtime)
|
|
3683
|
+
* - The built worker script can't be found
|
|
3684
|
+
* - Pool creation was attempted and failed
|
|
3685
|
+
*/
|
|
3686
|
+
isAvailable() {
|
|
3687
|
+
return !this.unavailable && this.workers.length > 0;
|
|
3688
|
+
}
|
|
3689
|
+
/**
|
|
3690
|
+
* Lazily create the worker pool. Returns true if the pool is ready, false
|
|
3691
|
+
* if it's unavailable (caller should fall back to inline parsing).
|
|
3692
|
+
*/
|
|
3693
|
+
async ensureReady() {
|
|
3694
|
+
if (this.isAvailable()) return true;
|
|
3695
|
+
if (this.unavailable) return false;
|
|
3696
|
+
if (this.creating) {
|
|
3697
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
3698
|
+
return this.isAvailable();
|
|
3699
|
+
}
|
|
3700
|
+
this.creating = true;
|
|
3701
|
+
try {
|
|
3702
|
+
const url = resolveWorkerScriptUrl();
|
|
3703
|
+
if (!url) {
|
|
3704
|
+
this.unavailable = true;
|
|
3705
|
+
return false;
|
|
3706
|
+
}
|
|
3707
|
+
for (let i = 0; i < this.maxWorkers; i++) {
|
|
3708
|
+
try {
|
|
3709
|
+
const w = new Worker(url, { name: `wstack-parser-${i}` });
|
|
3710
|
+
w.unref();
|
|
3711
|
+
w.on("message", (msg) => this.handleMessage(msg));
|
|
3712
|
+
w.on("error", (err) => this.handleError(err, w));
|
|
3713
|
+
this.workers.push({ worker: w, busy: false });
|
|
3714
|
+
} catch {
|
|
3715
|
+
if (this.workers.length === 0) {
|
|
3716
|
+
this.unavailable = true;
|
|
3717
|
+
return false;
|
|
3718
|
+
}
|
|
3719
|
+
break;
|
|
3720
|
+
}
|
|
3721
|
+
}
|
|
3722
|
+
return this.workers.length > 0;
|
|
3723
|
+
} finally {
|
|
3724
|
+
this.creating = false;
|
|
3725
|
+
}
|
|
3726
|
+
}
|
|
3727
|
+
/**
|
|
3728
|
+
* Parse files in parallel across the worker pool. Returns a flat
|
|
3729
|
+
* `FileSymbols[]` in completion order (caller sorts if needed).
|
|
3730
|
+
*
|
|
3731
|
+
* Content is pre-read by the main thread (for the content-hash check)
|
|
3732
|
+
* and passed to workers to avoid a second disk read. Files are
|
|
3733
|
+
* distributed round-robin across workers.
|
|
3734
|
+
*/
|
|
3735
|
+
async parseFiles(files) {
|
|
3736
|
+
if (!this.isAvailable()) {
|
|
3737
|
+
throw new Error("ParserWorkerPool.parseFiles called before ensureReady() succeeded");
|
|
3738
|
+
}
|
|
3739
|
+
if (files.length === 0) return [];
|
|
3740
|
+
const batchId = this.nextBatchId++;
|
|
3741
|
+
const workerCount = Math.min(this.workers.length, files.length);
|
|
3742
|
+
const chunks = Array.from(
|
|
3743
|
+
{ length: workerCount },
|
|
3744
|
+
() => []
|
|
3745
|
+
);
|
|
3746
|
+
for (let i = 0; i < files.length; i++) {
|
|
3747
|
+
chunks[i % workerCount].push(files[i]);
|
|
3748
|
+
}
|
|
3749
|
+
return new Promise((resolve4, reject) => {
|
|
3750
|
+
this.pending.set(batchId, {
|
|
3751
|
+
resolve: resolve4,
|
|
3752
|
+
reject,
|
|
3753
|
+
accumulated: [],
|
|
3754
|
+
expectedWorkers: workerCount,
|
|
3755
|
+
completedWorkers: 0
|
|
3756
|
+
});
|
|
3757
|
+
for (let i = 0; i < workerCount; i++) {
|
|
3758
|
+
const pw = this.workers[i];
|
|
3759
|
+
pw.busy = true;
|
|
3760
|
+
pw.worker.postMessage({
|
|
3761
|
+
type: "parse",
|
|
3762
|
+
id: batchId,
|
|
3763
|
+
files: chunks[i]
|
|
3764
|
+
});
|
|
3765
|
+
}
|
|
3766
|
+
});
|
|
3767
|
+
}
|
|
3768
|
+
/** Shut down all workers. Safe to call multiple times. */
|
|
3769
|
+
async shutdown() {
|
|
3770
|
+
const workers = this.workers.map((w) => w.worker);
|
|
3771
|
+
this.workers = [];
|
|
3772
|
+
this.unavailable = false;
|
|
3773
|
+
for (const w of workers) {
|
|
3774
|
+
try {
|
|
3775
|
+
w.postMessage({ type: "shutdown" });
|
|
3776
|
+
} catch {
|
|
3777
|
+
}
|
|
3778
|
+
}
|
|
3779
|
+
await Promise.allSettled(
|
|
3780
|
+
workers.map(
|
|
3781
|
+
(w) => Promise.race([
|
|
3782
|
+
new Promise((resolve4) => {
|
|
3783
|
+
w.once("exit", () => resolve4());
|
|
3784
|
+
}),
|
|
3785
|
+
new Promise((resolve4) => setTimeout(() => resolve4(), 2e3))
|
|
3786
|
+
]).then(() => {
|
|
3787
|
+
if (!w.threadId) return;
|
|
3788
|
+
return w.terminate().catch(() => {
|
|
3789
|
+
});
|
|
3790
|
+
})
|
|
3791
|
+
)
|
|
3792
|
+
);
|
|
3793
|
+
for (const [, p] of this.pending) p.reject(new Error("ParserWorkerPool shut down"));
|
|
3794
|
+
this.pending.clear();
|
|
3795
|
+
}
|
|
3796
|
+
handleMessage(msg) {
|
|
3797
|
+
const batch = this.pending.get(msg.id);
|
|
3798
|
+
if (!batch) return;
|
|
3799
|
+
batch.accumulated.push(...msg.results);
|
|
3800
|
+
batch.completedWorkers++;
|
|
3801
|
+
const freeWorker = this.workers.find((w) => w.busy);
|
|
3802
|
+
if (freeWorker) freeWorker.busy = false;
|
|
3803
|
+
if (batch.completedWorkers >= batch.expectedWorkers) {
|
|
3804
|
+
this.pending.delete(msg.id);
|
|
3805
|
+
batch.resolve(batch.accumulated);
|
|
3806
|
+
}
|
|
3807
|
+
}
|
|
3808
|
+
handleError(err, source) {
|
|
3809
|
+
this.workers = this.workers.filter((w) => w.worker !== source);
|
|
3810
|
+
if (this.workers.length === 0) {
|
|
3811
|
+
for (const [, p] of this.pending) p.reject(err);
|
|
3812
|
+
this.pending.clear();
|
|
3813
|
+
this.unavailable = true;
|
|
3814
|
+
}
|
|
3815
|
+
}
|
|
3816
|
+
};
|
|
3817
|
+
function defaultWorkerCount() {
|
|
3818
|
+
const cores = globalThis.navigator?.hardwareConcurrency ?? 4;
|
|
3819
|
+
return Math.max(1, Math.min(4, cores - 1));
|
|
3820
|
+
}
|
|
3821
|
+
function resolveWorkerScriptUrl() {
|
|
3822
|
+
for (const rel of [
|
|
3823
|
+
"./parser-worker-script.js",
|
|
3824
|
+
"./codebase-index/parser-worker-script.js"
|
|
3825
|
+
]) {
|
|
3826
|
+
try {
|
|
3827
|
+
const url = new URL(rel, import.meta.url);
|
|
3828
|
+
if (url.protocol === "file:" && fs6.existsSync(fileURLToPath2(url))) return url;
|
|
3829
|
+
} catch {
|
|
3830
|
+
}
|
|
3831
|
+
}
|
|
3832
|
+
return null;
|
|
3833
|
+
}
|
|
3834
|
+
var _pool = null;
|
|
3835
|
+
function getParserPool() {
|
|
3836
|
+
_pool ??= new ParserWorkerPool();
|
|
3837
|
+
return _pool;
|
|
3838
|
+
}
|
|
3839
|
+
|
|
2990
3840
|
// src/codebase-index/writer.ts
|
|
2991
3841
|
import { expectDefined as expectDefined4 } from "@wrongstack/core/utils";
|
|
2992
|
-
import * as
|
|
2993
|
-
import * as
|
|
3842
|
+
import * as fs8 from "node:fs";
|
|
3843
|
+
import * as path11 from "node:path";
|
|
2994
3844
|
|
|
2995
3845
|
// src/codebase-index/bm25.ts
|
|
2996
3846
|
var K1 = 1.5;
|
|
@@ -3264,8 +4114,8 @@ function runSqliteWithRetry(fn) {
|
|
|
3264
4114
|
}
|
|
3265
4115
|
|
|
3266
4116
|
// src/codebase-index/writer-admin.ts
|
|
3267
|
-
import * as
|
|
3268
|
-
import * as
|
|
4117
|
+
import * as fs7 from "node:fs";
|
|
4118
|
+
import * as path10 from "node:path";
|
|
3269
4119
|
var DB_FILE = "index.db";
|
|
3270
4120
|
function getAllIndexableWithStatement(stmt) {
|
|
3271
4121
|
return stmt("SELECT id, text FROM symbols").all().map(({ id, text }) => ({ id, text }));
|
|
@@ -3301,7 +4151,7 @@ function getMetadataWithStatement(stmt, key) {
|
|
|
3301
4151
|
}
|
|
3302
4152
|
function getFileMetaWithStatement(stmt, file) {
|
|
3303
4153
|
const rows = stmt(
|
|
3304
|
-
"SELECT file, lang, mtime_ms, symbol_count, last_indexed FROM files WHERE file = ?"
|
|
4154
|
+
"SELECT file, lang, mtime_ms, symbol_count, last_indexed, content_hash FROM files WHERE file = ?"
|
|
3305
4155
|
).all(file);
|
|
3306
4156
|
const r = rows[0];
|
|
3307
4157
|
if (!r) return null;
|
|
@@ -3310,21 +4160,25 @@ function getFileMetaWithStatement(stmt, file) {
|
|
|
3310
4160
|
lang: r.lang,
|
|
3311
4161
|
mtimeMs: r.mtime_ms,
|
|
3312
4162
|
symbolCount: r.symbol_count,
|
|
3313
|
-
lastIndexed: r.last_indexed
|
|
4163
|
+
lastIndexed: r.last_indexed,
|
|
4164
|
+
contentHash: r.content_hash
|
|
3314
4165
|
};
|
|
3315
4166
|
}
|
|
3316
4167
|
function getAllFileMetasWithStatement(stmt) {
|
|
3317
|
-
return stmt(
|
|
4168
|
+
return stmt(
|
|
4169
|
+
"SELECT file, lang, mtime_ms, symbol_count, last_indexed, content_hash FROM files"
|
|
4170
|
+
).all().map((r) => ({
|
|
3318
4171
|
file: r.file,
|
|
3319
4172
|
lang: r.lang,
|
|
3320
4173
|
mtimeMs: r.mtime_ms,
|
|
3321
4174
|
symbolCount: r.symbol_count,
|
|
3322
|
-
lastIndexed: r.last_indexed
|
|
4175
|
+
lastIndexed: r.last_indexed,
|
|
4176
|
+
contentHash: r.content_hash
|
|
3323
4177
|
}));
|
|
3324
4178
|
}
|
|
3325
4179
|
function getIndexDbSizeBytes(indexDir2) {
|
|
3326
4180
|
try {
|
|
3327
|
-
return
|
|
4181
|
+
return fs7.statSync(path10.join(indexDir2, DB_FILE)).size;
|
|
3328
4182
|
} catch {
|
|
3329
4183
|
return 0;
|
|
3330
4184
|
}
|
|
@@ -3373,6 +4227,18 @@ function bulkInsertFtsWithStatement(stmt, maxSqlVars, ftsAvailable, rows) {
|
|
|
3373
4227
|
insert.run(...binds);
|
|
3374
4228
|
}
|
|
3375
4229
|
}
|
|
4230
|
+
function bulkInsertVectorsWithStatement(stmt, maxSqlVars, rows) {
|
|
4231
|
+
if (rows.length === 0) return;
|
|
4232
|
+
const chunkSize = Math.max(1, Math.floor(maxSqlVars / 2));
|
|
4233
|
+
for (let i = 0; i < rows.length; i += chunkSize) {
|
|
4234
|
+
const chunk = rows.slice(i, i + chunkSize);
|
|
4235
|
+
const placeholders = chunk.map(() => "(?, ?)").join(", ");
|
|
4236
|
+
const insert = stmt(`INSERT INTO symbol_vectors(symbol_id, vector) VALUES ${placeholders}`);
|
|
4237
|
+
const binds = [];
|
|
4238
|
+
for (const r of chunk) binds.push(r.id, r.vector);
|
|
4239
|
+
insert.run(...binds);
|
|
4240
|
+
}
|
|
4241
|
+
}
|
|
3376
4242
|
function bulkInsertRefsWithStatement(stmt, maxSqlVars, refs) {
|
|
3377
4243
|
if (refs.length === 0) return;
|
|
3378
4244
|
const chunkSize = Math.max(1, Math.floor(maxSqlVars / 8));
|
|
@@ -3677,6 +4543,171 @@ function findOutgoingCallsByName(stmt, symbolName, file, limit) {
|
|
|
3677
4543
|
const calls = rows.map(mapCallSiteRow).slice(0, limit);
|
|
3678
4544
|
return { calls, symbolFound: true, unresolvedCount, totalMatches: rows.length };
|
|
3679
4545
|
}
|
|
4546
|
+
function runCteWithSeeds(stmt, seedIds, buildSql) {
|
|
4547
|
+
if (seedIds.length <= 900) {
|
|
4548
|
+
const ph = seedIds.map(() => "?").join(",");
|
|
4549
|
+
return stmt(buildSql(ph)).all(...seedIds);
|
|
4550
|
+
}
|
|
4551
|
+
stmt("DROP TABLE IF EXISTS _cte_seeds").run();
|
|
4552
|
+
try {
|
|
4553
|
+
stmt("CREATE TEMP TABLE _cte_seeds (id INTEGER PRIMARY KEY)").run();
|
|
4554
|
+
for (let i = 0; i < seedIds.length; i += 500) {
|
|
4555
|
+
const chunk = seedIds.slice(i, i + 500);
|
|
4556
|
+
const ph = chunk.map(() => "(?)").join(",");
|
|
4557
|
+
stmt(`INSERT OR IGNORE INTO _cte_seeds (id) VALUES ${ph}`).run(...chunk);
|
|
4558
|
+
}
|
|
4559
|
+
return stmt(buildSql("SELECT id FROM _cte_seeds")).all();
|
|
4560
|
+
} finally {
|
|
4561
|
+
stmt("DROP TABLE IF EXISTS _cte_seeds").run();
|
|
4562
|
+
}
|
|
4563
|
+
}
|
|
4564
|
+
function findTransitiveIncomingCallsByName(stmt, symbolName, file, limit) {
|
|
4565
|
+
const targetIds = resolveSymbolIds(stmt, symbolName, file);
|
|
4566
|
+
if (targetIds.length === 0)
|
|
4567
|
+
return { calls: [], symbolFound: false, ambiguous: false, totalMatches: 0 };
|
|
4568
|
+
let matchIds = targetIds;
|
|
4569
|
+
let ambiguous = false;
|
|
4570
|
+
if (file !== void 0) {
|
|
4571
|
+
const allNamedIds = resolveSymbolIds(stmt, symbolName, void 0);
|
|
4572
|
+
if (allNamedIds.length > targetIds.length) {
|
|
4573
|
+
matchIds = allNamedIds;
|
|
4574
|
+
ambiguous = true;
|
|
4575
|
+
}
|
|
4576
|
+
}
|
|
4577
|
+
const cteSql = (seedSource) => `WITH RECURSIVE incoming_tree(from_id) AS (
|
|
4578
|
+
SELECT r.from_id
|
|
4579
|
+
FROM refs r
|
|
4580
|
+
WHERE r.to_id IN (${seedSource})
|
|
4581
|
+
|
|
4582
|
+
UNION
|
|
4583
|
+
|
|
4584
|
+
SELECT r.from_id
|
|
4585
|
+
FROM refs r
|
|
4586
|
+
JOIN incoming_tree it ON r.to_id = it.from_id
|
|
4587
|
+
)
|
|
4588
|
+
SELECT
|
|
4589
|
+
s.id AS sym_id,
|
|
4590
|
+
s.name AS sym_name,
|
|
4591
|
+
s.kind AS sym_kind,
|
|
4592
|
+
s.lang AS sym_lang,
|
|
4593
|
+
s.file AS sym_file,
|
|
4594
|
+
s.line AS sym_line,
|
|
4595
|
+
s.signature AS sym_signature,
|
|
4596
|
+
'' AS call_type,
|
|
4597
|
+
0 AS ref_line
|
|
4598
|
+
FROM incoming_tree it
|
|
4599
|
+
JOIN symbols s ON s.id = it.from_id
|
|
4600
|
+
GROUP BY s.id
|
|
4601
|
+
ORDER BY s.file, s.line`;
|
|
4602
|
+
const rows = runCteWithSeeds(stmt, matchIds, cteSql);
|
|
4603
|
+
if (!file) {
|
|
4604
|
+
const fallbackRows = stmt(
|
|
4605
|
+
`SELECT
|
|
4606
|
+
s.id AS sym_id,
|
|
4607
|
+
s.name AS sym_name,
|
|
4608
|
+
s.kind AS sym_kind,
|
|
4609
|
+
s.lang AS sym_lang,
|
|
4610
|
+
s.file AS sym_file,
|
|
4611
|
+
s.line AS sym_line,
|
|
4612
|
+
s.signature AS sym_signature,
|
|
4613
|
+
r.call_type,
|
|
4614
|
+
r.line AS ref_line
|
|
4615
|
+
FROM refs r
|
|
4616
|
+
JOIN symbols s ON s.id = r.from_id
|
|
4617
|
+
WHERE r.to_id IS NULL AND r.to_name = ?
|
|
4618
|
+
ORDER BY r.line, r.id`
|
|
4619
|
+
).all(symbolName);
|
|
4620
|
+
rows.push(...fallbackRows);
|
|
4621
|
+
}
|
|
4622
|
+
rows.sort((a, b) => a.ref_line - b.ref_line || a.sym_id - b.sym_id);
|
|
4623
|
+
const allCalls = rows.map(mapCallSiteRow);
|
|
4624
|
+
return {
|
|
4625
|
+
calls: allCalls.slice(0, limit),
|
|
4626
|
+
symbolFound: true,
|
|
4627
|
+
ambiguous,
|
|
4628
|
+
totalMatches: allCalls.length
|
|
4629
|
+
};
|
|
4630
|
+
}
|
|
4631
|
+
function findTransitiveOutgoingCallsByName(stmt, symbolName, file, limit) {
|
|
4632
|
+
const sourceIds = resolveSymbolIds(stmt, symbolName, file);
|
|
4633
|
+
if (sourceIds.length === 0)
|
|
4634
|
+
return { calls: [], symbolFound: false, unresolvedCount: 0, totalMatches: 0 };
|
|
4635
|
+
const unresolvedCount = chunkedIdScalar(
|
|
4636
|
+
stmt,
|
|
4637
|
+
sourceIds,
|
|
4638
|
+
(ph) => `SELECT COUNT(*) AS n FROM refs WHERE from_id IN (${ph}) AND to_id IS NULL`
|
|
4639
|
+
);
|
|
4640
|
+
const cteSql = (seedSource) => `WITH RECURSIVE outgoing_tree(to_id) AS (
|
|
4641
|
+
SELECT r.to_id
|
|
4642
|
+
FROM refs r
|
|
4643
|
+
WHERE r.from_id IN (${seedSource}) AND r.to_id IS NOT NULL
|
|
4644
|
+
|
|
4645
|
+
UNION
|
|
4646
|
+
|
|
4647
|
+
SELECT r.to_id
|
|
4648
|
+
FROM refs r
|
|
4649
|
+
JOIN outgoing_tree ot ON r.from_id = ot.to_id
|
|
4650
|
+
WHERE r.to_id IS NOT NULL
|
|
4651
|
+
)
|
|
4652
|
+
SELECT
|
|
4653
|
+
s.id AS sym_id,
|
|
4654
|
+
s.name AS sym_name,
|
|
4655
|
+
s.kind AS sym_kind,
|
|
4656
|
+
s.lang AS sym_lang,
|
|
4657
|
+
s.file AS sym_file,
|
|
4658
|
+
s.line AS sym_line,
|
|
4659
|
+
s.signature AS sym_signature,
|
|
4660
|
+
'' AS call_type,
|
|
4661
|
+
0 AS ref_line
|
|
4662
|
+
FROM outgoing_tree ot
|
|
4663
|
+
JOIN symbols s ON s.id = ot.to_id
|
|
4664
|
+
GROUP BY s.id
|
|
4665
|
+
ORDER BY s.file, s.line`;
|
|
4666
|
+
const rows = runCteWithSeeds(stmt, sourceIds, cteSql);
|
|
4667
|
+
const calls = rows.map(mapCallSiteRow).slice(0, limit);
|
|
4668
|
+
return { calls, symbolFound: true, unresolvedCount, totalMatches: rows.length };
|
|
4669
|
+
}
|
|
4670
|
+
function findReachableSymbolIds(stmt, seedIds) {
|
|
4671
|
+
if (seedIds.length === 0) return /* @__PURE__ */ new Set();
|
|
4672
|
+
if (seedIds.length > 900) {
|
|
4673
|
+
stmt("DROP TABLE IF EXISTS _seeds").run();
|
|
4674
|
+
try {
|
|
4675
|
+
stmt("CREATE TEMP TABLE _seeds (id INTEGER PRIMARY KEY)").run();
|
|
4676
|
+
for (let i = 0; i < seedIds.length; i += 500) {
|
|
4677
|
+
const chunk = seedIds.slice(i, i + 500);
|
|
4678
|
+
const ph2 = chunk.map(() => "(?)").join(",");
|
|
4679
|
+
stmt(`INSERT OR IGNORE INTO _seeds (id) VALUES ${ph2}`).run(...chunk);
|
|
4680
|
+
}
|
|
4681
|
+
const rows2 = stmt(
|
|
4682
|
+
`WITH RECURSIVE reachable(id) AS (
|
|
4683
|
+
SELECT id FROM _seeds
|
|
4684
|
+
UNION
|
|
4685
|
+
SELECT r.to_id
|
|
4686
|
+
FROM refs r
|
|
4687
|
+
JOIN reachable ON r.from_id = reachable.id
|
|
4688
|
+
WHERE r.to_id IS NOT NULL
|
|
4689
|
+
)
|
|
4690
|
+
SELECT DISTINCT id FROM reachable`
|
|
4691
|
+
).all();
|
|
4692
|
+
return new Set(rows2.map((r) => r.id));
|
|
4693
|
+
} finally {
|
|
4694
|
+
stmt("DROP TABLE IF EXISTS _seeds").run();
|
|
4695
|
+
}
|
|
4696
|
+
}
|
|
4697
|
+
const ph = seedIds.map(() => "?").join(",");
|
|
4698
|
+
const rows = stmt(
|
|
4699
|
+
`WITH RECURSIVE reachable(id) AS (
|
|
4700
|
+
SELECT id FROM symbols WHERE id IN (${ph})
|
|
4701
|
+
UNION
|
|
4702
|
+
SELECT r.to_id
|
|
4703
|
+
FROM refs r
|
|
4704
|
+
JOIN reachable ON r.from_id = reachable.id
|
|
4705
|
+
WHERE r.to_id IS NOT NULL
|
|
4706
|
+
)
|
|
4707
|
+
SELECT DISTINCT id FROM reachable`
|
|
4708
|
+
).all(...seedIds);
|
|
4709
|
+
return new Set(rows.map((r) => r.id));
|
|
4710
|
+
}
|
|
3680
4711
|
function findRefsToWithStatement(stmt, symbolId) {
|
|
3681
4712
|
return stmt(
|
|
3682
4713
|
"SELECT id, from_id, to_name, to_id, call_type, line FROM refs WHERE to_id = ? OR to_name = (SELECT name FROM symbols WHERE id = ?)"
|
|
@@ -3916,6 +4947,12 @@ var CORE_TABLES_SQL = `
|
|
|
3916
4947
|
file TEXT PRIMARY KEY,
|
|
3917
4948
|
lang TEXT NOT NULL,
|
|
3918
4949
|
mtime_ms INTEGER NOT NULL,
|
|
4950
|
+
-- Phase 2: xxHash64 of the file's UTF-8 bytes. Empty string when the
|
|
4951
|
+
-- indexer hasn't populated it yet (legacy rows, schema repaired by
|
|
4952
|
+
-- repairMissingColumns). Compared on incremental re-index so that a
|
|
4953
|
+
-- touch or branch-switch that leaves content byte-identical skips the
|
|
4954
|
+
-- expensive parse phase entirely (refactoring proposal Phase 2).
|
|
4955
|
+
content_hash TEXT NOT NULL DEFAULT '',
|
|
3919
4956
|
symbol_count INTEGER NOT NULL DEFAULT 0,
|
|
3920
4957
|
last_indexed INTEGER NOT NULL,
|
|
3921
4958
|
-- Code Atlas grouping label, computed at index time from the ecosystem's
|
|
@@ -3983,7 +5020,14 @@ var LANG_FAMILY_TABLE_SQL = `
|
|
|
3983
5020
|
);
|
|
3984
5021
|
`;
|
|
3985
5022
|
var LANG_FAMILY_WILDCARD = "*";
|
|
3986
|
-
var SYMBOLS_FTS_SQL = "CREATE VIRTUAL TABLE IF NOT EXISTS symbols_fts USING fts5(text, tokenize = '
|
|
5023
|
+
var SYMBOLS_FTS_SQL = "CREATE VIRTUAL TABLE IF NOT EXISTS symbols_fts USING fts5(text, tokenize = 'trigram')";
|
|
5024
|
+
var SYMBOL_VECTORS_TABLE_SQL = `
|
|
5025
|
+
CREATE TABLE IF NOT EXISTS symbol_vectors (
|
|
5026
|
+
symbol_id INTEGER PRIMARY KEY,
|
|
5027
|
+
vector BLOB NOT NULL,
|
|
5028
|
+
FOREIGN KEY (symbol_id) REFERENCES symbols(id) ON DELETE CASCADE
|
|
5029
|
+
);
|
|
5030
|
+
`;
|
|
3987
5031
|
|
|
3988
5032
|
// src/codebase-index/writer-search-helpers.ts
|
|
3989
5033
|
var SEARCH_CANDIDATE_SCAN_CAP = 5e3;
|
|
@@ -4124,6 +5168,85 @@ var StorePool = class {
|
|
|
4124
5168
|
}
|
|
4125
5169
|
};
|
|
4126
5170
|
|
|
5171
|
+
// src/codebase-index/vector-search.ts
|
|
5172
|
+
var RRF_K = 60;
|
|
5173
|
+
var VECTOR_DIMENSIONS = 384;
|
|
5174
|
+
var NGRAM_SIZE = 3;
|
|
5175
|
+
function embedText(text) {
|
|
5176
|
+
const vec = new Float32Array(VECTOR_DIMENSIONS);
|
|
5177
|
+
const normalized = text.toLowerCase().trim();
|
|
5178
|
+
if (normalized.length < NGRAM_SIZE) {
|
|
5179
|
+
const padded = ` ${normalized} `.slice(0, Math.max(NGRAM_SIZE, normalized.length + 2));
|
|
5180
|
+
for (let i = 0; i <= padded.length - NGRAM_SIZE; i++) {
|
|
5181
|
+
const ngram = padded.slice(i, i + NGRAM_SIZE);
|
|
5182
|
+
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
5183
|
+
vec[bucket] += 1;
|
|
5184
|
+
}
|
|
5185
|
+
} else {
|
|
5186
|
+
for (let i = 0; i <= normalized.length - NGRAM_SIZE; i++) {
|
|
5187
|
+
const ngram = normalized.slice(i, i + NGRAM_SIZE);
|
|
5188
|
+
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
5189
|
+
vec[bucket] += 1;
|
|
5190
|
+
}
|
|
5191
|
+
}
|
|
5192
|
+
let norm = 0;
|
|
5193
|
+
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
5194
|
+
norm += vec[i] * vec[i];
|
|
5195
|
+
}
|
|
5196
|
+
norm = Math.sqrt(norm);
|
|
5197
|
+
if (norm > 0) {
|
|
5198
|
+
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
5199
|
+
vec[i] /= norm;
|
|
5200
|
+
}
|
|
5201
|
+
}
|
|
5202
|
+
return vec;
|
|
5203
|
+
}
|
|
5204
|
+
function hashNgram(str) {
|
|
5205
|
+
let hash = 2166136261;
|
|
5206
|
+
for (let i = 0; i < str.length; i++) {
|
|
5207
|
+
hash ^= str.charCodeAt(i);
|
|
5208
|
+
hash = Math.imul(hash, 16777619);
|
|
5209
|
+
}
|
|
5210
|
+
return hash >>> 0;
|
|
5211
|
+
}
|
|
5212
|
+
function cosineSimilarity(a, b) {
|
|
5213
|
+
let dot = 0;
|
|
5214
|
+
const len = Math.min(a.length, b.length);
|
|
5215
|
+
for (let i = 0; i < len; i++) {
|
|
5216
|
+
dot += a[i] * b[i];
|
|
5217
|
+
}
|
|
5218
|
+
return dot;
|
|
5219
|
+
}
|
|
5220
|
+
function encodeVector(vec) {
|
|
5221
|
+
return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
|
|
5222
|
+
}
|
|
5223
|
+
function decodeVector(buf) {
|
|
5224
|
+
const view = new DataView(
|
|
5225
|
+
buf.buffer,
|
|
5226
|
+
buf.byteOffset,
|
|
5227
|
+
buf.byteLength
|
|
5228
|
+
);
|
|
5229
|
+
const copy = new Float32Array(buf.byteLength / 4);
|
|
5230
|
+
for (let i = 0; i < copy.length; i++) {
|
|
5231
|
+
copy[i] = view.getFloat32(i * 4, true);
|
|
5232
|
+
}
|
|
5233
|
+
return copy;
|
|
5234
|
+
}
|
|
5235
|
+
function reciprocalRankFusion(bm25Ranks, vectorRanks, k = RRF_K) {
|
|
5236
|
+
const allIds = /* @__PURE__ */ new Set([...bm25Ranks.keys(), ...vectorRanks.keys()]);
|
|
5237
|
+
const scored = [];
|
|
5238
|
+
for (const id of allIds) {
|
|
5239
|
+
const bm25Rank = bm25Ranks.get(id);
|
|
5240
|
+
const vecRank = vectorRanks.get(id);
|
|
5241
|
+
let score = 0;
|
|
5242
|
+
if (bm25Rank !== void 0) score += 1 / (k + bm25Rank);
|
|
5243
|
+
if (vecRank !== void 0) score += 1 / (k + vecRank);
|
|
5244
|
+
scored.push([id, score]);
|
|
5245
|
+
}
|
|
5246
|
+
scored.sort((a, b) => b[1] - a[1]);
|
|
5247
|
+
return scored;
|
|
5248
|
+
}
|
|
5249
|
+
|
|
4127
5250
|
// src/codebase-index/writer.ts
|
|
4128
5251
|
var DB_FILE2 = "index.db";
|
|
4129
5252
|
var MAX_STATEMENT_CACHE = 128;
|
|
@@ -4136,6 +5259,12 @@ var IndexStore = class _IndexStore {
|
|
|
4136
5259
|
* When false, ranked search falls back to the LIKE + in-process BM25 path.
|
|
4137
5260
|
*/
|
|
4138
5261
|
ftsAvailable = false;
|
|
5262
|
+
/**
|
|
5263
|
+
* Phase 3: true when the `symbol_vectors` table was created successfully.
|
|
5264
|
+
* When false, hybrid search skips the vector pass and falls back to FTS5
|
|
5265
|
+
* (or LIKE) only.
|
|
5266
|
+
*/
|
|
5267
|
+
vectorsAvailable = false;
|
|
4139
5268
|
/**
|
|
4140
5269
|
* Cache of prepared statements keyed by their SQL text. `DatabaseSync`
|
|
4141
5270
|
* compiles SQL on every `.prepare()` call; for the fixed-SQL methods
|
|
@@ -4194,9 +5323,9 @@ var IndexStore = class _IndexStore {
|
|
|
4194
5323
|
}
|
|
4195
5324
|
constructor(projectRoot2, opts = {}) {
|
|
4196
5325
|
this.indexDir = resolveIndexDir(projectRoot2, opts.indexDir);
|
|
4197
|
-
|
|
5326
|
+
fs8.mkdirSync(this.indexDir, { recursive: true });
|
|
4198
5327
|
const Database = loadDatabaseSync();
|
|
4199
|
-
this.db = new Database(
|
|
5328
|
+
this.db = new Database(path11.join(this.indexDir, DB_FILE2));
|
|
4200
5329
|
applyIndexStorePragmas(this.db);
|
|
4201
5330
|
this.initSchema();
|
|
4202
5331
|
}
|
|
@@ -4234,7 +5363,13 @@ var IndexStore = class _IndexStore {
|
|
|
4234
5363
|
*/
|
|
4235
5364
|
repairMissingColumns() {
|
|
4236
5365
|
const expected = [
|
|
4237
|
-
{
|
|
5366
|
+
{
|
|
5367
|
+
table: "files",
|
|
5368
|
+
columns: [
|
|
5369
|
+
["package", "TEXT NOT NULL DEFAULT ''"],
|
|
5370
|
+
["content_hash", "TEXT NOT NULL DEFAULT ''"]
|
|
5371
|
+
]
|
|
5372
|
+
},
|
|
4238
5373
|
{
|
|
4239
5374
|
table: "refs",
|
|
4240
5375
|
columns: [
|
|
@@ -4266,6 +5401,7 @@ var IndexStore = class _IndexStore {
|
|
|
4266
5401
|
DROP TABLE IF EXISTS symbols;
|
|
4267
5402
|
DROP TABLE IF EXISTS files;
|
|
4268
5403
|
DROP TABLE IF EXISTS refs;
|
|
5404
|
+
DROP TABLE IF EXISTS symbol_vectors;
|
|
4269
5405
|
`);
|
|
4270
5406
|
this.db.exec("DROP TABLE IF EXISTS symbols_fts");
|
|
4271
5407
|
this.stmt("UPDATE metadata SET value = ? WHERE key = ?").run(
|
|
@@ -4287,6 +5423,12 @@ var IndexStore = class _IndexStore {
|
|
|
4287
5423
|
this.db.exec(LANG_FAMILY_TABLE_SQL);
|
|
4288
5424
|
this.seedLangFamilies();
|
|
4289
5425
|
try {
|
|
5426
|
+
const ftsSchema = this.stmt(
|
|
5427
|
+
"SELECT sql FROM sqlite_master WHERE type='table' AND name='symbols_fts'"
|
|
5428
|
+
).get();
|
|
5429
|
+
if (ftsSchema?.sql?.includes("unicode61")) {
|
|
5430
|
+
this.db.exec("DROP TABLE IF EXISTS symbols_fts");
|
|
5431
|
+
}
|
|
4290
5432
|
this.db.exec(SYMBOLS_FTS_SQL);
|
|
4291
5433
|
this.ftsAvailable = true;
|
|
4292
5434
|
const symbolCount = Number(
|
|
@@ -4297,6 +5439,7 @@ var IndexStore = class _IndexStore {
|
|
|
4297
5439
|
);
|
|
4298
5440
|
if (symbolCount !== ftsCount) {
|
|
4299
5441
|
this.db.exec("DELETE FROM symbols_fts");
|
|
5442
|
+
if (this.vectorsAvailable) this.db.exec("DELETE FROM symbol_vectors");
|
|
4300
5443
|
const rows = this.stmt(
|
|
4301
5444
|
"SELECT id, name, signature, doc_comment FROM symbols ORDER BY id"
|
|
4302
5445
|
).all();
|
|
@@ -4314,6 +5457,12 @@ var IndexStore = class _IndexStore {
|
|
|
4314
5457
|
} catch {
|
|
4315
5458
|
this.ftsAvailable = false;
|
|
4316
5459
|
}
|
|
5460
|
+
try {
|
|
5461
|
+
this.db.exec(SYMBOL_VECTORS_TABLE_SQL);
|
|
5462
|
+
this.vectorsAvailable = true;
|
|
5463
|
+
} catch {
|
|
5464
|
+
this.vectorsAvailable = false;
|
|
5465
|
+
}
|
|
4317
5466
|
this.ensureNextSymbolIdSeeded();
|
|
4318
5467
|
}
|
|
4319
5468
|
// ─── ID allocation & bulk helpers ────────────────────────────────────────────
|
|
@@ -4423,6 +5572,7 @@ var IndexStore = class _IndexStore {
|
|
|
4423
5572
|
const result = [];
|
|
4424
5573
|
const bulk = [];
|
|
4425
5574
|
const ftsRows = [];
|
|
5575
|
+
const vectorRows = [];
|
|
4426
5576
|
for (const s of symbols) {
|
|
4427
5577
|
const id = nextId++;
|
|
4428
5578
|
bulk.push({
|
|
@@ -4441,6 +5591,10 @@ var IndexStore = class _IndexStore {
|
|
|
4441
5591
|
if (this.ftsAvailable) {
|
|
4442
5592
|
ftsRows.push({ id, text: buildIndexableText(s.name, s.signature, s.docComment) });
|
|
4443
5593
|
}
|
|
5594
|
+
vectorRows.push({
|
|
5595
|
+
id,
|
|
5596
|
+
vector: encodeVector(embedText(s.text || buildIndexableText(s.name, s.signature, s.docComment)))
|
|
5597
|
+
});
|
|
4444
5598
|
result.push({ ...s, id });
|
|
4445
5599
|
}
|
|
4446
5600
|
bulkInsertSymbolsWithStatement((sql) => this.stmt(sql), _IndexStore.MAX_SQL_VARS, bulk);
|
|
@@ -4450,6 +5604,13 @@ var IndexStore = class _IndexStore {
|
|
|
4450
5604
|
this.ftsAvailable,
|
|
4451
5605
|
ftsRows
|
|
4452
5606
|
);
|
|
5607
|
+
if (this.vectorsAvailable) {
|
|
5608
|
+
bulkInsertVectorsWithStatement(
|
|
5609
|
+
(sql) => this.stmt(sql),
|
|
5610
|
+
_IndexStore.MAX_SQL_VARS,
|
|
5611
|
+
vectorRows
|
|
5612
|
+
);
|
|
5613
|
+
}
|
|
4453
5614
|
this.db.exec("COMMIT");
|
|
4454
5615
|
return result;
|
|
4455
5616
|
} catch (err) {
|
|
@@ -4469,6 +5630,11 @@ var IndexStore = class _IndexStore {
|
|
|
4469
5630
|
"DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
4470
5631
|
).run(file);
|
|
4471
5632
|
}
|
|
5633
|
+
if (this.vectorsAvailable) {
|
|
5634
|
+
this.stmt(
|
|
5635
|
+
"DELETE FROM symbol_vectors WHERE symbol_id IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
5636
|
+
).run(file);
|
|
5637
|
+
}
|
|
4472
5638
|
this.stmt("DELETE FROM symbols WHERE file_fk = ?").run(file);
|
|
4473
5639
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
4474
5640
|
this.db.exec("COMMIT");
|
|
@@ -4494,6 +5660,11 @@ var IndexStore = class _IndexStore {
|
|
|
4494
5660
|
"DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
4495
5661
|
).run(file);
|
|
4496
5662
|
}
|
|
5663
|
+
if (this.vectorsAvailable) {
|
|
5664
|
+
this.stmt(
|
|
5665
|
+
"DELETE FROM symbol_vectors WHERE symbol_id IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
5666
|
+
).run(file);
|
|
5667
|
+
}
|
|
4497
5668
|
this.stmt(
|
|
4498
5669
|
"DELETE FROM refs WHERE from_id IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
4499
5670
|
).run(file);
|
|
@@ -4511,14 +5682,22 @@ var IndexStore = class _IndexStore {
|
|
|
4511
5682
|
upsertFile(meta) {
|
|
4512
5683
|
this.runWithRetry(() => {
|
|
4513
5684
|
this.stmt(
|
|
4514
|
-
`INSERT INTO files(file, lang, mtime_ms, symbol_count, last_indexed)
|
|
4515
|
-
VALUES (?, ?, ?, ?, ?)
|
|
5685
|
+
`INSERT INTO files(file, lang, mtime_ms, content_hash, symbol_count, last_indexed)
|
|
5686
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
4516
5687
|
ON CONFLICT(file) DO UPDATE SET
|
|
4517
5688
|
lang = excluded.lang,
|
|
4518
5689
|
mtime_ms = excluded.mtime_ms,
|
|
5690
|
+
content_hash = excluded.content_hash,
|
|
4519
5691
|
symbol_count = excluded.symbol_count,
|
|
4520
5692
|
last_indexed = excluded.last_indexed`
|
|
4521
|
-
).run(
|
|
5693
|
+
).run(
|
|
5694
|
+
meta.file,
|
|
5695
|
+
meta.lang,
|
|
5696
|
+
meta.mtimeMs,
|
|
5697
|
+
meta.contentHash ?? "",
|
|
5698
|
+
meta.symbolCount,
|
|
5699
|
+
meta.lastIndexed
|
|
5700
|
+
);
|
|
4522
5701
|
});
|
|
4523
5702
|
}
|
|
4524
5703
|
getFileMeta(file) {
|
|
@@ -4685,9 +5864,18 @@ var IndexStore = class _IndexStore {
|
|
|
4685
5864
|
if (mapped === null) return { results: [], total: 0 };
|
|
4686
5865
|
effectiveKind = mapped;
|
|
4687
5866
|
}
|
|
4688
|
-
const
|
|
5867
|
+
const longTokens = tokens.filter((t) => t.length >= 3);
|
|
5868
|
+
const shortTokens = tokens.filter((t) => t.length < 3);
|
|
5869
|
+
if (longTokens.length === 0) {
|
|
5870
|
+
return this.searchRankedFallback(query, filter, safeLimit);
|
|
5871
|
+
}
|
|
5872
|
+
const match = longTokens.map((t) => `"${t.replaceAll('"', "")}"`).join(" OR ");
|
|
4689
5873
|
const conditions = ["symbols_fts MATCH ?"];
|
|
4690
5874
|
const values = [match];
|
|
5875
|
+
for (const shortTok of shortTokens) {
|
|
5876
|
+
conditions.push("s.text LIKE ? ESCAPE '\\'");
|
|
5877
|
+
values.push(`%${escapeLike(shortTok)}%`);
|
|
5878
|
+
}
|
|
4691
5879
|
if (effectiveKind) {
|
|
4692
5880
|
conditions.push("s.kind = ?");
|
|
4693
5881
|
values.push(effectiveKind);
|
|
@@ -4706,7 +5894,7 @@ var IndexStore = class _IndexStore {
|
|
|
4706
5894
|
).all(...values);
|
|
4707
5895
|
const total = countRows[0] ? Number(countRows[0].n) : 0;
|
|
4708
5896
|
if (total === 0) return { results: [], total: 0 };
|
|
4709
|
-
const
|
|
5897
|
+
const bm25Rows = this.stmt(
|
|
4710
5898
|
`SELECT s.id, s.lang, s.kind, s.name, s.file, s.line, s.col, s.signature, s.doc_comment,
|
|
4711
5899
|
-bm25(symbols_fts) AS score,
|
|
4712
5900
|
snippet(symbols_fts, 0, '', '', '\u2026', 12) AS snippet
|
|
@@ -4719,8 +5907,39 @@ var IndexStore = class _IndexStore {
|
|
|
4719
5907
|
bm25(symbols_fts), lower(s.name), s.file, s.line, s.col, s.id
|
|
4720
5908
|
LIMIT ?`
|
|
4721
5909
|
).all(...values, query.trim(), `${escapeLike(query.trim())}%`, safeLimit);
|
|
5910
|
+
if (this.vectorsAvailable && bm25Rows.length > 0) {
|
|
5911
|
+
const queryVec = embedText(query);
|
|
5912
|
+
const candidateIds = bm25Rows.map((r) => r.id);
|
|
5913
|
+
const placeholders = candidateIds.map(() => "?").join(",");
|
|
5914
|
+
const vecRows = this.stmt(
|
|
5915
|
+
`SELECT sv.symbol_id, sv.vector FROM symbol_vectors sv WHERE sv.symbol_id IN (${placeholders})`
|
|
5916
|
+
).all(...candidateIds);
|
|
5917
|
+
const vecScores = vecRows.map((r) => ({
|
|
5918
|
+
id: r.symbol_id,
|
|
5919
|
+
sim: cosineSimilarity(queryVec, decodeVector(r.vector))
|
|
5920
|
+
})).sort((a, b) => b.sim - a.sim);
|
|
5921
|
+
const bm25Rank = /* @__PURE__ */ new Map();
|
|
5922
|
+
bm25Rows.forEach((r, i) => {
|
|
5923
|
+
bm25Rank.set(r.id, i);
|
|
5924
|
+
});
|
|
5925
|
+
const vecRank = /* @__PURE__ */ new Map();
|
|
5926
|
+
vecScores.forEach((r, i) => {
|
|
5927
|
+
vecRank.set(r.id, i);
|
|
5928
|
+
});
|
|
5929
|
+
const fused = reciprocalRankFusion(bm25Rank, vecRank, 60);
|
|
5930
|
+
const fusedScore = new Map(fused);
|
|
5931
|
+
const sorted = [...bm25Rows].sort(
|
|
5932
|
+
(a, b) => (fusedScore.get(b.id) ?? 0) - (fusedScore.get(a.id) ?? 0)
|
|
5933
|
+
);
|
|
5934
|
+
return {
|
|
5935
|
+
results: sorted.map(
|
|
5936
|
+
(row) => mapWriterSearchRow(row, filter?.lspKind, Math.max(1e-4, row.score), row.snippet)
|
|
5937
|
+
),
|
|
5938
|
+
total
|
|
5939
|
+
};
|
|
5940
|
+
}
|
|
4722
5941
|
return {
|
|
4723
|
-
results:
|
|
5942
|
+
results: bm25Rows.map(
|
|
4724
5943
|
(row) => mapWriterSearchRow(row, filter?.lspKind, Math.max(1e-4, row.score), row.snippet)
|
|
4725
5944
|
),
|
|
4726
5945
|
total
|
|
@@ -4848,6 +6067,7 @@ var IndexStore = class _IndexStore {
|
|
|
4848
6067
|
this.db.exec("DROP TABLE IF EXISTS files");
|
|
4849
6068
|
this.db.exec("DROP TABLE IF EXISTS metadata");
|
|
4850
6069
|
if (this.ftsAvailable) this.db.exec("DROP TABLE IF EXISTS symbols_fts");
|
|
6070
|
+
this.db.exec("DROP TABLE IF EXISTS symbol_vectors");
|
|
4851
6071
|
this.db.exec("COMMIT");
|
|
4852
6072
|
this.stmtCache.clear();
|
|
4853
6073
|
this.initSchema();
|
|
@@ -4935,6 +6155,11 @@ var IndexStore = class _IndexStore {
|
|
|
4935
6155
|
`DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file IN (${placeholders}))`
|
|
4936
6156
|
).run(...options.deleteForFiles);
|
|
4937
6157
|
}
|
|
6158
|
+
if (this.vectorsAvailable) {
|
|
6159
|
+
this.stmt(
|
|
6160
|
+
`DELETE FROM symbol_vectors WHERE symbol_id IN (SELECT id FROM symbols WHERE file IN (${placeholders}))`
|
|
6161
|
+
).run(...options.deleteForFiles);
|
|
6162
|
+
}
|
|
4938
6163
|
this.stmt(
|
|
4939
6164
|
`DELETE FROM refs WHERE from_id IN (SELECT id FROM symbols WHERE file IN (${placeholders}))`
|
|
4940
6165
|
).run(...options.deleteForFiles);
|
|
@@ -4948,6 +6173,7 @@ var IndexStore = class _IndexStore {
|
|
|
4948
6173
|
const refsToInsert = [];
|
|
4949
6174
|
const bulkSyms = [];
|
|
4950
6175
|
const ftsRows = [];
|
|
6176
|
+
const vectorRows = [];
|
|
4951
6177
|
for (const entry of entries) {
|
|
4952
6178
|
const insertedForEntry = [];
|
|
4953
6179
|
for (const s of entry.symbols) {
|
|
@@ -4971,6 +6197,10 @@ var IndexStore = class _IndexStore {
|
|
|
4971
6197
|
text: buildIndexableText(s.name, s.signature, s.docComment)
|
|
4972
6198
|
});
|
|
4973
6199
|
}
|
|
6200
|
+
vectorRows.push({
|
|
6201
|
+
id,
|
|
6202
|
+
vector: encodeVector(embedText(s.text || buildIndexableText(s.name, s.signature, s.docComment)))
|
|
6203
|
+
});
|
|
4974
6204
|
const inserted = { ...s, id };
|
|
4975
6205
|
allInserted.push(inserted);
|
|
4976
6206
|
insertedForEntry.push(inserted);
|
|
@@ -4984,19 +6214,34 @@ var IndexStore = class _IndexStore {
|
|
|
4984
6214
|
this.ftsAvailable,
|
|
4985
6215
|
ftsRows
|
|
4986
6216
|
);
|
|
6217
|
+
if (this.vectorsAvailable) {
|
|
6218
|
+
bulkInsertVectorsWithStatement(
|
|
6219
|
+
(sql) => this.stmt(sql),
|
|
6220
|
+
_IndexStore.MAX_SQL_VARS,
|
|
6221
|
+
vectorRows
|
|
6222
|
+
);
|
|
6223
|
+
}
|
|
4987
6224
|
bulkInsertRefsWithStatement((sql) => this.stmt(sql), _IndexStore.MAX_SQL_VARS, refsToInsert);
|
|
4988
6225
|
const upsertStmt = this.stmt(
|
|
4989
|
-
`INSERT INTO files(file, lang, mtime_ms, symbol_count, last_indexed)
|
|
4990
|
-
VALUES (?, ?, ?, ?, ?)
|
|
6226
|
+
`INSERT INTO files(file, lang, mtime_ms, content_hash, symbol_count, last_indexed)
|
|
6227
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
4991
6228
|
ON CONFLICT(file) DO UPDATE SET
|
|
4992
6229
|
lang = excluded.lang,
|
|
4993
6230
|
mtime_ms = excluded.mtime_ms,
|
|
6231
|
+
content_hash = excluded.content_hash,
|
|
4994
6232
|
symbol_count = excluded.symbol_count,
|
|
4995
6233
|
last_indexed = excluded.last_indexed`
|
|
4996
6234
|
);
|
|
4997
6235
|
const now = Date.now();
|
|
4998
6236
|
for (const entry of entries) {
|
|
4999
|
-
upsertStmt.run(
|
|
6237
|
+
upsertStmt.run(
|
|
6238
|
+
entry.file,
|
|
6239
|
+
entry.lang,
|
|
6240
|
+
entry.mtimeMs,
|
|
6241
|
+
entry.contentHash ?? "",
|
|
6242
|
+
entry.symbolCount,
|
|
6243
|
+
now
|
|
6244
|
+
);
|
|
5000
6245
|
}
|
|
5001
6246
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
5002
6247
|
this.db.exec("COMMIT");
|
|
@@ -5087,19 +6332,32 @@ var IndexStore = class _IndexStore {
|
|
|
5087
6332
|
"DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
5088
6333
|
).run(meta.file);
|
|
5089
6334
|
}
|
|
6335
|
+
if (this.vectorsAvailable) {
|
|
6336
|
+
this.stmt(
|
|
6337
|
+
"DELETE FROM symbol_vectors WHERE symbol_id IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
6338
|
+
).run(meta.file);
|
|
6339
|
+
}
|
|
5090
6340
|
this.stmt(
|
|
5091
6341
|
"DELETE FROM refs WHERE from_id IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
5092
6342
|
).run(meta.file);
|
|
5093
6343
|
this.stmt("DELETE FROM symbols WHERE file_fk = ?").run(meta.file);
|
|
5094
6344
|
this.stmt(
|
|
5095
|
-
`INSERT INTO files(file, lang, mtime_ms, symbol_count, last_indexed)
|
|
5096
|
-
VALUES (?, ?, ?, ?, ?)
|
|
6345
|
+
`INSERT INTO files(file, lang, mtime_ms, content_hash, symbol_count, last_indexed)
|
|
6346
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
5097
6347
|
ON CONFLICT(file) DO UPDATE SET
|
|
5098
6348
|
lang = excluded.lang,
|
|
5099
6349
|
mtime_ms = excluded.mtime_ms,
|
|
6350
|
+
content_hash = excluded.content_hash,
|
|
5100
6351
|
symbol_count = excluded.symbol_count,
|
|
5101
6352
|
last_indexed = excluded.last_indexed`
|
|
5102
|
-
).run(
|
|
6353
|
+
).run(
|
|
6354
|
+
meta.file,
|
|
6355
|
+
meta.lang,
|
|
6356
|
+
meta.mtimeMs,
|
|
6357
|
+
meta.contentHash ?? "",
|
|
6358
|
+
meta.symbolCount,
|
|
6359
|
+
meta.lastIndexed
|
|
6360
|
+
);
|
|
5103
6361
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
5104
6362
|
this.db.exec("COMMIT");
|
|
5105
6363
|
} catch (err) {
|
|
@@ -5162,6 +6420,31 @@ var IndexStore = class _IndexStore {
|
|
|
5162
6420
|
findOutgoingCallsByName(symbolName, file, limit = 100) {
|
|
5163
6421
|
return findOutgoingCallsByName((sql) => this.stmt(sql), symbolName, file, limit);
|
|
5164
6422
|
}
|
|
6423
|
+
/**
|
|
6424
|
+
* Transitive incoming-call tree: all symbols that transitively call the
|
|
6425
|
+
* target, to an unbounded depth (cycle-safe via SQL UNION deduplication).
|
|
6426
|
+
* Used by `codebase-incoming-calls` when the caller wants the full call
|
|
6427
|
+
* chain rather than just direct callers.
|
|
6428
|
+
*/
|
|
6429
|
+
findTransitiveIncomingCallsByName(symbolName, file, limit = 200) {
|
|
6430
|
+
return findTransitiveIncomingCallsByName((sql) => this.stmt(sql), symbolName, file, limit);
|
|
6431
|
+
}
|
|
6432
|
+
/**
|
|
6433
|
+
* Transitive outgoing-call tree: all symbols the target transitively calls.
|
|
6434
|
+
* Used by `codebase-outgoing-calls` when the caller wants the full
|
|
6435
|
+
* dependency chain rather than just direct callees.
|
|
6436
|
+
*/
|
|
6437
|
+
findTransitiveOutgoingCallsByName(symbolName, file, limit = 200) {
|
|
6438
|
+
return findTransitiveOutgoingCallsByName((sql) => this.stmt(sql), symbolName, file, limit);
|
|
6439
|
+
}
|
|
6440
|
+
/**
|
|
6441
|
+
* Compute the set of symbol IDs reachable from the given seed IDs using a
|
|
6442
|
+
* native SQLite recursive CTE. Used by dead-code detection to replace the
|
|
6443
|
+
* in-memory BFS.
|
|
6444
|
+
*/
|
|
6445
|
+
findReachableSymbolIds(seedIds) {
|
|
6446
|
+
return findReachableSymbolIds((sql) => this.stmt(sql), seedIds);
|
|
6447
|
+
}
|
|
5165
6448
|
/**
|
|
5166
6449
|
* Find all references TO a given symbol (who calls / uses this symbol?).
|
|
5167
6450
|
*/
|
|
@@ -5252,6 +6535,9 @@ var YIELD_EVERY_N = 50;
|
|
|
5252
6535
|
function resolveParallelBatch() {
|
|
5253
6536
|
return indexParallelBatchSize(availableParallelism());
|
|
5254
6537
|
}
|
|
6538
|
+
function shouldUseParserWorkerPool(candidateFileCount, parseBatchCount) {
|
|
6539
|
+
return !isFrugalPerf() && candidateFileCount >= WORKER_POOL_THRESHOLD && parseBatchCount > 1;
|
|
6540
|
+
}
|
|
5255
6541
|
function yieldEventLoop() {
|
|
5256
6542
|
return new Promise((resolve4) => setImmediate(resolve4));
|
|
5257
6543
|
}
|
|
@@ -5269,15 +6555,15 @@ var INDEXABLE_EXTENSION_SET = new Set(INDEXABLE_EXTENSIONS);
|
|
|
5269
6555
|
var MAX_INDEX_FILE_BYTES = 5 * 1024 * 1024;
|
|
5270
6556
|
var MAX_GIT_FILE_LIST_BYTES = 64 * 1024 * 1024;
|
|
5271
6557
|
function isWithinProject(projectRoot2, file) {
|
|
5272
|
-
const rel =
|
|
5273
|
-
return rel !== "" && !rel.startsWith(`..${
|
|
6558
|
+
const rel = path12.relative(projectRoot2, file);
|
|
6559
|
+
return rel !== "" && !rel.startsWith(`..${path12.sep}`) && rel !== ".." && !path12.isAbsolute(rel);
|
|
5274
6560
|
}
|
|
5275
6561
|
function isMissingPathError(err) {
|
|
5276
6562
|
const code = err?.code;
|
|
5277
6563
|
return code === "ENOENT" || code === "ENOTDIR";
|
|
5278
6564
|
}
|
|
5279
6565
|
function normalizeComparablePath(value) {
|
|
5280
|
-
const resolved =
|
|
6566
|
+
const resolved = path12.resolve(value);
|
|
5281
6567
|
return process.platform === "win32" ? resolved.toLowerCase() : resolved;
|
|
5282
6568
|
}
|
|
5283
6569
|
function gitOutput(projectRoot2, args) {
|
|
@@ -5322,24 +6608,24 @@ async function findGitSourceFiles(projectRoot2, ignore, signal) {
|
|
|
5322
6608
|
const record = statusRecords[i];
|
|
5323
6609
|
if (!record) continue;
|
|
5324
6610
|
const status = record.slice(0, 2);
|
|
5325
|
-
const changedPath =
|
|
6611
|
+
const changedPath = path12.resolve(projectRoot2, record.slice(3));
|
|
5326
6612
|
dirty.add(changedPath);
|
|
5327
6613
|
if (status.includes("D")) deleted.add(changedPath);
|
|
5328
6614
|
if (status.includes("R") || status.includes("C")) {
|
|
5329
6615
|
const source = statusRecords[++i];
|
|
5330
|
-
if (source) dirty.add(
|
|
6616
|
+
if (source) dirty.add(path12.resolve(projectRoot2, source));
|
|
5331
6617
|
}
|
|
5332
6618
|
}
|
|
5333
6619
|
const files = [];
|
|
5334
6620
|
for (const relative3 of output.toString("utf8").split("\0")) {
|
|
5335
6621
|
if (!relative3) continue;
|
|
5336
6622
|
const portable = relative3.replace(/\\/g, "/");
|
|
5337
|
-
if (portable.split("/").some((segment) => ignoreSet.has(segment)) || DEFAULT_IGNORE_FILES.has(
|
|
6623
|
+
if (portable.split("/").some((segment) => ignoreSet.has(segment)) || DEFAULT_IGNORE_FILES.has(path12.posix.basename(portable))) {
|
|
5338
6624
|
continue;
|
|
5339
6625
|
}
|
|
5340
|
-
const full =
|
|
6626
|
+
const full = path12.resolve(projectRoot2, relative3);
|
|
5341
6627
|
if (deleted.has(full)) continue;
|
|
5342
|
-
const ext =
|
|
6628
|
+
const ext = path12.extname(relative3).toLowerCase();
|
|
5343
6629
|
if (INDEXABLE_EXTENSION_SET.has(ext) || detectLang(full) !== null) files.push(full);
|
|
5344
6630
|
}
|
|
5345
6631
|
return {
|
|
@@ -5374,7 +6660,7 @@ async function findSourceFiles(projectRoot2, ignore, isGitIgnored, signal) {
|
|
|
5374
6660
|
}
|
|
5375
6661
|
let entries;
|
|
5376
6662
|
try {
|
|
5377
|
-
entries = await
|
|
6663
|
+
entries = await fs9.readdir(dir, { withFileTypes: true });
|
|
5378
6664
|
} catch (err) {
|
|
5379
6665
|
complete = false;
|
|
5380
6666
|
errors.push(`scan error: ${dir}: ${err instanceof Error ? err.message : String(err)}`);
|
|
@@ -5383,14 +6669,14 @@ async function findSourceFiles(projectRoot2, ignore, isGitIgnored, signal) {
|
|
|
5383
6669
|
dirCount++;
|
|
5384
6670
|
for (const e of entries) {
|
|
5385
6671
|
if (ignoreSet.has(e.name)) continue;
|
|
5386
|
-
const full =
|
|
5387
|
-
const rel =
|
|
6672
|
+
const full = path12.join(dir, e.name);
|
|
6673
|
+
const rel = path12.relative(projectRoot2, full).replace(/\\/g, "/");
|
|
5388
6674
|
if (e.isDirectory()) {
|
|
5389
6675
|
if (isGitIgnored(rel, true)) continue;
|
|
5390
6676
|
await walk(full);
|
|
5391
6677
|
} else if (e.isFile()) {
|
|
5392
6678
|
if (DEFAULT_IGNORE_FILES.has(e.name) || isGitIgnored(rel, false)) continue;
|
|
5393
|
-
const ext =
|
|
6679
|
+
const ext = path12.extname(e.name).toLowerCase();
|
|
5394
6680
|
if (indexableExts.has(ext) || detectLang(full) !== null) {
|
|
5395
6681
|
results.push(full);
|
|
5396
6682
|
}
|
|
@@ -5428,11 +6714,7 @@ async function resolveProjectRelations(store, projectRoot2, opts) {
|
|
|
5428
6714
|
const structure = await detectModuleRoots(projectRoot2, indexedFiles);
|
|
5429
6715
|
if (opts.signal?.aborted) return;
|
|
5430
6716
|
store.setFilePackages(assignPackageLabels(structure, indexedFiles));
|
|
5431
|
-
const resolver = new ModuleResolver(
|
|
5432
|
-
structure,
|
|
5433
|
-
indexedFiles,
|
|
5434
|
-
store.getNamespaceDeclarations()
|
|
5435
|
-
);
|
|
6717
|
+
const resolver = new ModuleResolver(structure, indexedFiles, store.getNamespaceDeclarations());
|
|
5436
6718
|
const pending = store.getUnresolvedImports(opts.onlyFiles);
|
|
5437
6719
|
const resolutions = [];
|
|
5438
6720
|
for (const entry of pending) {
|
|
@@ -5457,6 +6739,10 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5457
6739
|
const errors = [];
|
|
5458
6740
|
const langStats = {};
|
|
5459
6741
|
let filesIndexed = 0;
|
|
6742
|
+
let filesParsed = 0;
|
|
6743
|
+
let filesSkipped = 0;
|
|
6744
|
+
let filesEmpty = 0;
|
|
6745
|
+
let filesFailed = 0;
|
|
5460
6746
|
let symbolsIndexed = 0;
|
|
5461
6747
|
const isGitIgnored = await loadGitignoreMatcher(projectRoot2);
|
|
5462
6748
|
let files;
|
|
@@ -5464,10 +6750,10 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5464
6750
|
let discoveryComplete = true;
|
|
5465
6751
|
let trustedUnchanged;
|
|
5466
6752
|
if (opts.files && opts.files.length > 0) {
|
|
5467
|
-
files = opts.files.map((f) =>
|
|
6753
|
+
files = opts.files.map((f) => path12.resolve(projectRoot2, f)).filter((f) => {
|
|
5468
6754
|
if (!isWithinProject(projectRoot2, f)) return false;
|
|
5469
|
-
const rel =
|
|
5470
|
-
return !rel.split("/").some((seg) => DEFAULT_IGNORE.includes(seg)) && !DEFAULT_IGNORE_FILES.has(
|
|
6755
|
+
const rel = path12.relative(projectRoot2, f).replace(/\\/g, "/");
|
|
6756
|
+
return !rel.split("/").some((seg) => DEFAULT_IGNORE.includes(seg)) && !DEFAULT_IGNORE_FILES.has(path12.basename(f)) && !isGitIgnored(rel, false);
|
|
5471
6757
|
});
|
|
5472
6758
|
} else {
|
|
5473
6759
|
const discovery = await findSourceFiles(projectRoot2, ignore, isGitIgnored, signal);
|
|
@@ -5498,12 +6784,14 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5498
6784
|
langStats[meta.lang] = (langStats[meta.lang] ?? 0) + meta.symbolCount;
|
|
5499
6785
|
symbolsIndexed += meta.symbolCount;
|
|
5500
6786
|
filesIndexed++;
|
|
6787
|
+
filesSkipped++;
|
|
5501
6788
|
filesPreSkipped++;
|
|
5502
6789
|
return false;
|
|
5503
6790
|
});
|
|
5504
6791
|
if (filesPreSkipped > 0) opts.onProgress?.(filesPreSkipped, totalFilesForProgress);
|
|
5505
6792
|
}
|
|
5506
6793
|
const parallelBatch = resolveParallelBatch();
|
|
6794
|
+
const parserPoolCandidateCount = files.length;
|
|
5507
6795
|
let filesSinceLastYield = 0;
|
|
5508
6796
|
for (let batchStart = 0; batchStart < files.length; batchStart += parallelBatch) {
|
|
5509
6797
|
const batchEnd = Math.min(batchStart + parallelBatch, files.length);
|
|
@@ -5524,7 +6812,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5524
6812
|
async (file) => {
|
|
5525
6813
|
let stat2;
|
|
5526
6814
|
try {
|
|
5527
|
-
stat2 = await
|
|
6815
|
+
stat2 = await fs9.stat(file, statOpts);
|
|
5528
6816
|
} catch (e) {
|
|
5529
6817
|
if (isAbortError(e)) throw e;
|
|
5530
6818
|
return {
|
|
@@ -5554,7 +6842,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5554
6842
|
}
|
|
5555
6843
|
let content;
|
|
5556
6844
|
try {
|
|
5557
|
-
content = await
|
|
6845
|
+
content = await fs9.readFile(file, { encoding: "utf8", signal });
|
|
5558
6846
|
} catch (e) {
|
|
5559
6847
|
if (isAbortError(e)) throw e;
|
|
5560
6848
|
return {
|
|
@@ -5565,22 +6853,78 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5565
6853
|
error: `read error: ${e instanceof Error ? e.message : String(e)}`
|
|
5566
6854
|
};
|
|
5567
6855
|
}
|
|
5568
|
-
|
|
5569
|
-
|
|
5570
|
-
parsed2 = await parseFileContent(file, content, lang);
|
|
5571
|
-
} catch (e) {
|
|
6856
|
+
const contentHash = xxhash64String(content);
|
|
6857
|
+
if (!force && meta && meta.contentHash && contentHash === meta.contentHash) {
|
|
5572
6858
|
return {
|
|
5573
6859
|
file,
|
|
5574
6860
|
stat: stat2,
|
|
5575
6861
|
lang,
|
|
5576
6862
|
parsed: null,
|
|
5577
|
-
|
|
6863
|
+
content,
|
|
6864
|
+
contentHash,
|
|
6865
|
+
skippedMeta: { ...meta, mtimeMs: Math.floor(stat2.mtimeMs) }
|
|
5578
6866
|
};
|
|
5579
6867
|
}
|
|
5580
|
-
return { file, stat: stat2, lang, parsed:
|
|
6868
|
+
return { file, stat: stat2, lang, parsed: null, content, contentHash };
|
|
5581
6869
|
}
|
|
5582
6870
|
)
|
|
5583
6871
|
);
|
|
6872
|
+
const toParse = [];
|
|
6873
|
+
for (let pi = 0; pi < statReadParse.length; pi++) {
|
|
6874
|
+
const s = statReadParse[pi];
|
|
6875
|
+
if (s.status !== "fulfilled") continue;
|
|
6876
|
+
const r = s.value;
|
|
6877
|
+
if (r.error || r.skippedMeta || !r.lang || r.parsed) continue;
|
|
6878
|
+
if (r.content === void 0) continue;
|
|
6879
|
+
toParse.push({
|
|
6880
|
+
index: pi,
|
|
6881
|
+
file: batchFiles[pi],
|
|
6882
|
+
content: r.content,
|
|
6883
|
+
lang: r.lang
|
|
6884
|
+
});
|
|
6885
|
+
}
|
|
6886
|
+
if (toParse.length > 0) {
|
|
6887
|
+
let pool = shouldUseParserWorkerPool(parserPoolCandidateCount, toParse.length) ? getParserPool() : null;
|
|
6888
|
+
if (pool) {
|
|
6889
|
+
try {
|
|
6890
|
+
await pool.ensureReady();
|
|
6891
|
+
const parsedResults = await pool.parseFiles(
|
|
6892
|
+
toParse.map((p) => ({ file: p.file, content: p.content, lang: p.lang }))
|
|
6893
|
+
);
|
|
6894
|
+
const byFile = new Map(parsedResults.map((r) => [r.file, r]));
|
|
6895
|
+
for (const item of toParse) {
|
|
6896
|
+
const parsed2 = byFile.get(item.file);
|
|
6897
|
+
const settled = statReadParse[item.index];
|
|
6898
|
+
if (settled.status !== "fulfilled") continue;
|
|
6899
|
+
if (parsed2) {
|
|
6900
|
+
settled.value.parsed = parsed2;
|
|
6901
|
+
} else {
|
|
6902
|
+
settled.value.error = `parse error: worker returned no result for ${item.file}`;
|
|
6903
|
+
}
|
|
6904
|
+
}
|
|
6905
|
+
} catch {
|
|
6906
|
+
pool = null;
|
|
6907
|
+
}
|
|
6908
|
+
}
|
|
6909
|
+
if (!pool) {
|
|
6910
|
+
await Promise.all(
|
|
6911
|
+
toParse.map(async (item) => {
|
|
6912
|
+
try {
|
|
6913
|
+
const parsed2 = await parseFileContent(item.file, item.content, item.lang);
|
|
6914
|
+
const settled = statReadParse[item.index];
|
|
6915
|
+
if (settled.status === "fulfilled") {
|
|
6916
|
+
settled.value.parsed = parsed2;
|
|
6917
|
+
}
|
|
6918
|
+
} catch (e) {
|
|
6919
|
+
const settled = statReadParse[item.index];
|
|
6920
|
+
if (settled.status === "fulfilled") {
|
|
6921
|
+
settled.value.error = `parse error: ${e instanceof Error ? e.message : String(e)}`;
|
|
6922
|
+
}
|
|
6923
|
+
}
|
|
6924
|
+
})
|
|
6925
|
+
);
|
|
6926
|
+
}
|
|
6927
|
+
}
|
|
5584
6928
|
const batchEntries = [];
|
|
5585
6929
|
const deleteForFiles = [];
|
|
5586
6930
|
for (let fi = 0; fi < statReadParse.length; fi++) {
|
|
@@ -5590,12 +6934,14 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5590
6934
|
const err = settled.reason;
|
|
5591
6935
|
if (err instanceof Error && isAbortError(err)) throw err;
|
|
5592
6936
|
errors.push(`batch error: ${file}: ${err instanceof Error ? err.message : String(err)}`);
|
|
6937
|
+
filesFailed++;
|
|
5593
6938
|
continue;
|
|
5594
6939
|
}
|
|
5595
6940
|
const result = settled.value;
|
|
5596
6941
|
if (result.error) {
|
|
5597
6942
|
if (result.missing) store.deleteFile(file);
|
|
5598
6943
|
errors.push(`${file}: ${result.error}`);
|
|
6944
|
+
filesFailed++;
|
|
5599
6945
|
continue;
|
|
5600
6946
|
}
|
|
5601
6947
|
const { stat: stat2, lang, parsed: parsed2 } = result;
|
|
@@ -5603,6 +6949,18 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5603
6949
|
langStats[lang] = (langStats[lang] ?? 0) + result.skippedMeta.symbolCount;
|
|
5604
6950
|
symbolsIndexed += result.skippedMeta.symbolCount;
|
|
5605
6951
|
filesIndexed++;
|
|
6952
|
+
filesSkipped++;
|
|
6953
|
+
const stored = existingMeta.get(file);
|
|
6954
|
+
if (stored && stored.mtimeMs !== result.skippedMeta.mtimeMs) {
|
|
6955
|
+
store.upsertFile({
|
|
6956
|
+
file,
|
|
6957
|
+
lang,
|
|
6958
|
+
mtimeMs: result.skippedMeta.mtimeMs,
|
|
6959
|
+
symbolCount: result.skippedMeta.symbolCount,
|
|
6960
|
+
lastIndexed: Date.now(),
|
|
6961
|
+
contentHash: result.skippedMeta.contentHash
|
|
6962
|
+
});
|
|
6963
|
+
}
|
|
5606
6964
|
continue;
|
|
5607
6965
|
}
|
|
5608
6966
|
if (!lang || !parsed2) {
|
|
@@ -5612,9 +6970,11 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5612
6970
|
lang,
|
|
5613
6971
|
mtimeMs: Math.floor(stat2.mtimeMs),
|
|
5614
6972
|
symbolCount: 0,
|
|
5615
|
-
lastIndexed: Date.now()
|
|
6973
|
+
lastIndexed: Date.now(),
|
|
6974
|
+
contentHash: result.contentHash ?? ""
|
|
5616
6975
|
});
|
|
5617
6976
|
filesIndexed++;
|
|
6977
|
+
filesEmpty++;
|
|
5618
6978
|
}
|
|
5619
6979
|
continue;
|
|
5620
6980
|
}
|
|
@@ -5624,9 +6984,11 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5624
6984
|
lang,
|
|
5625
6985
|
mtimeMs: Math.floor(stat2.mtimeMs),
|
|
5626
6986
|
symbolCount: 0,
|
|
5627
|
-
lastIndexed: Date.now()
|
|
6987
|
+
lastIndexed: Date.now(),
|
|
6988
|
+
contentHash: result.contentHash ?? ""
|
|
5628
6989
|
});
|
|
5629
6990
|
filesIndexed++;
|
|
6991
|
+
filesEmpty++;
|
|
5630
6992
|
continue;
|
|
5631
6993
|
}
|
|
5632
6994
|
batchEntries.push({
|
|
@@ -5635,7 +6997,8 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5635
6997
|
symbols: parsed2.symbols,
|
|
5636
6998
|
refs: parsed2.refs ?? [],
|
|
5637
6999
|
mtimeMs: Math.floor(stat2.mtimeMs),
|
|
5638
|
-
symbolCount: parsed2.symbols.length
|
|
7000
|
+
symbolCount: parsed2.symbols.length,
|
|
7001
|
+
contentHash: result.contentHash ?? ""
|
|
5639
7002
|
});
|
|
5640
7003
|
deleteForFiles.push(file);
|
|
5641
7004
|
}
|
|
@@ -5647,6 +7010,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5647
7010
|
symbolsIndexed += count;
|
|
5648
7011
|
langStats[entry.lang] = (langStats[entry.lang] ?? 0) + count;
|
|
5649
7012
|
filesIndexed++;
|
|
7013
|
+
filesParsed++;
|
|
5650
7014
|
}
|
|
5651
7015
|
} catch (err) {
|
|
5652
7016
|
const message = err instanceof Error ? err.message : String(err);
|
|
@@ -5659,6 +7023,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5659
7023
|
symbolsIndexed += symbolsWithIds.length;
|
|
5660
7024
|
langStats[entry.lang] = (langStats[entry.lang] ?? 0) + symbolsWithIds.length;
|
|
5661
7025
|
filesIndexed++;
|
|
7026
|
+
filesParsed++;
|
|
5662
7027
|
if (entry.refs.length > 0 && symbolsWithIds.length > 0) {
|
|
5663
7028
|
const fallbackBatch = assignRefsToSymbols2(entry.refs, symbolsWithIds);
|
|
5664
7029
|
if (fallbackBatch.length > 0) store.insertRefsBatch(fallbackBatch);
|
|
@@ -5672,9 +7037,11 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5672
7037
|
lang: entry.lang,
|
|
5673
7038
|
mtimeMs: entry.mtimeMs,
|
|
5674
7039
|
symbolCount: entry.symbolCount,
|
|
5675
|
-
lastIndexed: Date.now()
|
|
7040
|
+
lastIndexed: Date.now(),
|
|
7041
|
+
contentHash: entry.contentHash
|
|
5676
7042
|
});
|
|
5677
7043
|
} catch (innerErr) {
|
|
7044
|
+
filesFailed++;
|
|
5678
7045
|
errors.push(
|
|
5679
7046
|
`fallback write failed: ${entry.file}: ${innerErr instanceof Error ? innerErr.message : String(innerErr)}`
|
|
5680
7047
|
);
|
|
@@ -5707,6 +7074,12 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5707
7074
|
const durationMs = Date.now() - startMs;
|
|
5708
7075
|
return {
|
|
5709
7076
|
filesIndexed,
|
|
7077
|
+
fileOutcomes: {
|
|
7078
|
+
parsed: filesParsed,
|
|
7079
|
+
skipped: filesSkipped,
|
|
7080
|
+
empty: filesEmpty,
|
|
7081
|
+
failed: filesFailed
|
|
7082
|
+
},
|
|
5710
7083
|
symbolsIndexed,
|
|
5711
7084
|
langStats,
|
|
5712
7085
|
durationMs,
|
|
@@ -5784,6 +7157,9 @@ function symbolGraphService(args) {
|
|
|
5784
7157
|
function incomingCallsService(args) {
|
|
5785
7158
|
const store = indexStorePool.acquire(args.projectRoot, { indexDir: args.indexDir });
|
|
5786
7159
|
try {
|
|
7160
|
+
if (args.transitive) {
|
|
7161
|
+
return store.findTransitiveIncomingCallsByName(args.symbol, args.file, args.limit ?? 100);
|
|
7162
|
+
}
|
|
5787
7163
|
return store.findIncomingCallsByName(args.symbol, args.file, args.limit ?? 100);
|
|
5788
7164
|
} finally {
|
|
5789
7165
|
indexStorePool.release(store);
|
|
@@ -5792,6 +7168,9 @@ function incomingCallsService(args) {
|
|
|
5792
7168
|
function outgoingCallsService(args) {
|
|
5793
7169
|
const store = indexStorePool.acquire(args.projectRoot, { indexDir: args.indexDir });
|
|
5794
7170
|
try {
|
|
7171
|
+
if (args.transitive) {
|
|
7172
|
+
return store.findTransitiveOutgoingCallsByName(args.symbol, args.file, args.limit ?? 100);
|
|
7173
|
+
}
|
|
5795
7174
|
return store.findOutgoingCallsByName(args.symbol, args.file, args.limit ?? 100);
|
|
5796
7175
|
} finally {
|
|
5797
7176
|
indexStorePool.release(store);
|
|
@@ -5835,10 +7214,10 @@ var GenerationLruCache = class {
|
|
|
5835
7214
|
|
|
5836
7215
|
// src/codebase-index/project-server-endpoint.ts
|
|
5837
7216
|
import { createHash } from "node:crypto";
|
|
5838
|
-
import * as
|
|
7217
|
+
import * as fs10 from "node:fs";
|
|
5839
7218
|
import * as os3 from "node:os";
|
|
5840
|
-
import * as
|
|
5841
|
-
import { fileURLToPath } from "node:url";
|
|
7219
|
+
import * as path13 from "node:path";
|
|
7220
|
+
import { fileURLToPath as fileURLToPath3 } from "node:url";
|
|
5842
7221
|
import { assertUnixSocketPathWithinLimit } from "@wrongstack/core/utils";
|
|
5843
7222
|
var PROJECT_INDEX_SERVER_PROTOCOL_VERSION = 1;
|
|
5844
7223
|
var PROJECT_INDEX_SERVER_METADATA_FILE = "server.json";
|
|
@@ -5847,21 +7226,21 @@ var buildIdCache;
|
|
|
5847
7226
|
function projectIndexServerBuildId(entrypoint) {
|
|
5848
7227
|
const href = entrypoint instanceof URL ? entrypoint.href : entrypoint;
|
|
5849
7228
|
const cleanHref = href.split(/[?#]/, 1)[0] ?? href;
|
|
5850
|
-
const file = cleanHref.startsWith("file:") ?
|
|
7229
|
+
const file = cleanHref.startsWith("file:") ? fileURLToPath3(cleanHref) : path13.resolve(cleanHref);
|
|
5851
7230
|
try {
|
|
5852
|
-
const stat2 =
|
|
7231
|
+
const stat2 = fs10.statSync(file);
|
|
5853
7232
|
if (buildIdCache?.file === file && buildIdCache.mtimeMs === stat2.mtimeMs && buildIdCache.size === stat2.size) {
|
|
5854
7233
|
return buildIdCache.buildId;
|
|
5855
7234
|
}
|
|
5856
|
-
const buildId = createHash("sha256").update(
|
|
7235
|
+
const buildId = createHash("sha256").update(fs10.readFileSync(file)).digest("hex").slice(0, 24);
|
|
5857
7236
|
buildIdCache = { file, mtimeMs: stat2.mtimeMs, size: stat2.size, buildId };
|
|
5858
7237
|
return buildId;
|
|
5859
7238
|
} catch {
|
|
5860
|
-
return `unreadable:${
|
|
7239
|
+
return `unreadable:${path13.basename(file)}`;
|
|
5861
7240
|
}
|
|
5862
7241
|
}
|
|
5863
7242
|
function normalizeLocalPath(value) {
|
|
5864
|
-
const resolved =
|
|
7243
|
+
const resolved = path13.resolve(value);
|
|
5865
7244
|
return process.platform === "win32" ? resolved.toLowerCase() : resolved;
|
|
5866
7245
|
}
|
|
5867
7246
|
function projectIndexServerKey(projectRoot2, indexDir2) {
|
|
@@ -5873,18 +7252,18 @@ function projectIndexServerEndpoint(projectRoot2, indexDir2) {
|
|
|
5873
7252
|
if (process.platform === "win32") {
|
|
5874
7253
|
return `\\\\.\\pipe\\wrongstack-codebase-index-v${PROJECT_INDEX_SERVER_PROTOCOL_VERSION}-${key}`;
|
|
5875
7254
|
}
|
|
5876
|
-
return
|
|
7255
|
+
return path13.join(os3.tmpdir(), PROJECT_INDEX_SERVER_SOCKET_DIR, `${key}.sock`);
|
|
5877
7256
|
}
|
|
5878
7257
|
function projectIndexServerMetadataPath(projectRoot2, indexDir2) {
|
|
5879
|
-
return
|
|
5880
|
-
|
|
7258
|
+
return path13.join(
|
|
7259
|
+
path13.resolve(resolveIndexDir(projectRoot2, indexDir2)),
|
|
5881
7260
|
PROJECT_INDEX_SERVER_METADATA_FILE
|
|
5882
7261
|
);
|
|
5883
7262
|
}
|
|
5884
7263
|
function ensureProjectIndexSocketDirectory(endpoint2) {
|
|
5885
7264
|
if (process.platform !== "win32") {
|
|
5886
7265
|
assertUnixSocketPathWithinLimit(endpoint2, "codebase-index");
|
|
5887
|
-
|
|
7266
|
+
fs10.mkdirSync(path13.dirname(endpoint2), { recursive: true, mode: 448 });
|
|
5888
7267
|
}
|
|
5889
7268
|
}
|
|
5890
7269
|
|
|
@@ -5908,8 +7287,8 @@ function parseArgs(argv) {
|
|
|
5908
7287
|
}
|
|
5909
7288
|
if (!projectRoot2) throw new Error("codebase-index project server requires --project-root");
|
|
5910
7289
|
return {
|
|
5911
|
-
projectRoot:
|
|
5912
|
-
...indexDir2 ? { indexDir:
|
|
7290
|
+
projectRoot: path14.resolve(projectRoot2),
|
|
7291
|
+
...indexDir2 ? { indexDir: path14.resolve(indexDir2) } : {}
|
|
5913
7292
|
};
|
|
5914
7293
|
}
|
|
5915
7294
|
useDaemonPerfDefaults();
|
|
@@ -5938,7 +7317,7 @@ var serverInfo = {
|
|
|
5938
7317
|
endpoint,
|
|
5939
7318
|
startedAt
|
|
5940
7319
|
};
|
|
5941
|
-
process.title = `wrongstack-codebase-index:${
|
|
7320
|
+
process.title = `wrongstack-codebase-index:${path14.basename(projectRoot)}`;
|
|
5942
7321
|
var clients = /* @__PURE__ */ new Set();
|
|
5943
7322
|
var idleTimer;
|
|
5944
7323
|
var stopping = false;
|
|
@@ -6202,12 +7581,12 @@ async function dispatchOperation(state, message) {
|
|
|
6202
7581
|
);
|
|
6203
7582
|
case "incomingCalls": {
|
|
6204
7583
|
const callArgs = fixedArgs(message.args);
|
|
6205
|
-
const cacheKey = JSON.stringify([callArgs.symbol, callArgs.file ?? "", callArgs.limit ?? 100]);
|
|
7584
|
+
const cacheKey = JSON.stringify([callArgs.symbol, callArgs.file ?? "", callArgs.limit ?? 100, callArgs.transitive ?? false]);
|
|
6206
7585
|
return cachedRead(incomingCallsCache, cacheKey, () => incomingCallsService(callArgs));
|
|
6207
7586
|
}
|
|
6208
7587
|
case "outgoingCalls": {
|
|
6209
7588
|
const callArgs = fixedArgs(message.args);
|
|
6210
|
-
const cacheKey = JSON.stringify([callArgs.symbol, callArgs.file ?? "", callArgs.limit ?? 100]);
|
|
7589
|
+
const cacheKey = JSON.stringify([callArgs.symbol, callArgs.file ?? "", callArgs.limit ?? 100, callArgs.transitive ?? false]);
|
|
6211
7590
|
return cachedRead(outgoingCallsCache, cacheKey, () => outgoingCallsService(callArgs));
|
|
6212
7591
|
}
|
|
6213
7592
|
default:
|
|
@@ -6336,9 +7715,9 @@ function ensureExternalWatcher() {
|
|
|
6336
7715
|
projectRoot,
|
|
6337
7716
|
({ filename }) => {
|
|
6338
7717
|
if (!filename || isIgnoredRelativePath(filename)) return;
|
|
6339
|
-
const absolute =
|
|
6340
|
-
const relative3 =
|
|
6341
|
-
if (relative3 === ".." || relative3.startsWith(`..${
|
|
7718
|
+
const absolute = path14.resolve(projectRoot, filename);
|
|
7719
|
+
const relative3 = path14.relative(projectRoot, absolute);
|
|
7720
|
+
if (relative3 === ".." || relative3.startsWith(`..${path14.sep}`) || path14.isAbsolute(relative3) || !isIndexablePath(absolute)) {
|
|
6342
7721
|
return;
|
|
6343
7722
|
}
|
|
6344
7723
|
enqueueExternalFile(absolute);
|
|
@@ -6407,22 +7786,22 @@ function scheduleIdleStop() {
|
|
|
6407
7786
|
}
|
|
6408
7787
|
function removeMetadataIfOwned() {
|
|
6409
7788
|
try {
|
|
6410
|
-
const current = JSON.parse(
|
|
6411
|
-
if (current.pid === process.pid)
|
|
7789
|
+
const current = JSON.parse(fs11.readFileSync(metadataPath, "utf8"));
|
|
7790
|
+
if (current.pid === process.pid) fs11.rmSync(metadataPath, { force: true });
|
|
6412
7791
|
} catch {
|
|
6413
7792
|
}
|
|
6414
7793
|
}
|
|
6415
7794
|
function writeMetadata() {
|
|
6416
|
-
|
|
7795
|
+
fs11.mkdirSync(path14.dirname(metadataPath), { recursive: true });
|
|
6417
7796
|
const temporary = `${metadataPath}.${process.pid}.tmp`;
|
|
6418
7797
|
const metadata = { ...serverInfo, authToken };
|
|
6419
|
-
|
|
7798
|
+
fs11.writeFileSync(temporary, `${JSON.stringify(metadata, null, 2)}
|
|
6420
7799
|
`, { mode: 384 });
|
|
6421
7800
|
try {
|
|
6422
|
-
|
|
7801
|
+
fs11.renameSync(temporary, metadataPath);
|
|
6423
7802
|
} catch {
|
|
6424
|
-
|
|
6425
|
-
|
|
7803
|
+
fs11.rmSync(metadataPath, { force: true });
|
|
7804
|
+
fs11.renameSync(temporary, metadataPath);
|
|
6426
7805
|
}
|
|
6427
7806
|
}
|
|
6428
7807
|
var server = net.createServer((socket) => {
|
|
@@ -6492,7 +7871,7 @@ async function stop(_reason) {
|
|
|
6492
7871
|
});
|
|
6493
7872
|
if (process.platform !== "win32") {
|
|
6494
7873
|
try {
|
|
6495
|
-
|
|
7874
|
+
fs11.rmSync(endpoint, { force: true });
|
|
6496
7875
|
} catch {
|
|
6497
7876
|
}
|
|
6498
7877
|
}
|
|
@@ -6511,7 +7890,7 @@ server.once("error", (error) => {
|
|
|
6511
7890
|
server.listen(endpoint, () => {
|
|
6512
7891
|
if (process.platform !== "win32") {
|
|
6513
7892
|
try {
|
|
6514
|
-
|
|
7893
|
+
fs11.chmodSync(endpoint, 384);
|
|
6515
7894
|
} catch {
|
|
6516
7895
|
}
|
|
6517
7896
|
}
|