@wrongstack/tools 0.302.2 → 0.303.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/builtin.js +2799 -609
- package/dist/codebase-index/binary-frame.d.ts +43 -0
- package/dist/codebase-index/codebase-incoming-calls-tool.d.ts +1 -0
- package/dist/codebase-index/codebase-outgoing-calls-tool.d.ts +1 -0
- package/dist/codebase-index/content-hash.d.ts +66 -0
- package/dist/codebase-index/index.js +1569 -148
- package/dist/codebase-index/parser-worker-pool.d.ts +63 -0
- package/dist/codebase-index/parser-worker-script.d.ts +42 -0
- package/dist/codebase-index/project-server-protocol.d.ts +2 -0
- package/dist/codebase-index/project-server.js +1455 -99
- package/dist/codebase-index/schema.d.ts +7 -0
- package/dist/codebase-index/tree-sitter/queries.d.ts +48 -0
- package/dist/codebase-index/tree-sitter/util.d.ts +31 -0
- package/dist/codebase-index/tree-sitter/visitor.d.ts +47 -0
- package/dist/codebase-index/tree-sitter-parser.d.ts +58 -0
- package/dist/codebase-index/vector-search.d.ts +62 -0
- package/dist/codebase-index/worker-protocol.d.ts +2 -0
- package/dist/codebase-index/worker.js +1424 -68
- package/dist/codebase-index/writer-bulk-insert.d.ts +5 -0
- package/dist/codebase-index/writer-graph-reader.d.ts +39 -0
- package/dist/codebase-index/writer-schema.d.ts +9 -2
- package/dist/codebase-index/writer.d.ts +36 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +2785 -634
- package/dist/kanban-task-inputs.d.ts +1 -0
- package/dist/kanban-tool-schema.d.ts +2 -2
- package/dist/kanban-tool-types.d.ts +18 -2
- package/dist/kanban.js +392 -126
- package/dist/pack.js +2799 -609
- package/dist/plan.d.ts +4 -1
- package/dist/plan.js +2380 -9
- package/dist/read.js +1531 -98
- package/dist/session-kanban.d.ts +8 -0
- package/dist/session-kanban.js +111 -17
- package/dist/task.d.ts +5 -4
- package/dist/task.js +2418 -43
- package/dist/todo.d.ts +10 -1
- package/dist/todo.js +2152 -20
- package/dist/tool-tier.js +2799 -609
- package/package.json +8 -4
|
@@ -2127,15 +2127,586 @@ var init_yaml_parser = __esm({
|
|
|
2127
2127
|
}
|
|
2128
2128
|
});
|
|
2129
2129
|
|
|
2130
|
+
// src/codebase-index/tree-sitter/queries.ts
|
|
2131
|
+
function getQueries(lang) {
|
|
2132
|
+
return LANG_QUERIES[lang] ?? DEFAULT_QUERIES;
|
|
2133
|
+
}
|
|
2134
|
+
function readFirstString(node) {
|
|
2135
|
+
if (!node) return null;
|
|
2136
|
+
if (node.type === "string_literal" || node.type === "alias") {
|
|
2137
|
+
return node.text.replace(/^"|"$/g, "");
|
|
2138
|
+
}
|
|
2139
|
+
const child = node.namedChild(0);
|
|
2140
|
+
return child ? readFirstString(child) : null;
|
|
2141
|
+
}
|
|
2142
|
+
var DEFAULT_QUERIES, LANG_QUERIES;
|
|
2143
|
+
var init_queries = __esm({
|
|
2144
|
+
"src/codebase-index/tree-sitter/queries.ts"() {
|
|
2145
|
+
"use strict";
|
|
2146
|
+
DEFAULT_QUERIES = {
|
|
2147
|
+
declKinds: {}
|
|
2148
|
+
};
|
|
2149
|
+
LANG_QUERIES = {
|
|
2150
|
+
// ─── C family ──────────────────────────────────────────────────────────────
|
|
2151
|
+
c: {
|
|
2152
|
+
declKinds: {
|
|
2153
|
+
function_definition: "function",
|
|
2154
|
+
declaration: "function",
|
|
2155
|
+
// K&R-style `int foo(...)` ambiguous w/ local var; the visitor prefers the function branch when the declarator field is present
|
|
2156
|
+
struct_specifier: "struct",
|
|
2157
|
+
union_specifier: "struct",
|
|
2158
|
+
enum_specifier: "enum",
|
|
2159
|
+
type_definition: "type",
|
|
2160
|
+
// `typedef … X;`
|
|
2161
|
+
preproc_def: "const"
|
|
2162
|
+
// `#define NAME …`
|
|
2163
|
+
},
|
|
2164
|
+
nameField: {
|
|
2165
|
+
function_definition: "declarator",
|
|
2166
|
+
declaration: "declarator",
|
|
2167
|
+
struct_specifier: "name",
|
|
2168
|
+
enum_specifier: "name",
|
|
2169
|
+
type_definition: "declarator",
|
|
2170
|
+
preproc_def: "name"
|
|
2171
|
+
},
|
|
2172
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2173
|
+
"translation_unit",
|
|
2174
|
+
"function_definition",
|
|
2175
|
+
"struct_specifier",
|
|
2176
|
+
"union_specifier",
|
|
2177
|
+
"enum_specifier"
|
|
2178
|
+
])
|
|
2179
|
+
},
|
|
2180
|
+
cpp: {
|
|
2181
|
+
declKinds: {
|
|
2182
|
+
function_definition: "function",
|
|
2183
|
+
template_declaration: "function",
|
|
2184
|
+
// `template<typename T> …`
|
|
2185
|
+
class_specifier: "class",
|
|
2186
|
+
struct_specifier: "struct",
|
|
2187
|
+
union_specifier: "struct",
|
|
2188
|
+
enum_specifier: "enum",
|
|
2189
|
+
namespace_definition: "namespace",
|
|
2190
|
+
type_definition: "type"
|
|
2191
|
+
},
|
|
2192
|
+
nameField: {
|
|
2193
|
+
function_definition: "declarator",
|
|
2194
|
+
template_declaration: "name",
|
|
2195
|
+
class_specifier: "name",
|
|
2196
|
+
struct_specifier: "name",
|
|
2197
|
+
enum_specifier: "name",
|
|
2198
|
+
namespace_definition: "name",
|
|
2199
|
+
type_definition: "declarator"
|
|
2200
|
+
},
|
|
2201
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2202
|
+
"translation_unit",
|
|
2203
|
+
"function_definition",
|
|
2204
|
+
"class_specifier",
|
|
2205
|
+
"struct_specifier",
|
|
2206
|
+
"union_specifier",
|
|
2207
|
+
"enum_specifier",
|
|
2208
|
+
"namespace_definition"
|
|
2209
|
+
])
|
|
2210
|
+
},
|
|
2211
|
+
java: {
|
|
2212
|
+
declKinds: {
|
|
2213
|
+
class_declaration: "class",
|
|
2214
|
+
interface_declaration: "interface",
|
|
2215
|
+
enum_declaration: "enum",
|
|
2216
|
+
record_declaration: "class",
|
|
2217
|
+
annotation_type_declaration: "interface",
|
|
2218
|
+
method_declaration: "method",
|
|
2219
|
+
constructor_declaration: "method",
|
|
2220
|
+
field_declaration: "property"
|
|
2221
|
+
},
|
|
2222
|
+
nameField: {
|
|
2223
|
+
class_declaration: "name",
|
|
2224
|
+
interface_declaration: "name",
|
|
2225
|
+
enum_declaration: "name",
|
|
2226
|
+
record_declaration: "name",
|
|
2227
|
+
annotation_type_declaration: "name",
|
|
2228
|
+
method_declaration: "name",
|
|
2229
|
+
constructor_declaration: "name"
|
|
2230
|
+
},
|
|
2231
|
+
// `field_declaration` has no single `name` field — it carries a list of
|
|
2232
|
+
// variable declarators. We emit one Symbol per node using the first
|
|
2233
|
+
// identifier-shaped named child (see `extractName` fallback in
|
|
2234
|
+
// `visitor.ts`). `int a, b, c;` therefore indexes only `a` — splitting
|
|
2235
|
+
// multi-declarator fields into separate Symbols is a separate refactor
|
|
2236
|
+
// that needs the visitor to know it has multiple names per node, and no
|
|
2237
|
+
// current test relies on it.
|
|
2238
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2239
|
+
"program",
|
|
2240
|
+
"class_declaration",
|
|
2241
|
+
"interface_declaration",
|
|
2242
|
+
"enum_declaration",
|
|
2243
|
+
"record_declaration"
|
|
2244
|
+
])
|
|
2245
|
+
},
|
|
2246
|
+
csharp: {
|
|
2247
|
+
// C# 10+ `namespace Foo.Bar;` produces this node type. The legacy block
|
|
2248
|
+
// form `namespace Foo.Bar { ... }` produces `namespace_declaration`. Both
|
|
2249
|
+
// carry a `qualified_name` child whose text already includes the dots.
|
|
2250
|
+
// `using_directive` is intentionally not a declaration. Imports are
|
|
2251
|
+
// extracted separately; indexing a using directive as a namespace makes
|
|
2252
|
+
// the resolver bind it to its own source file before the real declaration.
|
|
2253
|
+
declKinds: {
|
|
2254
|
+
file_scoped_namespace_declaration: "namespace",
|
|
2255
|
+
class_declaration: "class",
|
|
2256
|
+
interface_declaration: "interface",
|
|
2257
|
+
struct_declaration: "struct",
|
|
2258
|
+
enum_declaration: "enum",
|
|
2259
|
+
record_declaration: "class",
|
|
2260
|
+
method_declaration: "method",
|
|
2261
|
+
constructor_declaration: "method",
|
|
2262
|
+
property_declaration: "property",
|
|
2263
|
+
field_declaration: "property",
|
|
2264
|
+
namespace_declaration: "namespace"
|
|
2265
|
+
},
|
|
2266
|
+
// Custom name extractor: take the full dotted name verbatim.
|
|
2267
|
+
nameExtractor: (node) => {
|
|
2268
|
+
const inner = node.namedChild(0);
|
|
2269
|
+
if (inner && (inner.type === "qualified_name" || inner.type === "name")) {
|
|
2270
|
+
return inner.text;
|
|
2271
|
+
}
|
|
2272
|
+
return null;
|
|
2273
|
+
},
|
|
2274
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2275
|
+
"compilation_unit",
|
|
2276
|
+
"namespace_declaration",
|
|
2277
|
+
"class_declaration",
|
|
2278
|
+
"interface_declaration",
|
|
2279
|
+
"struct_declaration",
|
|
2280
|
+
"enum_declaration",
|
|
2281
|
+
"record_declaration"
|
|
2282
|
+
])
|
|
2283
|
+
},
|
|
2284
|
+
php: {
|
|
2285
|
+
declKinds: {
|
|
2286
|
+
function_definition: "function",
|
|
2287
|
+
method_declaration: "method",
|
|
2288
|
+
class_declaration: "class",
|
|
2289
|
+
interface_declaration: "interface",
|
|
2290
|
+
trait_declaration: "class",
|
|
2291
|
+
enum_declaration: "enum",
|
|
2292
|
+
namespace_definition: "namespace"
|
|
2293
|
+
},
|
|
2294
|
+
nameField: {
|
|
2295
|
+
function_definition: "name",
|
|
2296
|
+
method_declaration: "name",
|
|
2297
|
+
class_declaration: "name",
|
|
2298
|
+
interface_declaration: "name",
|
|
2299
|
+
trait_declaration: "name",
|
|
2300
|
+
enum_declaration: "name",
|
|
2301
|
+
namespace_declaration: "name"
|
|
2302
|
+
},
|
|
2303
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2304
|
+
"program",
|
|
2305
|
+
"namespace_definition",
|
|
2306
|
+
"class_declaration",
|
|
2307
|
+
"interface_declaration",
|
|
2308
|
+
"trait_declaration",
|
|
2309
|
+
"enum_declaration"
|
|
2310
|
+
])
|
|
2311
|
+
},
|
|
2312
|
+
// ─── Scripting / mobile ────────────────────────────────────────────────────
|
|
2313
|
+
ruby: {
|
|
2314
|
+
declKinds: {
|
|
2315
|
+
method: "function",
|
|
2316
|
+
singleton_method: "method",
|
|
2317
|
+
class: "class",
|
|
2318
|
+
module: "namespace",
|
|
2319
|
+
constant: "const"
|
|
2320
|
+
},
|
|
2321
|
+
nameField: {
|
|
2322
|
+
method: "name",
|
|
2323
|
+
singleton_method: "name",
|
|
2324
|
+
class: "name",
|
|
2325
|
+
module: "name",
|
|
2326
|
+
constant: "name"
|
|
2327
|
+
},
|
|
2328
|
+
scopeNodes: /* @__PURE__ */ new Set(["program", "class", "module", "singleton_method", "method"])
|
|
2329
|
+
},
|
|
2330
|
+
swift: {
|
|
2331
|
+
declKinds: {
|
|
2332
|
+
function_declaration: "function",
|
|
2333
|
+
class_declaration: "class",
|
|
2334
|
+
struct_declaration: "struct",
|
|
2335
|
+
enum_declaration: "enum",
|
|
2336
|
+
protocol_declaration: "interface",
|
|
2337
|
+
actor_declaration: "class",
|
|
2338
|
+
extension_declaration: "class",
|
|
2339
|
+
initializer: "method",
|
|
2340
|
+
property_declaration: "property"
|
|
2341
|
+
},
|
|
2342
|
+
nameField: {
|
|
2343
|
+
function_declaration: "name",
|
|
2344
|
+
class_declaration: "name",
|
|
2345
|
+
struct_declaration: "name",
|
|
2346
|
+
enum_declaration: "name",
|
|
2347
|
+
protocol_declaration: "name",
|
|
2348
|
+
actor_declaration: "name",
|
|
2349
|
+
extension_declaration: "name",
|
|
2350
|
+
initializer: "name",
|
|
2351
|
+
property_declaration: "name"
|
|
2352
|
+
},
|
|
2353
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2354
|
+
"source_file",
|
|
2355
|
+
"class_declaration",
|
|
2356
|
+
"struct_declaration",
|
|
2357
|
+
"enum_declaration",
|
|
2358
|
+
"protocol_declaration",
|
|
2359
|
+
"actor_declaration",
|
|
2360
|
+
"extension_declaration"
|
|
2361
|
+
])
|
|
2362
|
+
},
|
|
2363
|
+
kotlin: {
|
|
2364
|
+
declKinds: {
|
|
2365
|
+
class_declaration: "class",
|
|
2366
|
+
object_declaration: "class",
|
|
2367
|
+
interface_declaration: "interface",
|
|
2368
|
+
function_declaration: "function",
|
|
2369
|
+
property_declaration: "property",
|
|
2370
|
+
type_alias: "type"
|
|
2371
|
+
},
|
|
2372
|
+
nameField: {
|
|
2373
|
+
class_declaration: "name",
|
|
2374
|
+
object_declaration: "name",
|
|
2375
|
+
interface_declaration: "name",
|
|
2376
|
+
function_declaration: "name",
|
|
2377
|
+
property_declaration: "name",
|
|
2378
|
+
type_alias: "name"
|
|
2379
|
+
},
|
|
2380
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2381
|
+
"source_file",
|
|
2382
|
+
"class_declaration",
|
|
2383
|
+
"object_declaration",
|
|
2384
|
+
"interface_declaration",
|
|
2385
|
+
"function_declaration"
|
|
2386
|
+
])
|
|
2387
|
+
},
|
|
2388
|
+
elixir: {
|
|
2389
|
+
declKinds: {
|
|
2390
|
+
// `def foo`, `defp foo`, `defmacro foo`, `macrop foo` all surface as
|
|
2391
|
+
// `call` nodes in the tree-sitter grammar — there is no
|
|
2392
|
+
// `function_definition`. The `nameExtractor` walks the call's
|
|
2393
|
+
// children to pick the right sibling identifier.
|
|
2394
|
+
call: "function",
|
|
2395
|
+
module: "namespace"
|
|
2396
|
+
},
|
|
2397
|
+
nameExtractor: (node) => {
|
|
2398
|
+
if (node.type === "module") {
|
|
2399
|
+
const aliasNode = node.childForFieldName("alias");
|
|
2400
|
+
return readFirstString(aliasNode) ?? null;
|
|
2401
|
+
}
|
|
2402
|
+
if (node.type !== "call") return null;
|
|
2403
|
+
const first = node.namedChild(0);
|
|
2404
|
+
if (!first) return null;
|
|
2405
|
+
const target = first.text;
|
|
2406
|
+
if (target !== "def" && target !== "defp" && target !== "defmacro" && target !== "defp_macro" && target !== "macrop" && target !== "defprotocol" && target !== "defguard" && target !== "defguardp") {
|
|
2407
|
+
return null;
|
|
2408
|
+
}
|
|
2409
|
+
const nameNode = node.namedChild(1);
|
|
2410
|
+
return nameNode?.text ?? null;
|
|
2411
|
+
},
|
|
2412
|
+
scopeNodes: /* @__PURE__ */ new Set(["source", "module"])
|
|
2413
|
+
},
|
|
2414
|
+
shell: {
|
|
2415
|
+
declKinds: {
|
|
2416
|
+
function_definition: "function"
|
|
2417
|
+
},
|
|
2418
|
+
nameField: { function_definition: "name" },
|
|
2419
|
+
scopeNodes: /* @__PURE__ */ new Set(["program", "function_definition"])
|
|
2420
|
+
}
|
|
2421
|
+
};
|
|
2422
|
+
}
|
|
2423
|
+
});
|
|
2424
|
+
|
|
2425
|
+
// src/codebase-index/tree-sitter/util.ts
|
|
2426
|
+
function lineColAt2(offsets, index) {
|
|
2427
|
+
let low = 0;
|
|
2428
|
+
let high = offsets.length;
|
|
2429
|
+
while (low < high) {
|
|
2430
|
+
const mid = low + high >>> 1;
|
|
2431
|
+
if ((offsets[mid] ?? 0) < index) low = mid + 1;
|
|
2432
|
+
else high = mid;
|
|
2433
|
+
}
|
|
2434
|
+
const lastNl = low > 0 ? offsets[low - 1] ?? -1 : -1;
|
|
2435
|
+
return { line: low + 1, col: index - lastNl };
|
|
2436
|
+
}
|
|
2437
|
+
function newlineOffsets3(content) {
|
|
2438
|
+
const offsets = [];
|
|
2439
|
+
for (let i = 0; i < content.length; i++) {
|
|
2440
|
+
if (content.charCodeAt(i) === 10) offsets.push(i);
|
|
2441
|
+
}
|
|
2442
|
+
return offsets;
|
|
2443
|
+
}
|
|
2444
|
+
var TREE_SITTER_MAX_FILE_CHARS, TREE_SITTER_MAX_SYMBOLS;
|
|
2445
|
+
var init_util = __esm({
|
|
2446
|
+
"src/codebase-index/tree-sitter/util.ts"() {
|
|
2447
|
+
"use strict";
|
|
2448
|
+
TREE_SITTER_MAX_FILE_CHARS = 512 * 1024;
|
|
2449
|
+
TREE_SITTER_MAX_SYMBOLS = 500;
|
|
2450
|
+
}
|
|
2451
|
+
});
|
|
2452
|
+
|
|
2453
|
+
// src/codebase-index/tree-sitter/visitor.ts
|
|
2454
|
+
function visitTree(tree, content, file, lang, queries) {
|
|
2455
|
+
const boundedContent = content.length > TREE_SITTER_MAX_FILE_CHARS ? content.slice(0, TREE_SITTER_MAX_FILE_CHARS) : content;
|
|
2456
|
+
const nlOffsets = newlineOffsets3(boundedContent);
|
|
2457
|
+
const symbols = [];
|
|
2458
|
+
const scopeStack = [];
|
|
2459
|
+
function visit(node, depth) {
|
|
2460
|
+
if (symbols.length >= TREE_SITTER_MAX_SYMBOLS) return;
|
|
2461
|
+
if (node.isMissing || node.isError) {
|
|
2462
|
+
} else {
|
|
2463
|
+
const kind = queries.declKinds[node.type];
|
|
2464
|
+
if (kind) {
|
|
2465
|
+
const emitted = emitSymbol(
|
|
2466
|
+
node,
|
|
2467
|
+
kind,
|
|
2468
|
+
file,
|
|
2469
|
+
lang,
|
|
2470
|
+
scopeStack,
|
|
2471
|
+
boundedContent,
|
|
2472
|
+
nlOffsets,
|
|
2473
|
+
queries
|
|
2474
|
+
);
|
|
2475
|
+
if (emitted) symbols.push(emitted);
|
|
2476
|
+
}
|
|
2477
|
+
}
|
|
2478
|
+
const pushesScope = queries.scopeNodes?.has(node.type) ?? false;
|
|
2479
|
+
const pushIdx = pushesScope ? pushScope(scopeStack, node, queries) : -1;
|
|
2480
|
+
if (queries.skipNamedChildren) {
|
|
2481
|
+
if (pushIdx !== -1) scopeStack.pop();
|
|
2482
|
+
return;
|
|
2483
|
+
}
|
|
2484
|
+
for (const child of node.namedChildren) {
|
|
2485
|
+
visit(child, depth + 1);
|
|
2486
|
+
if (symbols.length >= TREE_SITTER_MAX_SYMBOLS) {
|
|
2487
|
+
if (pushIdx !== -1) scopeStack.pop();
|
|
2488
|
+
return;
|
|
2489
|
+
}
|
|
2490
|
+
}
|
|
2491
|
+
if (pushIdx !== -1) scopeStack.pop();
|
|
2492
|
+
}
|
|
2493
|
+
visit(tree.rootNode, 0);
|
|
2494
|
+
return { symbols };
|
|
2495
|
+
}
|
|
2496
|
+
function pushScope(scopeStack, node, queries) {
|
|
2497
|
+
const name = extractName(node, queries);
|
|
2498
|
+
if (!name) return -1;
|
|
2499
|
+
scopeStack.push(name);
|
|
2500
|
+
return scopeStack.length - 1;
|
|
2501
|
+
}
|
|
2502
|
+
function extractName(node, queries) {
|
|
2503
|
+
if (queries.nameExtractor) {
|
|
2504
|
+
const extracted = queries.nameExtractor(node);
|
|
2505
|
+
if (extracted) return extracted;
|
|
2506
|
+
}
|
|
2507
|
+
const fieldName = queries.nameField?.[node.type] ?? "name";
|
|
2508
|
+
const field = node.childForFieldName(fieldName);
|
|
2509
|
+
if (field) {
|
|
2510
|
+
if (IDENTIFIER_NODE_TYPES.has(field.type)) {
|
|
2511
|
+
return field.text;
|
|
2512
|
+
}
|
|
2513
|
+
const inner = field.childForFieldName("name") ?? field.namedChild(0);
|
|
2514
|
+
if (inner && IDENTIFIER_NODE_TYPES.has(inner.type)) {
|
|
2515
|
+
return inner.text;
|
|
2516
|
+
}
|
|
2517
|
+
}
|
|
2518
|
+
for (let i = 0; i < node.namedChildCount; i++) {
|
|
2519
|
+
const child = node.namedChild(i);
|
|
2520
|
+
if (child && IDENTIFIER_NODE_TYPES.has(child.type)) {
|
|
2521
|
+
return child.text;
|
|
2522
|
+
}
|
|
2523
|
+
}
|
|
2524
|
+
return null;
|
|
2525
|
+
}
|
|
2526
|
+
function emitSymbol(node, kind, file, lang, scopeStack, content, nlOffsets, queries) {
|
|
2527
|
+
const name = extractName(node, queries);
|
|
2528
|
+
if (!name) return null;
|
|
2529
|
+
const pos = node.startIndex;
|
|
2530
|
+
const { line, col } = lineColAt2(nlOffsets, pos);
|
|
2531
|
+
const end = Math.min(node.endIndex, content.length);
|
|
2532
|
+
const signature = content.slice(pos, end).replace(/\s+/g, " ").trim().slice(0, 500);
|
|
2533
|
+
const scope = scopeStack.join(".");
|
|
2534
|
+
const text = [name, signature].filter(Boolean).join(" | ").trim().slice(0, 1e3);
|
|
2535
|
+
return {
|
|
2536
|
+
id: 0,
|
|
2537
|
+
// caller assigns during bulk insertion
|
|
2538
|
+
lang,
|
|
2539
|
+
kind,
|
|
2540
|
+
name: name.slice(0, 200),
|
|
2541
|
+
file,
|
|
2542
|
+
line,
|
|
2543
|
+
col,
|
|
2544
|
+
signature,
|
|
2545
|
+
docComment: "",
|
|
2546
|
+
// doc-comment extraction lands with ref emission on Day 4
|
|
2547
|
+
scope,
|
|
2548
|
+
text
|
|
2549
|
+
};
|
|
2550
|
+
}
|
|
2551
|
+
var IDENTIFIER_NODE_TYPES;
|
|
2552
|
+
var init_visitor = __esm({
|
|
2553
|
+
"src/codebase-index/tree-sitter/visitor.ts"() {
|
|
2554
|
+
"use strict";
|
|
2555
|
+
init_util();
|
|
2556
|
+
IDENTIFIER_NODE_TYPES = /* @__PURE__ */ new Set([
|
|
2557
|
+
"identifier",
|
|
2558
|
+
"simple_identifier",
|
|
2559
|
+
"type_identifier",
|
|
2560
|
+
"field_identifier",
|
|
2561
|
+
"property_identifier",
|
|
2562
|
+
"name",
|
|
2563
|
+
"word",
|
|
2564
|
+
"variable_name",
|
|
2565
|
+
"constant",
|
|
2566
|
+
"sym"
|
|
2567
|
+
]);
|
|
2568
|
+
}
|
|
2569
|
+
});
|
|
2570
|
+
|
|
2571
|
+
// src/codebase-index/tree-sitter-parser.ts
|
|
2572
|
+
var tree_sitter_parser_exports = {};
|
|
2573
|
+
__export(tree_sitter_parser_exports, {
|
|
2574
|
+
__smokeRootType: () => __smokeRootType,
|
|
2575
|
+
getGrammarWasmPath: () => getGrammarWasmPath,
|
|
2576
|
+
isTreeSitterSupported: () => isTreeSitterSupported,
|
|
2577
|
+
loadTreeSitterLanguage: () => loadTreeSitterLanguage,
|
|
2578
|
+
parseSymbols: () => parseSymbols8
|
|
2579
|
+
});
|
|
2580
|
+
import * as path9 from "node:path";
|
|
2581
|
+
import { fileURLToPath } from "node:url";
|
|
2582
|
+
function optInEnabled(env) {
|
|
2583
|
+
return process.env[env] === "1" || process.env[env] === "true";
|
|
2584
|
+
}
|
|
2585
|
+
function getRuntime() {
|
|
2586
|
+
if (!runtimePromise) {
|
|
2587
|
+
runtimePromise = (async () => {
|
|
2588
|
+
const mod = await import("web-tree-sitter");
|
|
2589
|
+
const init = () => mod.Parser.init({ locateFile: () => RUNTIME_WASM });
|
|
2590
|
+
return { Parser: mod.Parser, Language: mod.Language, init };
|
|
2591
|
+
})();
|
|
2592
|
+
}
|
|
2593
|
+
return runtimePromise;
|
|
2594
|
+
}
|
|
2595
|
+
async function loadLanguage(lang) {
|
|
2596
|
+
const existing = languageCache.get(lang);
|
|
2597
|
+
if (existing) return existing;
|
|
2598
|
+
const promise = (async () => {
|
|
2599
|
+
const grammarName = resolveGrammarName(lang);
|
|
2600
|
+
if (!grammarName) {
|
|
2601
|
+
throw new Error(`tree-sitter: no grammar registered for lang "${lang}"`);
|
|
2602
|
+
}
|
|
2603
|
+
const wasmPath = path9.join(WASM_DIR, grammarName, `tree-sitter-${grammarName}.wasm`);
|
|
2604
|
+
const { Language, init } = await getRuntime();
|
|
2605
|
+
await init();
|
|
2606
|
+
const languageObj = await Language.load(wasmPath);
|
|
2607
|
+
return { lang, Language: languageObj };
|
|
2608
|
+
})();
|
|
2609
|
+
languageCache.set(lang, promise);
|
|
2610
|
+
return promise;
|
|
2611
|
+
}
|
|
2612
|
+
function resolveGrammarName(lang) {
|
|
2613
|
+
if (lang === "go" && optInEnabled(GO_OPT_IN)) return "go";
|
|
2614
|
+
if (lang === "py" && optInEnabled(PY_OPT_IN)) return "python";
|
|
2615
|
+
if (lang === "rs" && optInEnabled(RS_OPT_IN)) return "rust";
|
|
2616
|
+
return LANG_TO_GRAMMAR[lang];
|
|
2617
|
+
}
|
|
2618
|
+
function isTreeSitterSupported(lang) {
|
|
2619
|
+
return resolveGrammarName(lang) !== void 0;
|
|
2620
|
+
}
|
|
2621
|
+
function getGrammarWasmPath(lang) {
|
|
2622
|
+
const name = resolveGrammarName(lang);
|
|
2623
|
+
if (!name) return void 0;
|
|
2624
|
+
return path9.join(WASM_DIR, name, `tree-sitter-${name}.wasm`);
|
|
2625
|
+
}
|
|
2626
|
+
async function parseSymbols8(opts) {
|
|
2627
|
+
const { file, content, lang } = opts;
|
|
2628
|
+
if (!isTreeSitterSupported(lang)) {
|
|
2629
|
+
return { file, lang, symbols: [], mtimeMs: Date.now() };
|
|
2630
|
+
}
|
|
2631
|
+
try {
|
|
2632
|
+
const { Parser } = await getRuntime();
|
|
2633
|
+
const cached = await loadLanguage(lang);
|
|
2634
|
+
const parser = new Parser();
|
|
2635
|
+
parser.setLanguage(cached.Language);
|
|
2636
|
+
const tree = parser.parse(content);
|
|
2637
|
+
if (!tree) {
|
|
2638
|
+
return { file, lang, symbols: [], mtimeMs: Date.now() };
|
|
2639
|
+
}
|
|
2640
|
+
const { symbols } = visitTree(tree, content, file, lang, getQueries(lang));
|
|
2641
|
+
parser.delete();
|
|
2642
|
+
tree.delete();
|
|
2643
|
+
return { file, lang, symbols, refs: [], mtimeMs: Date.now() };
|
|
2644
|
+
} catch {
|
|
2645
|
+
return { file, lang, symbols: [], mtimeMs: Date.now() };
|
|
2646
|
+
}
|
|
2647
|
+
}
|
|
2648
|
+
async function loadTreeSitterLanguage(lang) {
|
|
2649
|
+
const cached = await loadLanguage(lang);
|
|
2650
|
+
return cached.Language;
|
|
2651
|
+
}
|
|
2652
|
+
async function __smokeRootType(opts) {
|
|
2653
|
+
if (!isTreeSitterSupported(opts.lang)) {
|
|
2654
|
+
throw new Error(`tree-sitter: no grammar registered for lang "${opts.lang}"`);
|
|
2655
|
+
}
|
|
2656
|
+
const { Parser } = await getRuntime();
|
|
2657
|
+
const cached = await loadLanguage(opts.lang);
|
|
2658
|
+
const parser = new Parser();
|
|
2659
|
+
parser.setLanguage(cached.Language);
|
|
2660
|
+
let tree = null;
|
|
2661
|
+
try {
|
|
2662
|
+
tree = parser.parse(opts.content);
|
|
2663
|
+
if (!tree) throw new Error("tree-sitter: parser.parse returned null");
|
|
2664
|
+
return tree.rootNode.type;
|
|
2665
|
+
} finally {
|
|
2666
|
+
tree?.delete();
|
|
2667
|
+
parser.delete();
|
|
2668
|
+
}
|
|
2669
|
+
}
|
|
2670
|
+
var WASM_DIR, RUNTIME_WASM, LANG_TO_GRAMMAR, GO_OPT_IN, PY_OPT_IN, RS_OPT_IN, runtimePromise, languageCache;
|
|
2671
|
+
var init_tree_sitter_parser = __esm({
|
|
2672
|
+
"src/codebase-index/tree-sitter-parser.ts"() {
|
|
2673
|
+
"use strict";
|
|
2674
|
+
init_queries();
|
|
2675
|
+
init_visitor();
|
|
2676
|
+
WASM_DIR = fileURLToPath(new URL("./wasm/", import.meta.url));
|
|
2677
|
+
RUNTIME_WASM = path9.join(WASM_DIR, "tree-sitter-runtime.wasm");
|
|
2678
|
+
LANG_TO_GRAMMAR = {
|
|
2679
|
+
c: "c",
|
|
2680
|
+
cpp: "cpp",
|
|
2681
|
+
java: "java",
|
|
2682
|
+
csharp: "c_sharp",
|
|
2683
|
+
// tree-sitter directory uses underscore
|
|
2684
|
+
php: "php",
|
|
2685
|
+
ruby: "ruby",
|
|
2686
|
+
swift: "swift",
|
|
2687
|
+
kotlin: "kotlin",
|
|
2688
|
+
shell: "bash",
|
|
2689
|
+
// we treat `.sh` / `.bash` / `.zsh` via the bash grammar
|
|
2690
|
+
// Go/Python/Rust are opt-in only (see goOptIn / pyOptIn / rsOptIn below)
|
|
2691
|
+
elixir: "elixir"
|
|
2692
|
+
};
|
|
2693
|
+
GO_OPT_IN = "WRONGSTACK_USE_TS_GO";
|
|
2694
|
+
PY_OPT_IN = "WRONGSTACK_USE_TS_PY";
|
|
2695
|
+
RS_OPT_IN = "WRONGSTACK_USE_TS_RS";
|
|
2696
|
+
runtimePromise = null;
|
|
2697
|
+
languageCache = /* @__PURE__ */ new Map();
|
|
2698
|
+
}
|
|
2699
|
+
});
|
|
2700
|
+
|
|
2130
2701
|
// src/codebase-index/worker.ts
|
|
2131
2702
|
import { parentPort } from "node:worker_threads";
|
|
2132
2703
|
|
|
2133
2704
|
// src/codebase-index/indexer.ts
|
|
2134
2705
|
import { expectDefined as expectDefined5 } from "@wrongstack/core/utils";
|
|
2135
2706
|
import { execFile } from "node:child_process";
|
|
2136
|
-
import * as
|
|
2707
|
+
import * as fs9 from "node:fs/promises";
|
|
2137
2708
|
import { availableParallelism } from "node:os";
|
|
2138
|
-
import * as
|
|
2709
|
+
import * as path12 from "node:path";
|
|
2139
2710
|
import {
|
|
2140
2711
|
DEFAULT_WALK_IGNORE_DIRS,
|
|
2141
2712
|
indexParallelBatchSize,
|
|
@@ -2937,32 +3508,56 @@ async function dispatch(file, content, lang) {
|
|
|
2937
3508
|
case "tsx":
|
|
2938
3509
|
case "js":
|
|
2939
3510
|
case "jsx": {
|
|
2940
|
-
const { parseSymbols:
|
|
2941
|
-
return
|
|
3511
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_ts_parser(), ts_parser_exports));
|
|
3512
|
+
return parseSymbols9({ file, content, lang });
|
|
2942
3513
|
}
|
|
2943
3514
|
case "go": {
|
|
2944
|
-
const { parseSymbols:
|
|
2945
|
-
return
|
|
3515
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_go_parser(), go_parser_exports));
|
|
3516
|
+
return parseSymbols9({ file, content, lang: "go" });
|
|
2946
3517
|
}
|
|
2947
3518
|
case "py": {
|
|
2948
|
-
const { parseSymbols:
|
|
2949
|
-
return
|
|
3519
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_py_parser(), py_parser_exports));
|
|
3520
|
+
return parseSymbols9({ file, content, lang: "py" });
|
|
2950
3521
|
}
|
|
2951
3522
|
case "rs": {
|
|
2952
|
-
const { parseSymbols:
|
|
2953
|
-
return
|
|
3523
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_rs_parser(), rs_parser_exports));
|
|
3524
|
+
return parseSymbols9({ file, content, lang: "rs" });
|
|
2954
3525
|
}
|
|
2955
3526
|
case "json": {
|
|
2956
|
-
const { parseSymbols:
|
|
2957
|
-
return
|
|
3527
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_json_parser(), json_parser_exports));
|
|
3528
|
+
return parseSymbols9({ file, content, lang: "json" });
|
|
2958
3529
|
}
|
|
2959
3530
|
case "yaml": {
|
|
2960
|
-
const { parseSymbols:
|
|
2961
|
-
return
|
|
3531
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_yaml_parser(), yaml_parser_exports));
|
|
3532
|
+
return parseSymbols9({ file, content, lang: "yaml" });
|
|
3533
|
+
}
|
|
3534
|
+
// Phase 1: ten languages now route through the Tree-Sitter WASM
|
|
3535
|
+
// universal parser (`tree-sitter-parser.ts`). The dispatch falls back to
|
|
3536
|
+
// the regex extractor in `generic-parser.ts` whenever WASM loading fails
|
|
3537
|
+
// or the parser returns zero symbols — preserving the indexable-file
|
|
3538
|
+
// contract that "missing a parser must never mean skipping the file".
|
|
3539
|
+
case "c":
|
|
3540
|
+
case "cpp":
|
|
3541
|
+
case "java":
|
|
3542
|
+
case "csharp":
|
|
3543
|
+
case "php":
|
|
3544
|
+
case "ruby":
|
|
3545
|
+
case "swift":
|
|
3546
|
+
case "kotlin":
|
|
3547
|
+
case "shell":
|
|
3548
|
+
case "elixir": {
|
|
3549
|
+
try {
|
|
3550
|
+
const { parseSymbols: parseSymbols10 } = await Promise.resolve().then(() => (init_tree_sitter_parser(), tree_sitter_parser_exports));
|
|
3551
|
+
const parsed = await parseSymbols10({ file, content, lang });
|
|
3552
|
+
if (parsed.symbols.length > 0) return parsed;
|
|
3553
|
+
} catch {
|
|
3554
|
+
}
|
|
3555
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_generic_parser(), generic_parser_exports));
|
|
3556
|
+
return parseSymbols9({ file, content, lang });
|
|
2962
3557
|
}
|
|
2963
3558
|
default: {
|
|
2964
|
-
const { parseSymbols:
|
|
2965
|
-
return
|
|
3559
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_generic_parser(), generic_parser_exports));
|
|
3560
|
+
return parseSymbols9({ file, content, lang });
|
|
2966
3561
|
}
|
|
2967
3562
|
}
|
|
2968
3563
|
}
|
|
@@ -2974,10 +3569,265 @@ function withRelations(parsed, content, lang) {
|
|
|
2974
3569
|
return { ...parsed, refs: refs.map((ref) => ref.lang ? ref : { ...ref, lang }) };
|
|
2975
3570
|
}
|
|
2976
3571
|
|
|
3572
|
+
// src/codebase-index/parser-worker-pool.ts
|
|
3573
|
+
import { Worker } from "node:worker_threads";
|
|
3574
|
+
import { fileURLToPath as fileURLToPath2 } from "node:url";
|
|
3575
|
+
import * as fs6 from "node:fs";
|
|
3576
|
+
var WORKER_POOL_THRESHOLD = 500;
|
|
3577
|
+
var ParserWorkerPool = class {
|
|
3578
|
+
constructor(maxWorkers = defaultWorkerCount()) {
|
|
3579
|
+
this.maxWorkers = maxWorkers;
|
|
3580
|
+
}
|
|
3581
|
+
maxWorkers;
|
|
3582
|
+
workers = [];
|
|
3583
|
+
nextBatchId = 1;
|
|
3584
|
+
pending = /* @__PURE__ */ new Map();
|
|
3585
|
+
creating = false;
|
|
3586
|
+
unavailable = false;
|
|
3587
|
+
/**
|
|
3588
|
+
* True if the pool is available for use. Returns false when:
|
|
3589
|
+
* - Worker threads aren't supported (sandbox, exotic runtime)
|
|
3590
|
+
* - The built worker script can't be found
|
|
3591
|
+
* - Pool creation was attempted and failed
|
|
3592
|
+
*/
|
|
3593
|
+
isAvailable() {
|
|
3594
|
+
return !this.unavailable && this.workers.length > 0;
|
|
3595
|
+
}
|
|
3596
|
+
/**
|
|
3597
|
+
* Lazily create the worker pool. Returns true if the pool is ready, false
|
|
3598
|
+
* if it's unavailable (caller should fall back to inline parsing).
|
|
3599
|
+
*/
|
|
3600
|
+
async ensureReady() {
|
|
3601
|
+
if (this.isAvailable()) return true;
|
|
3602
|
+
if (this.unavailable) return false;
|
|
3603
|
+
if (this.creating) {
|
|
3604
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
3605
|
+
return this.isAvailable();
|
|
3606
|
+
}
|
|
3607
|
+
this.creating = true;
|
|
3608
|
+
try {
|
|
3609
|
+
const url = resolveWorkerScriptUrl();
|
|
3610
|
+
if (!url) {
|
|
3611
|
+
this.unavailable = true;
|
|
3612
|
+
return false;
|
|
3613
|
+
}
|
|
3614
|
+
for (let i = 0; i < this.maxWorkers; i++) {
|
|
3615
|
+
try {
|
|
3616
|
+
const w = new Worker(url, { name: `wstack-parser-${i}` });
|
|
3617
|
+
w.unref();
|
|
3618
|
+
w.on("message", (msg) => this.handleMessage(msg));
|
|
3619
|
+
w.on("error", (err) => this.handleError(err, w));
|
|
3620
|
+
this.workers.push({ worker: w, busy: false });
|
|
3621
|
+
} catch {
|
|
3622
|
+
if (this.workers.length === 0) {
|
|
3623
|
+
this.unavailable = true;
|
|
3624
|
+
return false;
|
|
3625
|
+
}
|
|
3626
|
+
break;
|
|
3627
|
+
}
|
|
3628
|
+
}
|
|
3629
|
+
return this.workers.length > 0;
|
|
3630
|
+
} finally {
|
|
3631
|
+
this.creating = false;
|
|
3632
|
+
}
|
|
3633
|
+
}
|
|
3634
|
+
/**
|
|
3635
|
+
* Parse files in parallel across the worker pool. Returns a flat
|
|
3636
|
+
* `FileSymbols[]` in completion order (caller sorts if needed).
|
|
3637
|
+
*
|
|
3638
|
+
* Content is pre-read by the main thread (for the content-hash check)
|
|
3639
|
+
* and passed to workers to avoid a second disk read. Files are
|
|
3640
|
+
* distributed round-robin across workers.
|
|
3641
|
+
*/
|
|
3642
|
+
async parseFiles(files) {
|
|
3643
|
+
if (!this.isAvailable()) {
|
|
3644
|
+
throw new Error("ParserWorkerPool.parseFiles called before ensureReady() succeeded");
|
|
3645
|
+
}
|
|
3646
|
+
if (files.length === 0) return [];
|
|
3647
|
+
const batchId = this.nextBatchId++;
|
|
3648
|
+
const workerCount = Math.min(this.workers.length, files.length);
|
|
3649
|
+
const chunks = Array.from(
|
|
3650
|
+
{ length: workerCount },
|
|
3651
|
+
() => []
|
|
3652
|
+
);
|
|
3653
|
+
for (let i = 0; i < files.length; i++) {
|
|
3654
|
+
chunks[i % workerCount].push(files[i]);
|
|
3655
|
+
}
|
|
3656
|
+
return new Promise((resolve2, reject) => {
|
|
3657
|
+
this.pending.set(batchId, {
|
|
3658
|
+
resolve: resolve2,
|
|
3659
|
+
reject,
|
|
3660
|
+
accumulated: [],
|
|
3661
|
+
expectedWorkers: workerCount,
|
|
3662
|
+
completedWorkers: 0
|
|
3663
|
+
});
|
|
3664
|
+
for (let i = 0; i < workerCount; i++) {
|
|
3665
|
+
const pw = this.workers[i];
|
|
3666
|
+
pw.busy = true;
|
|
3667
|
+
pw.worker.postMessage({
|
|
3668
|
+
type: "parse",
|
|
3669
|
+
id: batchId,
|
|
3670
|
+
files: chunks[i]
|
|
3671
|
+
});
|
|
3672
|
+
}
|
|
3673
|
+
});
|
|
3674
|
+
}
|
|
3675
|
+
/** Shut down all workers. Safe to call multiple times. */
|
|
3676
|
+
async shutdown() {
|
|
3677
|
+
const workers = this.workers.map((w) => w.worker);
|
|
3678
|
+
this.workers = [];
|
|
3679
|
+
this.unavailable = false;
|
|
3680
|
+
for (const w of workers) {
|
|
3681
|
+
try {
|
|
3682
|
+
w.postMessage({ type: "shutdown" });
|
|
3683
|
+
} catch {
|
|
3684
|
+
}
|
|
3685
|
+
}
|
|
3686
|
+
await Promise.allSettled(
|
|
3687
|
+
workers.map(
|
|
3688
|
+
(w) => Promise.race([
|
|
3689
|
+
new Promise((resolve2) => {
|
|
3690
|
+
w.once("exit", () => resolve2());
|
|
3691
|
+
}),
|
|
3692
|
+
new Promise((resolve2) => setTimeout(() => resolve2(), 2e3))
|
|
3693
|
+
]).then(() => {
|
|
3694
|
+
if (!w.threadId) return;
|
|
3695
|
+
return w.terminate().catch(() => {
|
|
3696
|
+
});
|
|
3697
|
+
})
|
|
3698
|
+
)
|
|
3699
|
+
);
|
|
3700
|
+
for (const [, p] of this.pending) p.reject(new Error("ParserWorkerPool shut down"));
|
|
3701
|
+
this.pending.clear();
|
|
3702
|
+
}
|
|
3703
|
+
handleMessage(msg) {
|
|
3704
|
+
const batch = this.pending.get(msg.id);
|
|
3705
|
+
if (!batch) return;
|
|
3706
|
+
batch.accumulated.push(...msg.results);
|
|
3707
|
+
batch.completedWorkers++;
|
|
3708
|
+
const freeWorker = this.workers.find((w) => w.busy);
|
|
3709
|
+
if (freeWorker) freeWorker.busy = false;
|
|
3710
|
+
if (batch.completedWorkers >= batch.expectedWorkers) {
|
|
3711
|
+
this.pending.delete(msg.id);
|
|
3712
|
+
batch.resolve(batch.accumulated);
|
|
3713
|
+
}
|
|
3714
|
+
}
|
|
3715
|
+
handleError(err, source) {
|
|
3716
|
+
this.workers = this.workers.filter((w) => w.worker !== source);
|
|
3717
|
+
if (this.workers.length === 0) {
|
|
3718
|
+
for (const [, p] of this.pending) p.reject(err);
|
|
3719
|
+
this.pending.clear();
|
|
3720
|
+
this.unavailable = true;
|
|
3721
|
+
}
|
|
3722
|
+
}
|
|
3723
|
+
};
|
|
3724
|
+
function defaultWorkerCount() {
|
|
3725
|
+
const cores = globalThis.navigator?.hardwareConcurrency ?? 4;
|
|
3726
|
+
return Math.max(1, Math.min(4, cores - 1));
|
|
3727
|
+
}
|
|
3728
|
+
function resolveWorkerScriptUrl() {
|
|
3729
|
+
for (const rel of [
|
|
3730
|
+
"./parser-worker-script.js",
|
|
3731
|
+
"./codebase-index/parser-worker-script.js"
|
|
3732
|
+
]) {
|
|
3733
|
+
try {
|
|
3734
|
+
const url = new URL(rel, import.meta.url);
|
|
3735
|
+
if (url.protocol === "file:" && fs6.existsSync(fileURLToPath2(url))) return url;
|
|
3736
|
+
} catch {
|
|
3737
|
+
}
|
|
3738
|
+
}
|
|
3739
|
+
return null;
|
|
3740
|
+
}
|
|
3741
|
+
var _pool = null;
|
|
3742
|
+
function getParserPool() {
|
|
3743
|
+
_pool ??= new ParserWorkerPool();
|
|
3744
|
+
return _pool;
|
|
3745
|
+
}
|
|
3746
|
+
|
|
3747
|
+
// src/codebase-index/content-hash.ts
|
|
3748
|
+
var PRIME64_1 = 0x9e3779b185ebca87n;
|
|
3749
|
+
var PRIME64_2 = 0xc2b2ae3d27d4eb4fn;
|
|
3750
|
+
var PRIME64_3 = 0x165667b19e3779f9n;
|
|
3751
|
+
var PRIME64_4 = 0x85ebca77c2b2ae63n;
|
|
3752
|
+
var PRIME64_5 = 0x27d4eb2f165667c5n;
|
|
3753
|
+
var MASK64 = 0xffffffffffffffffn;
|
|
3754
|
+
function mul64(a, b) {
|
|
3755
|
+
return (a & MASK64) * (b & MASK64) & MASK64;
|
|
3756
|
+
}
|
|
3757
|
+
function rotl64(x, n) {
|
|
3758
|
+
const v = x & MASK64;
|
|
3759
|
+
return (v << BigInt(n) | v >> BigInt(64 - n)) & MASK64;
|
|
3760
|
+
}
|
|
3761
|
+
function readU64LE(buf, off) {
|
|
3762
|
+
let v = 0n;
|
|
3763
|
+
for (let i = 7; i >= 0; i--) {
|
|
3764
|
+
v = v << 8n | BigInt(buf[off + i] ?? 0);
|
|
3765
|
+
}
|
|
3766
|
+
return v & MASK64;
|
|
3767
|
+
}
|
|
3768
|
+
function readU32LE(buf, off) {
|
|
3769
|
+
return (BigInt(buf[off] ?? 0) | BigInt(buf[off + 1] ?? 0) << 8n | BigInt(buf[off + 2] ?? 0) << 16n | BigInt(buf[off + 3] ?? 0) << 24n) & MASK64;
|
|
3770
|
+
}
|
|
3771
|
+
function xxh64Round(acc, lane) {
|
|
3772
|
+
return mul64(rotl64(acc + mul64(lane, PRIME64_2) & MASK64, 31), PRIME64_1);
|
|
3773
|
+
}
|
|
3774
|
+
function xxh64MergeRound(acc, val) {
|
|
3775
|
+
return mul64(acc ^ xxh64Round(0n, val), PRIME64_1) + PRIME64_4 & MASK64;
|
|
3776
|
+
}
|
|
3777
|
+
function xxhash64Hex(buf, explicitLen) {
|
|
3778
|
+
const length = explicitLen ?? buf.length;
|
|
3779
|
+
let h;
|
|
3780
|
+
let off = 0;
|
|
3781
|
+
if (length >= 32) {
|
|
3782
|
+
let v1 = PRIME64_1 + PRIME64_2 & MASK64;
|
|
3783
|
+
let v2 = PRIME64_2;
|
|
3784
|
+
let v3 = 0n;
|
|
3785
|
+
let v4 = 0n - PRIME64_1 & MASK64;
|
|
3786
|
+
const end32 = length - 32;
|
|
3787
|
+
while (off <= end32) {
|
|
3788
|
+
v1 = xxh64Round(v1, readU64LE(buf, off));
|
|
3789
|
+
v2 = xxh64Round(v2, readU64LE(buf, off + 8));
|
|
3790
|
+
v3 = xxh64Round(v3, readU64LE(buf, off + 16));
|
|
3791
|
+
v4 = xxh64Round(v4, readU64LE(buf, off + 24));
|
|
3792
|
+
off += 32;
|
|
3793
|
+
}
|
|
3794
|
+
h = rotl64(v1, 1) + rotl64(v2, 7) + rotl64(v3, 12) + rotl64(v4, 18) & MASK64;
|
|
3795
|
+
h = xxh64MergeRound(h, v1);
|
|
3796
|
+
h = xxh64MergeRound(h, v2);
|
|
3797
|
+
h = xxh64MergeRound(h, v3);
|
|
3798
|
+
h = xxh64MergeRound(h, v4);
|
|
3799
|
+
} else {
|
|
3800
|
+
h = PRIME64_5;
|
|
3801
|
+
}
|
|
3802
|
+
h = h + BigInt(length) & MASK64;
|
|
3803
|
+
while (off + 8 <= length) {
|
|
3804
|
+
const k1 = xxh64Round(0n, readU64LE(buf, off));
|
|
3805
|
+
h = mul64(rotl64(h ^ k1, 27), PRIME64_1) + PRIME64_4 & MASK64;
|
|
3806
|
+
off += 8;
|
|
3807
|
+
}
|
|
3808
|
+
if (off + 4 <= length) {
|
|
3809
|
+
h = mul64(rotl64(h ^ mul64(readU32LE(buf, off), PRIME64_1), 23), PRIME64_2) + PRIME64_3 & MASK64;
|
|
3810
|
+
off += 4;
|
|
3811
|
+
}
|
|
3812
|
+
while (off < length) {
|
|
3813
|
+
h = mul64(rotl64(h ^ mul64(BigInt(buf[off] ?? 0), PRIME64_5), 11), PRIME64_1) & MASK64;
|
|
3814
|
+
off += 1;
|
|
3815
|
+
}
|
|
3816
|
+
h = (h ^ h >> 33n) & MASK64;
|
|
3817
|
+
h = mul64(h, PRIME64_2);
|
|
3818
|
+
h = (h ^ h >> 29n) & MASK64;
|
|
3819
|
+
h = mul64(h, PRIME64_3);
|
|
3820
|
+
h = (h ^ h >> 32n) & MASK64;
|
|
3821
|
+
return h.toString(16).padStart(16, "0");
|
|
3822
|
+
}
|
|
3823
|
+
function xxhash64String(content) {
|
|
3824
|
+
return xxhash64Hex(new TextEncoder().encode(content));
|
|
3825
|
+
}
|
|
3826
|
+
|
|
2977
3827
|
// src/codebase-index/writer.ts
|
|
2978
3828
|
import { expectDefined as expectDefined4 } from "@wrongstack/core/utils";
|
|
2979
|
-
import * as
|
|
2980
|
-
import * as
|
|
3829
|
+
import * as fs8 from "node:fs";
|
|
3830
|
+
import * as path11 from "node:path";
|
|
2981
3831
|
|
|
2982
3832
|
// src/codebase-index/bm25.ts
|
|
2983
3833
|
var K1 = 1.5;
|
|
@@ -3251,8 +4101,8 @@ function runSqliteWithRetry(fn) {
|
|
|
3251
4101
|
}
|
|
3252
4102
|
|
|
3253
4103
|
// src/codebase-index/writer-admin.ts
|
|
3254
|
-
import * as
|
|
3255
|
-
import * as
|
|
4104
|
+
import * as fs7 from "node:fs";
|
|
4105
|
+
import * as path10 from "node:path";
|
|
3256
4106
|
var DB_FILE = "index.db";
|
|
3257
4107
|
function getAllIndexableWithStatement(stmt) {
|
|
3258
4108
|
return stmt("SELECT id, text FROM symbols").all().map(({ id, text }) => ({ id, text }));
|
|
@@ -3288,7 +4138,7 @@ function getMetadataWithStatement(stmt, key) {
|
|
|
3288
4138
|
}
|
|
3289
4139
|
function getFileMetaWithStatement(stmt, file) {
|
|
3290
4140
|
const rows = stmt(
|
|
3291
|
-
"SELECT file, lang, mtime_ms, symbol_count, last_indexed FROM files WHERE file = ?"
|
|
4141
|
+
"SELECT file, lang, mtime_ms, symbol_count, last_indexed, content_hash FROM files WHERE file = ?"
|
|
3292
4142
|
).all(file);
|
|
3293
4143
|
const r = rows[0];
|
|
3294
4144
|
if (!r) return null;
|
|
@@ -3297,21 +4147,25 @@ function getFileMetaWithStatement(stmt, file) {
|
|
|
3297
4147
|
lang: r.lang,
|
|
3298
4148
|
mtimeMs: r.mtime_ms,
|
|
3299
4149
|
symbolCount: r.symbol_count,
|
|
3300
|
-
lastIndexed: r.last_indexed
|
|
4150
|
+
lastIndexed: r.last_indexed,
|
|
4151
|
+
contentHash: r.content_hash
|
|
3301
4152
|
};
|
|
3302
4153
|
}
|
|
3303
4154
|
function getAllFileMetasWithStatement(stmt) {
|
|
3304
|
-
return stmt(
|
|
4155
|
+
return stmt(
|
|
4156
|
+
"SELECT file, lang, mtime_ms, symbol_count, last_indexed, content_hash FROM files"
|
|
4157
|
+
).all().map((r) => ({
|
|
3305
4158
|
file: r.file,
|
|
3306
4159
|
lang: r.lang,
|
|
3307
4160
|
mtimeMs: r.mtime_ms,
|
|
3308
4161
|
symbolCount: r.symbol_count,
|
|
3309
|
-
lastIndexed: r.last_indexed
|
|
4162
|
+
lastIndexed: r.last_indexed,
|
|
4163
|
+
contentHash: r.content_hash
|
|
3310
4164
|
}));
|
|
3311
4165
|
}
|
|
3312
4166
|
function getIndexDbSizeBytes(indexDir) {
|
|
3313
4167
|
try {
|
|
3314
|
-
return
|
|
4168
|
+
return fs7.statSync(path10.join(indexDir, DB_FILE)).size;
|
|
3315
4169
|
} catch {
|
|
3316
4170
|
return 0;
|
|
3317
4171
|
}
|
|
@@ -3360,6 +4214,18 @@ function bulkInsertFtsWithStatement(stmt, maxSqlVars, ftsAvailable, rows) {
|
|
|
3360
4214
|
insert.run(...binds);
|
|
3361
4215
|
}
|
|
3362
4216
|
}
|
|
4217
|
+
function bulkInsertVectorsWithStatement(stmt, maxSqlVars, rows) {
|
|
4218
|
+
if (rows.length === 0) return;
|
|
4219
|
+
const chunkSize = Math.max(1, Math.floor(maxSqlVars / 2));
|
|
4220
|
+
for (let i = 0; i < rows.length; i += chunkSize) {
|
|
4221
|
+
const chunk = rows.slice(i, i + chunkSize);
|
|
4222
|
+
const placeholders = chunk.map(() => "(?, ?)").join(", ");
|
|
4223
|
+
const insert = stmt(`INSERT INTO symbol_vectors(symbol_id, vector) VALUES ${placeholders}`);
|
|
4224
|
+
const binds = [];
|
|
4225
|
+
for (const r of chunk) binds.push(r.id, r.vector);
|
|
4226
|
+
insert.run(...binds);
|
|
4227
|
+
}
|
|
4228
|
+
}
|
|
3363
4229
|
function bulkInsertRefsWithStatement(stmt, maxSqlVars, refs) {
|
|
3364
4230
|
if (refs.length === 0) return;
|
|
3365
4231
|
const chunkSize = Math.max(1, Math.floor(maxSqlVars / 8));
|
|
@@ -3664,6 +4530,171 @@ function findOutgoingCallsByName(stmt, symbolName, file, limit) {
|
|
|
3664
4530
|
const calls = rows.map(mapCallSiteRow).slice(0, limit);
|
|
3665
4531
|
return { calls, symbolFound: true, unresolvedCount, totalMatches: rows.length };
|
|
3666
4532
|
}
|
|
4533
|
+
function runCteWithSeeds(stmt, seedIds, buildSql) {
|
|
4534
|
+
if (seedIds.length <= 900) {
|
|
4535
|
+
const ph = seedIds.map(() => "?").join(",");
|
|
4536
|
+
return stmt(buildSql(ph)).all(...seedIds);
|
|
4537
|
+
}
|
|
4538
|
+
stmt("DROP TABLE IF EXISTS _cte_seeds").run();
|
|
4539
|
+
try {
|
|
4540
|
+
stmt("CREATE TEMP TABLE _cte_seeds (id INTEGER PRIMARY KEY)").run();
|
|
4541
|
+
for (let i = 0; i < seedIds.length; i += 500) {
|
|
4542
|
+
const chunk = seedIds.slice(i, i + 500);
|
|
4543
|
+
const ph = chunk.map(() => "(?)").join(",");
|
|
4544
|
+
stmt(`INSERT OR IGNORE INTO _cte_seeds (id) VALUES ${ph}`).run(...chunk);
|
|
4545
|
+
}
|
|
4546
|
+
return stmt(buildSql("SELECT id FROM _cte_seeds")).all();
|
|
4547
|
+
} finally {
|
|
4548
|
+
stmt("DROP TABLE IF EXISTS _cte_seeds").run();
|
|
4549
|
+
}
|
|
4550
|
+
}
|
|
4551
|
+
function findTransitiveIncomingCallsByName(stmt, symbolName, file, limit) {
|
|
4552
|
+
const targetIds = resolveSymbolIds(stmt, symbolName, file);
|
|
4553
|
+
if (targetIds.length === 0)
|
|
4554
|
+
return { calls: [], symbolFound: false, ambiguous: false, totalMatches: 0 };
|
|
4555
|
+
let matchIds = targetIds;
|
|
4556
|
+
let ambiguous = false;
|
|
4557
|
+
if (file !== void 0) {
|
|
4558
|
+
const allNamedIds = resolveSymbolIds(stmt, symbolName, void 0);
|
|
4559
|
+
if (allNamedIds.length > targetIds.length) {
|
|
4560
|
+
matchIds = allNamedIds;
|
|
4561
|
+
ambiguous = true;
|
|
4562
|
+
}
|
|
4563
|
+
}
|
|
4564
|
+
const cteSql = (seedSource) => `WITH RECURSIVE incoming_tree(from_id) AS (
|
|
4565
|
+
SELECT r.from_id
|
|
4566
|
+
FROM refs r
|
|
4567
|
+
WHERE r.to_id IN (${seedSource})
|
|
4568
|
+
|
|
4569
|
+
UNION
|
|
4570
|
+
|
|
4571
|
+
SELECT r.from_id
|
|
4572
|
+
FROM refs r
|
|
4573
|
+
JOIN incoming_tree it ON r.to_id = it.from_id
|
|
4574
|
+
)
|
|
4575
|
+
SELECT
|
|
4576
|
+
s.id AS sym_id,
|
|
4577
|
+
s.name AS sym_name,
|
|
4578
|
+
s.kind AS sym_kind,
|
|
4579
|
+
s.lang AS sym_lang,
|
|
4580
|
+
s.file AS sym_file,
|
|
4581
|
+
s.line AS sym_line,
|
|
4582
|
+
s.signature AS sym_signature,
|
|
4583
|
+
'' AS call_type,
|
|
4584
|
+
0 AS ref_line
|
|
4585
|
+
FROM incoming_tree it
|
|
4586
|
+
JOIN symbols s ON s.id = it.from_id
|
|
4587
|
+
GROUP BY s.id
|
|
4588
|
+
ORDER BY s.file, s.line`;
|
|
4589
|
+
const rows = runCteWithSeeds(stmt, matchIds, cteSql);
|
|
4590
|
+
if (!file) {
|
|
4591
|
+
const fallbackRows = stmt(
|
|
4592
|
+
`SELECT
|
|
4593
|
+
s.id AS sym_id,
|
|
4594
|
+
s.name AS sym_name,
|
|
4595
|
+
s.kind AS sym_kind,
|
|
4596
|
+
s.lang AS sym_lang,
|
|
4597
|
+
s.file AS sym_file,
|
|
4598
|
+
s.line AS sym_line,
|
|
4599
|
+
s.signature AS sym_signature,
|
|
4600
|
+
r.call_type,
|
|
4601
|
+
r.line AS ref_line
|
|
4602
|
+
FROM refs r
|
|
4603
|
+
JOIN symbols s ON s.id = r.from_id
|
|
4604
|
+
WHERE r.to_id IS NULL AND r.to_name = ?
|
|
4605
|
+
ORDER BY r.line, r.id`
|
|
4606
|
+
).all(symbolName);
|
|
4607
|
+
rows.push(...fallbackRows);
|
|
4608
|
+
}
|
|
4609
|
+
rows.sort((a, b) => a.ref_line - b.ref_line || a.sym_id - b.sym_id);
|
|
4610
|
+
const allCalls = rows.map(mapCallSiteRow);
|
|
4611
|
+
return {
|
|
4612
|
+
calls: allCalls.slice(0, limit),
|
|
4613
|
+
symbolFound: true,
|
|
4614
|
+
ambiguous,
|
|
4615
|
+
totalMatches: allCalls.length
|
|
4616
|
+
};
|
|
4617
|
+
}
|
|
4618
|
+
function findTransitiveOutgoingCallsByName(stmt, symbolName, file, limit) {
|
|
4619
|
+
const sourceIds = resolveSymbolIds(stmt, symbolName, file);
|
|
4620
|
+
if (sourceIds.length === 0)
|
|
4621
|
+
return { calls: [], symbolFound: false, unresolvedCount: 0, totalMatches: 0 };
|
|
4622
|
+
const unresolvedCount = chunkedIdScalar(
|
|
4623
|
+
stmt,
|
|
4624
|
+
sourceIds,
|
|
4625
|
+
(ph) => `SELECT COUNT(*) AS n FROM refs WHERE from_id IN (${ph}) AND to_id IS NULL`
|
|
4626
|
+
);
|
|
4627
|
+
const cteSql = (seedSource) => `WITH RECURSIVE outgoing_tree(to_id) AS (
|
|
4628
|
+
SELECT r.to_id
|
|
4629
|
+
FROM refs r
|
|
4630
|
+
WHERE r.from_id IN (${seedSource}) AND r.to_id IS NOT NULL
|
|
4631
|
+
|
|
4632
|
+
UNION
|
|
4633
|
+
|
|
4634
|
+
SELECT r.to_id
|
|
4635
|
+
FROM refs r
|
|
4636
|
+
JOIN outgoing_tree ot ON r.from_id = ot.to_id
|
|
4637
|
+
WHERE r.to_id IS NOT NULL
|
|
4638
|
+
)
|
|
4639
|
+
SELECT
|
|
4640
|
+
s.id AS sym_id,
|
|
4641
|
+
s.name AS sym_name,
|
|
4642
|
+
s.kind AS sym_kind,
|
|
4643
|
+
s.lang AS sym_lang,
|
|
4644
|
+
s.file AS sym_file,
|
|
4645
|
+
s.line AS sym_line,
|
|
4646
|
+
s.signature AS sym_signature,
|
|
4647
|
+
'' AS call_type,
|
|
4648
|
+
0 AS ref_line
|
|
4649
|
+
FROM outgoing_tree ot
|
|
4650
|
+
JOIN symbols s ON s.id = ot.to_id
|
|
4651
|
+
GROUP BY s.id
|
|
4652
|
+
ORDER BY s.file, s.line`;
|
|
4653
|
+
const rows = runCteWithSeeds(stmt, sourceIds, cteSql);
|
|
4654
|
+
const calls = rows.map(mapCallSiteRow).slice(0, limit);
|
|
4655
|
+
return { calls, symbolFound: true, unresolvedCount, totalMatches: rows.length };
|
|
4656
|
+
}
|
|
4657
|
+
function findReachableSymbolIds(stmt, seedIds) {
|
|
4658
|
+
if (seedIds.length === 0) return /* @__PURE__ */ new Set();
|
|
4659
|
+
if (seedIds.length > 900) {
|
|
4660
|
+
stmt("DROP TABLE IF EXISTS _seeds").run();
|
|
4661
|
+
try {
|
|
4662
|
+
stmt("CREATE TEMP TABLE _seeds (id INTEGER PRIMARY KEY)").run();
|
|
4663
|
+
for (let i = 0; i < seedIds.length; i += 500) {
|
|
4664
|
+
const chunk = seedIds.slice(i, i + 500);
|
|
4665
|
+
const ph2 = chunk.map(() => "(?)").join(",");
|
|
4666
|
+
stmt(`INSERT OR IGNORE INTO _seeds (id) VALUES ${ph2}`).run(...chunk);
|
|
4667
|
+
}
|
|
4668
|
+
const rows2 = stmt(
|
|
4669
|
+
`WITH RECURSIVE reachable(id) AS (
|
|
4670
|
+
SELECT id FROM _seeds
|
|
4671
|
+
UNION
|
|
4672
|
+
SELECT r.to_id
|
|
4673
|
+
FROM refs r
|
|
4674
|
+
JOIN reachable ON r.from_id = reachable.id
|
|
4675
|
+
WHERE r.to_id IS NOT NULL
|
|
4676
|
+
)
|
|
4677
|
+
SELECT DISTINCT id FROM reachable`
|
|
4678
|
+
).all();
|
|
4679
|
+
return new Set(rows2.map((r) => r.id));
|
|
4680
|
+
} finally {
|
|
4681
|
+
stmt("DROP TABLE IF EXISTS _seeds").run();
|
|
4682
|
+
}
|
|
4683
|
+
}
|
|
4684
|
+
const ph = seedIds.map(() => "?").join(",");
|
|
4685
|
+
const rows = stmt(
|
|
4686
|
+
`WITH RECURSIVE reachable(id) AS (
|
|
4687
|
+
SELECT id FROM symbols WHERE id IN (${ph})
|
|
4688
|
+
UNION
|
|
4689
|
+
SELECT r.to_id
|
|
4690
|
+
FROM refs r
|
|
4691
|
+
JOIN reachable ON r.from_id = reachable.id
|
|
4692
|
+
WHERE r.to_id IS NOT NULL
|
|
4693
|
+
)
|
|
4694
|
+
SELECT DISTINCT id FROM reachable`
|
|
4695
|
+
).all(...seedIds);
|
|
4696
|
+
return new Set(rows.map((r) => r.id));
|
|
4697
|
+
}
|
|
3667
4698
|
function findRefsToWithStatement(stmt, symbolId) {
|
|
3668
4699
|
return stmt(
|
|
3669
4700
|
"SELECT id, from_id, to_name, to_id, call_type, line FROM refs WHERE to_id = ? OR to_name = (SELECT name FROM symbols WHERE id = ?)"
|
|
@@ -3903,6 +4934,12 @@ var CORE_TABLES_SQL = `
|
|
|
3903
4934
|
file TEXT PRIMARY KEY,
|
|
3904
4935
|
lang TEXT NOT NULL,
|
|
3905
4936
|
mtime_ms INTEGER NOT NULL,
|
|
4937
|
+
-- Phase 2: xxHash64 of the file's UTF-8 bytes. Empty string when the
|
|
4938
|
+
-- indexer hasn't populated it yet (legacy rows, schema repaired by
|
|
4939
|
+
-- repairMissingColumns). Compared on incremental re-index so that a
|
|
4940
|
+
-- touch or branch-switch that leaves content byte-identical skips the
|
|
4941
|
+
-- expensive parse phase entirely (refactoring proposal Phase 2).
|
|
4942
|
+
content_hash TEXT NOT NULL DEFAULT '',
|
|
3906
4943
|
symbol_count INTEGER NOT NULL DEFAULT 0,
|
|
3907
4944
|
last_indexed INTEGER NOT NULL,
|
|
3908
4945
|
-- Code Atlas grouping label, computed at index time from the ecosystem's
|
|
@@ -3970,7 +5007,14 @@ var LANG_FAMILY_TABLE_SQL = `
|
|
|
3970
5007
|
);
|
|
3971
5008
|
`;
|
|
3972
5009
|
var LANG_FAMILY_WILDCARD = "*";
|
|
3973
|
-
var SYMBOLS_FTS_SQL = "CREATE VIRTUAL TABLE IF NOT EXISTS symbols_fts USING fts5(text, tokenize = '
|
|
5010
|
+
var SYMBOLS_FTS_SQL = "CREATE VIRTUAL TABLE IF NOT EXISTS symbols_fts USING fts5(text, tokenize = 'trigram')";
|
|
5011
|
+
var SYMBOL_VECTORS_TABLE_SQL = `
|
|
5012
|
+
CREATE TABLE IF NOT EXISTS symbol_vectors (
|
|
5013
|
+
symbol_id INTEGER PRIMARY KEY,
|
|
5014
|
+
vector BLOB NOT NULL,
|
|
5015
|
+
FOREIGN KEY (symbol_id) REFERENCES symbols(id) ON DELETE CASCADE
|
|
5016
|
+
);
|
|
5017
|
+
`;
|
|
3974
5018
|
|
|
3975
5019
|
// src/codebase-index/writer-search-helpers.ts
|
|
3976
5020
|
var SEARCH_CANDIDATE_SCAN_CAP = 5e3;
|
|
@@ -4111,6 +5155,85 @@ var StorePool = class {
|
|
|
4111
5155
|
}
|
|
4112
5156
|
};
|
|
4113
5157
|
|
|
5158
|
+
// src/codebase-index/vector-search.ts
|
|
5159
|
+
var RRF_K = 60;
|
|
5160
|
+
var VECTOR_DIMENSIONS = 384;
|
|
5161
|
+
var NGRAM_SIZE = 3;
|
|
5162
|
+
function embedText(text) {
|
|
5163
|
+
const vec = new Float32Array(VECTOR_DIMENSIONS);
|
|
5164
|
+
const normalized = text.toLowerCase().trim();
|
|
5165
|
+
if (normalized.length < NGRAM_SIZE) {
|
|
5166
|
+
const padded = ` ${normalized} `.slice(0, Math.max(NGRAM_SIZE, normalized.length + 2));
|
|
5167
|
+
for (let i = 0; i <= padded.length - NGRAM_SIZE; i++) {
|
|
5168
|
+
const ngram = padded.slice(i, i + NGRAM_SIZE);
|
|
5169
|
+
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
5170
|
+
vec[bucket] += 1;
|
|
5171
|
+
}
|
|
5172
|
+
} else {
|
|
5173
|
+
for (let i = 0; i <= normalized.length - NGRAM_SIZE; i++) {
|
|
5174
|
+
const ngram = normalized.slice(i, i + NGRAM_SIZE);
|
|
5175
|
+
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
5176
|
+
vec[bucket] += 1;
|
|
5177
|
+
}
|
|
5178
|
+
}
|
|
5179
|
+
let norm = 0;
|
|
5180
|
+
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
5181
|
+
norm += vec[i] * vec[i];
|
|
5182
|
+
}
|
|
5183
|
+
norm = Math.sqrt(norm);
|
|
5184
|
+
if (norm > 0) {
|
|
5185
|
+
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
5186
|
+
vec[i] /= norm;
|
|
5187
|
+
}
|
|
5188
|
+
}
|
|
5189
|
+
return vec;
|
|
5190
|
+
}
|
|
5191
|
+
function hashNgram(str) {
|
|
5192
|
+
let hash = 2166136261;
|
|
5193
|
+
for (let i = 0; i < str.length; i++) {
|
|
5194
|
+
hash ^= str.charCodeAt(i);
|
|
5195
|
+
hash = Math.imul(hash, 16777619);
|
|
5196
|
+
}
|
|
5197
|
+
return hash >>> 0;
|
|
5198
|
+
}
|
|
5199
|
+
function cosineSimilarity(a, b) {
|
|
5200
|
+
let dot = 0;
|
|
5201
|
+
const len = Math.min(a.length, b.length);
|
|
5202
|
+
for (let i = 0; i < len; i++) {
|
|
5203
|
+
dot += a[i] * b[i];
|
|
5204
|
+
}
|
|
5205
|
+
return dot;
|
|
5206
|
+
}
|
|
5207
|
+
function encodeVector(vec) {
|
|
5208
|
+
return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
|
|
5209
|
+
}
|
|
5210
|
+
function decodeVector(buf) {
|
|
5211
|
+
const view = new DataView(
|
|
5212
|
+
buf.buffer,
|
|
5213
|
+
buf.byteOffset,
|
|
5214
|
+
buf.byteLength
|
|
5215
|
+
);
|
|
5216
|
+
const copy = new Float32Array(buf.byteLength / 4);
|
|
5217
|
+
for (let i = 0; i < copy.length; i++) {
|
|
5218
|
+
copy[i] = view.getFloat32(i * 4, true);
|
|
5219
|
+
}
|
|
5220
|
+
return copy;
|
|
5221
|
+
}
|
|
5222
|
+
function reciprocalRankFusion(bm25Ranks, vectorRanks, k = RRF_K) {
|
|
5223
|
+
const allIds = /* @__PURE__ */ new Set([...bm25Ranks.keys(), ...vectorRanks.keys()]);
|
|
5224
|
+
const scored = [];
|
|
5225
|
+
for (const id of allIds) {
|
|
5226
|
+
const bm25Rank = bm25Ranks.get(id);
|
|
5227
|
+
const vecRank = vectorRanks.get(id);
|
|
5228
|
+
let score = 0;
|
|
5229
|
+
if (bm25Rank !== void 0) score += 1 / (k + bm25Rank);
|
|
5230
|
+
if (vecRank !== void 0) score += 1 / (k + vecRank);
|
|
5231
|
+
scored.push([id, score]);
|
|
5232
|
+
}
|
|
5233
|
+
scored.sort((a, b) => b[1] - a[1]);
|
|
5234
|
+
return scored;
|
|
5235
|
+
}
|
|
5236
|
+
|
|
4114
5237
|
// src/codebase-index/writer.ts
|
|
4115
5238
|
var DB_FILE2 = "index.db";
|
|
4116
5239
|
var MAX_STATEMENT_CACHE = 128;
|
|
@@ -4123,6 +5246,12 @@ var IndexStore = class _IndexStore {
|
|
|
4123
5246
|
* When false, ranked search falls back to the LIKE + in-process BM25 path.
|
|
4124
5247
|
*/
|
|
4125
5248
|
ftsAvailable = false;
|
|
5249
|
+
/**
|
|
5250
|
+
* Phase 3: true when the `symbol_vectors` table was created successfully.
|
|
5251
|
+
* When false, hybrid search skips the vector pass and falls back to FTS5
|
|
5252
|
+
* (or LIKE) only.
|
|
5253
|
+
*/
|
|
5254
|
+
vectorsAvailable = false;
|
|
4126
5255
|
/**
|
|
4127
5256
|
* Cache of prepared statements keyed by their SQL text. `DatabaseSync`
|
|
4128
5257
|
* compiles SQL on every `.prepare()` call; for the fixed-SQL methods
|
|
@@ -4181,9 +5310,9 @@ var IndexStore = class _IndexStore {
|
|
|
4181
5310
|
}
|
|
4182
5311
|
constructor(projectRoot, opts = {}) {
|
|
4183
5312
|
this.indexDir = resolveIndexDir(projectRoot, opts.indexDir);
|
|
4184
|
-
|
|
5313
|
+
fs8.mkdirSync(this.indexDir, { recursive: true });
|
|
4185
5314
|
const Database = loadDatabaseSync();
|
|
4186
|
-
this.db = new Database(
|
|
5315
|
+
this.db = new Database(path11.join(this.indexDir, DB_FILE2));
|
|
4187
5316
|
applyIndexStorePragmas(this.db);
|
|
4188
5317
|
this.initSchema();
|
|
4189
5318
|
}
|
|
@@ -4221,7 +5350,13 @@ var IndexStore = class _IndexStore {
|
|
|
4221
5350
|
*/
|
|
4222
5351
|
repairMissingColumns() {
|
|
4223
5352
|
const expected = [
|
|
4224
|
-
{
|
|
5353
|
+
{
|
|
5354
|
+
table: "files",
|
|
5355
|
+
columns: [
|
|
5356
|
+
["package", "TEXT NOT NULL DEFAULT ''"],
|
|
5357
|
+
["content_hash", "TEXT NOT NULL DEFAULT ''"]
|
|
5358
|
+
]
|
|
5359
|
+
},
|
|
4225
5360
|
{
|
|
4226
5361
|
table: "refs",
|
|
4227
5362
|
columns: [
|
|
@@ -4253,6 +5388,7 @@ var IndexStore = class _IndexStore {
|
|
|
4253
5388
|
DROP TABLE IF EXISTS symbols;
|
|
4254
5389
|
DROP TABLE IF EXISTS files;
|
|
4255
5390
|
DROP TABLE IF EXISTS refs;
|
|
5391
|
+
DROP TABLE IF EXISTS symbol_vectors;
|
|
4256
5392
|
`);
|
|
4257
5393
|
this.db.exec("DROP TABLE IF EXISTS symbols_fts");
|
|
4258
5394
|
this.stmt("UPDATE metadata SET value = ? WHERE key = ?").run(
|
|
@@ -4274,6 +5410,12 @@ var IndexStore = class _IndexStore {
|
|
|
4274
5410
|
this.db.exec(LANG_FAMILY_TABLE_SQL);
|
|
4275
5411
|
this.seedLangFamilies();
|
|
4276
5412
|
try {
|
|
5413
|
+
const ftsSchema = this.stmt(
|
|
5414
|
+
"SELECT sql FROM sqlite_master WHERE type='table' AND name='symbols_fts'"
|
|
5415
|
+
).get();
|
|
5416
|
+
if (ftsSchema?.sql && ftsSchema.sql.includes("unicode61")) {
|
|
5417
|
+
this.db.exec("DROP TABLE IF EXISTS symbols_fts");
|
|
5418
|
+
}
|
|
4277
5419
|
this.db.exec(SYMBOLS_FTS_SQL);
|
|
4278
5420
|
this.ftsAvailable = true;
|
|
4279
5421
|
const symbolCount = Number(
|
|
@@ -4284,6 +5426,7 @@ var IndexStore = class _IndexStore {
|
|
|
4284
5426
|
);
|
|
4285
5427
|
if (symbolCount !== ftsCount) {
|
|
4286
5428
|
this.db.exec("DELETE FROM symbols_fts");
|
|
5429
|
+
if (this.vectorsAvailable) this.db.exec("DELETE FROM symbol_vectors");
|
|
4287
5430
|
const rows = this.stmt(
|
|
4288
5431
|
"SELECT id, name, signature, doc_comment FROM symbols ORDER BY id"
|
|
4289
5432
|
).all();
|
|
@@ -4301,6 +5444,12 @@ var IndexStore = class _IndexStore {
|
|
|
4301
5444
|
} catch {
|
|
4302
5445
|
this.ftsAvailable = false;
|
|
4303
5446
|
}
|
|
5447
|
+
try {
|
|
5448
|
+
this.db.exec(SYMBOL_VECTORS_TABLE_SQL);
|
|
5449
|
+
this.vectorsAvailable = true;
|
|
5450
|
+
} catch {
|
|
5451
|
+
this.vectorsAvailable = false;
|
|
5452
|
+
}
|
|
4304
5453
|
this.ensureNextSymbolIdSeeded();
|
|
4305
5454
|
}
|
|
4306
5455
|
// ─── ID allocation & bulk helpers ────────────────────────────────────────────
|
|
@@ -4410,6 +5559,7 @@ var IndexStore = class _IndexStore {
|
|
|
4410
5559
|
const result = [];
|
|
4411
5560
|
const bulk = [];
|
|
4412
5561
|
const ftsRows = [];
|
|
5562
|
+
const vectorRows = [];
|
|
4413
5563
|
for (const s of symbols) {
|
|
4414
5564
|
const id = nextId++;
|
|
4415
5565
|
bulk.push({
|
|
@@ -4428,6 +5578,10 @@ var IndexStore = class _IndexStore {
|
|
|
4428
5578
|
if (this.ftsAvailable) {
|
|
4429
5579
|
ftsRows.push({ id, text: buildIndexableText(s.name, s.signature, s.docComment) });
|
|
4430
5580
|
}
|
|
5581
|
+
vectorRows.push({
|
|
5582
|
+
id,
|
|
5583
|
+
vector: encodeVector(embedText(s.text || buildIndexableText(s.name, s.signature, s.docComment)))
|
|
5584
|
+
});
|
|
4431
5585
|
result.push({ ...s, id });
|
|
4432
5586
|
}
|
|
4433
5587
|
bulkInsertSymbolsWithStatement((sql) => this.stmt(sql), _IndexStore.MAX_SQL_VARS, bulk);
|
|
@@ -4437,6 +5591,13 @@ var IndexStore = class _IndexStore {
|
|
|
4437
5591
|
this.ftsAvailable,
|
|
4438
5592
|
ftsRows
|
|
4439
5593
|
);
|
|
5594
|
+
if (this.vectorsAvailable) {
|
|
5595
|
+
bulkInsertVectorsWithStatement(
|
|
5596
|
+
(sql) => this.stmt(sql),
|
|
5597
|
+
_IndexStore.MAX_SQL_VARS,
|
|
5598
|
+
vectorRows
|
|
5599
|
+
);
|
|
5600
|
+
}
|
|
4440
5601
|
this.db.exec("COMMIT");
|
|
4441
5602
|
return result;
|
|
4442
5603
|
} catch (err) {
|
|
@@ -4456,6 +5617,11 @@ var IndexStore = class _IndexStore {
|
|
|
4456
5617
|
"DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
4457
5618
|
).run(file);
|
|
4458
5619
|
}
|
|
5620
|
+
if (this.vectorsAvailable) {
|
|
5621
|
+
this.stmt(
|
|
5622
|
+
"DELETE FROM symbol_vectors WHERE symbol_id IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
5623
|
+
).run(file);
|
|
5624
|
+
}
|
|
4459
5625
|
this.stmt("DELETE FROM symbols WHERE file_fk = ?").run(file);
|
|
4460
5626
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
4461
5627
|
this.db.exec("COMMIT");
|
|
@@ -4481,6 +5647,11 @@ var IndexStore = class _IndexStore {
|
|
|
4481
5647
|
"DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
4482
5648
|
).run(file);
|
|
4483
5649
|
}
|
|
5650
|
+
if (this.vectorsAvailable) {
|
|
5651
|
+
this.stmt(
|
|
5652
|
+
"DELETE FROM symbol_vectors WHERE symbol_id IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
5653
|
+
).run(file);
|
|
5654
|
+
}
|
|
4484
5655
|
this.stmt(
|
|
4485
5656
|
"DELETE FROM refs WHERE from_id IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
4486
5657
|
).run(file);
|
|
@@ -4498,14 +5669,22 @@ var IndexStore = class _IndexStore {
|
|
|
4498
5669
|
upsertFile(meta) {
|
|
4499
5670
|
this.runWithRetry(() => {
|
|
4500
5671
|
this.stmt(
|
|
4501
|
-
`INSERT INTO files(file, lang, mtime_ms, symbol_count, last_indexed)
|
|
4502
|
-
VALUES (?, ?, ?, ?, ?)
|
|
5672
|
+
`INSERT INTO files(file, lang, mtime_ms, content_hash, symbol_count, last_indexed)
|
|
5673
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
4503
5674
|
ON CONFLICT(file) DO UPDATE SET
|
|
4504
5675
|
lang = excluded.lang,
|
|
4505
5676
|
mtime_ms = excluded.mtime_ms,
|
|
5677
|
+
content_hash = excluded.content_hash,
|
|
4506
5678
|
symbol_count = excluded.symbol_count,
|
|
4507
5679
|
last_indexed = excluded.last_indexed`
|
|
4508
|
-
).run(
|
|
5680
|
+
).run(
|
|
5681
|
+
meta.file,
|
|
5682
|
+
meta.lang,
|
|
5683
|
+
meta.mtimeMs,
|
|
5684
|
+
meta.contentHash ?? "",
|
|
5685
|
+
meta.symbolCount,
|
|
5686
|
+
meta.lastIndexed
|
|
5687
|
+
);
|
|
4509
5688
|
});
|
|
4510
5689
|
}
|
|
4511
5690
|
getFileMeta(file) {
|
|
@@ -4672,9 +5851,18 @@ var IndexStore = class _IndexStore {
|
|
|
4672
5851
|
if (mapped === null) return { results: [], total: 0 };
|
|
4673
5852
|
effectiveKind = mapped;
|
|
4674
5853
|
}
|
|
4675
|
-
const
|
|
5854
|
+
const longTokens = tokens.filter((t) => t.length >= 3);
|
|
5855
|
+
const shortTokens = tokens.filter((t) => t.length < 3);
|
|
5856
|
+
if (longTokens.length === 0) {
|
|
5857
|
+
return this.searchRankedFallback(query, filter, safeLimit);
|
|
5858
|
+
}
|
|
5859
|
+
const match = longTokens.map((t) => `"${t.replaceAll('"', "")}"`).join(" OR ");
|
|
4676
5860
|
const conditions = ["symbols_fts MATCH ?"];
|
|
4677
5861
|
const values = [match];
|
|
5862
|
+
for (const shortTok of shortTokens) {
|
|
5863
|
+
conditions.push("s.text LIKE ? ESCAPE '\\'");
|
|
5864
|
+
values.push(`%${escapeLike(shortTok)}%`);
|
|
5865
|
+
}
|
|
4678
5866
|
if (effectiveKind) {
|
|
4679
5867
|
conditions.push("s.kind = ?");
|
|
4680
5868
|
values.push(effectiveKind);
|
|
@@ -4693,7 +5881,7 @@ var IndexStore = class _IndexStore {
|
|
|
4693
5881
|
).all(...values);
|
|
4694
5882
|
const total = countRows[0] ? Number(countRows[0].n) : 0;
|
|
4695
5883
|
if (total === 0) return { results: [], total: 0 };
|
|
4696
|
-
const
|
|
5884
|
+
const bm25Rows = this.stmt(
|
|
4697
5885
|
`SELECT s.id, s.lang, s.kind, s.name, s.file, s.line, s.col, s.signature, s.doc_comment,
|
|
4698
5886
|
-bm25(symbols_fts) AS score,
|
|
4699
5887
|
snippet(symbols_fts, 0, '', '', '\u2026', 12) AS snippet
|
|
@@ -4706,8 +5894,35 @@ var IndexStore = class _IndexStore {
|
|
|
4706
5894
|
bm25(symbols_fts), lower(s.name), s.file, s.line, s.col, s.id
|
|
4707
5895
|
LIMIT ?`
|
|
4708
5896
|
).all(...values, query.trim(), `${escapeLike(query.trim())}%`, safeLimit);
|
|
5897
|
+
if (this.vectorsAvailable && bm25Rows.length > 0) {
|
|
5898
|
+
const queryVec = embedText(query);
|
|
5899
|
+
const candidateIds = bm25Rows.map((r) => r.id);
|
|
5900
|
+
const placeholders = candidateIds.map(() => "?").join(",");
|
|
5901
|
+
const vecRows = this.stmt(
|
|
5902
|
+
`SELECT sv.symbol_id, sv.vector FROM symbol_vectors sv WHERE sv.symbol_id IN (${placeholders})`
|
|
5903
|
+
).all(...candidateIds);
|
|
5904
|
+
const vecScores = vecRows.map((r) => ({
|
|
5905
|
+
id: r.symbol_id,
|
|
5906
|
+
sim: cosineSimilarity(queryVec, decodeVector(r.vector))
|
|
5907
|
+
})).sort((a, b) => b.sim - a.sim);
|
|
5908
|
+
const bm25Rank = /* @__PURE__ */ new Map();
|
|
5909
|
+
bm25Rows.forEach((r, i) => bm25Rank.set(r.id, i));
|
|
5910
|
+
const vecRank = /* @__PURE__ */ new Map();
|
|
5911
|
+
vecScores.forEach((r, i) => vecRank.set(r.id, i));
|
|
5912
|
+
const fused = reciprocalRankFusion(bm25Rank, vecRank, 60);
|
|
5913
|
+
const fusedScore = new Map(fused);
|
|
5914
|
+
const sorted = [...bm25Rows].sort(
|
|
5915
|
+
(a, b) => (fusedScore.get(b.id) ?? 0) - (fusedScore.get(a.id) ?? 0)
|
|
5916
|
+
);
|
|
5917
|
+
return {
|
|
5918
|
+
results: sorted.map(
|
|
5919
|
+
(row) => mapWriterSearchRow(row, filter?.lspKind, Math.max(1e-4, row.score), row.snippet)
|
|
5920
|
+
),
|
|
5921
|
+
total
|
|
5922
|
+
};
|
|
5923
|
+
}
|
|
4709
5924
|
return {
|
|
4710
|
-
results:
|
|
5925
|
+
results: bm25Rows.map(
|
|
4711
5926
|
(row) => mapWriterSearchRow(row, filter?.lspKind, Math.max(1e-4, row.score), row.snippet)
|
|
4712
5927
|
),
|
|
4713
5928
|
total
|
|
@@ -4835,6 +6050,7 @@ var IndexStore = class _IndexStore {
|
|
|
4835
6050
|
this.db.exec("DROP TABLE IF EXISTS files");
|
|
4836
6051
|
this.db.exec("DROP TABLE IF EXISTS metadata");
|
|
4837
6052
|
if (this.ftsAvailable) this.db.exec("DROP TABLE IF EXISTS symbols_fts");
|
|
6053
|
+
this.db.exec("DROP TABLE IF EXISTS symbol_vectors");
|
|
4838
6054
|
this.db.exec("COMMIT");
|
|
4839
6055
|
this.stmtCache.clear();
|
|
4840
6056
|
this.initSchema();
|
|
@@ -4922,6 +6138,11 @@ var IndexStore = class _IndexStore {
|
|
|
4922
6138
|
`DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file IN (${placeholders}))`
|
|
4923
6139
|
).run(...options.deleteForFiles);
|
|
4924
6140
|
}
|
|
6141
|
+
if (this.vectorsAvailable) {
|
|
6142
|
+
this.stmt(
|
|
6143
|
+
`DELETE FROM symbol_vectors WHERE symbol_id IN (SELECT id FROM symbols WHERE file IN (${placeholders}))`
|
|
6144
|
+
).run(...options.deleteForFiles);
|
|
6145
|
+
}
|
|
4925
6146
|
this.stmt(
|
|
4926
6147
|
`DELETE FROM refs WHERE from_id IN (SELECT id FROM symbols WHERE file IN (${placeholders}))`
|
|
4927
6148
|
).run(...options.deleteForFiles);
|
|
@@ -4935,6 +6156,7 @@ var IndexStore = class _IndexStore {
|
|
|
4935
6156
|
const refsToInsert = [];
|
|
4936
6157
|
const bulkSyms = [];
|
|
4937
6158
|
const ftsRows = [];
|
|
6159
|
+
const vectorRows = [];
|
|
4938
6160
|
for (const entry of entries) {
|
|
4939
6161
|
const insertedForEntry = [];
|
|
4940
6162
|
for (const s of entry.symbols) {
|
|
@@ -4958,6 +6180,10 @@ var IndexStore = class _IndexStore {
|
|
|
4958
6180
|
text: buildIndexableText(s.name, s.signature, s.docComment)
|
|
4959
6181
|
});
|
|
4960
6182
|
}
|
|
6183
|
+
vectorRows.push({
|
|
6184
|
+
id,
|
|
6185
|
+
vector: encodeVector(embedText(s.text || buildIndexableText(s.name, s.signature, s.docComment)))
|
|
6186
|
+
});
|
|
4961
6187
|
const inserted = { ...s, id };
|
|
4962
6188
|
allInserted.push(inserted);
|
|
4963
6189
|
insertedForEntry.push(inserted);
|
|
@@ -4971,19 +6197,34 @@ var IndexStore = class _IndexStore {
|
|
|
4971
6197
|
this.ftsAvailable,
|
|
4972
6198
|
ftsRows
|
|
4973
6199
|
);
|
|
6200
|
+
if (this.vectorsAvailable) {
|
|
6201
|
+
bulkInsertVectorsWithStatement(
|
|
6202
|
+
(sql) => this.stmt(sql),
|
|
6203
|
+
_IndexStore.MAX_SQL_VARS,
|
|
6204
|
+
vectorRows
|
|
6205
|
+
);
|
|
6206
|
+
}
|
|
4974
6207
|
bulkInsertRefsWithStatement((sql) => this.stmt(sql), _IndexStore.MAX_SQL_VARS, refsToInsert);
|
|
4975
6208
|
const upsertStmt = this.stmt(
|
|
4976
|
-
`INSERT INTO files(file, lang, mtime_ms, symbol_count, last_indexed)
|
|
4977
|
-
VALUES (?, ?, ?, ?, ?)
|
|
6209
|
+
`INSERT INTO files(file, lang, mtime_ms, content_hash, symbol_count, last_indexed)
|
|
6210
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
4978
6211
|
ON CONFLICT(file) DO UPDATE SET
|
|
4979
6212
|
lang = excluded.lang,
|
|
4980
6213
|
mtime_ms = excluded.mtime_ms,
|
|
6214
|
+
content_hash = excluded.content_hash,
|
|
4981
6215
|
symbol_count = excluded.symbol_count,
|
|
4982
6216
|
last_indexed = excluded.last_indexed`
|
|
4983
6217
|
);
|
|
4984
6218
|
const now = Date.now();
|
|
4985
6219
|
for (const entry of entries) {
|
|
4986
|
-
upsertStmt.run(
|
|
6220
|
+
upsertStmt.run(
|
|
6221
|
+
entry.file,
|
|
6222
|
+
entry.lang,
|
|
6223
|
+
entry.mtimeMs,
|
|
6224
|
+
entry.contentHash ?? "",
|
|
6225
|
+
entry.symbolCount,
|
|
6226
|
+
now
|
|
6227
|
+
);
|
|
4987
6228
|
}
|
|
4988
6229
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
4989
6230
|
this.db.exec("COMMIT");
|
|
@@ -5074,19 +6315,32 @@ var IndexStore = class _IndexStore {
|
|
|
5074
6315
|
"DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
5075
6316
|
).run(meta.file);
|
|
5076
6317
|
}
|
|
6318
|
+
if (this.vectorsAvailable) {
|
|
6319
|
+
this.stmt(
|
|
6320
|
+
"DELETE FROM symbol_vectors WHERE symbol_id IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
6321
|
+
).run(meta.file);
|
|
6322
|
+
}
|
|
5077
6323
|
this.stmt(
|
|
5078
6324
|
"DELETE FROM refs WHERE from_id IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
5079
6325
|
).run(meta.file);
|
|
5080
6326
|
this.stmt("DELETE FROM symbols WHERE file_fk = ?").run(meta.file);
|
|
5081
6327
|
this.stmt(
|
|
5082
|
-
`INSERT INTO files(file, lang, mtime_ms, symbol_count, last_indexed)
|
|
5083
|
-
VALUES (?, ?, ?, ?, ?)
|
|
6328
|
+
`INSERT INTO files(file, lang, mtime_ms, content_hash, symbol_count, last_indexed)
|
|
6329
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
5084
6330
|
ON CONFLICT(file) DO UPDATE SET
|
|
5085
6331
|
lang = excluded.lang,
|
|
5086
6332
|
mtime_ms = excluded.mtime_ms,
|
|
6333
|
+
content_hash = excluded.content_hash,
|
|
5087
6334
|
symbol_count = excluded.symbol_count,
|
|
5088
6335
|
last_indexed = excluded.last_indexed`
|
|
5089
|
-
).run(
|
|
6336
|
+
).run(
|
|
6337
|
+
meta.file,
|
|
6338
|
+
meta.lang,
|
|
6339
|
+
meta.mtimeMs,
|
|
6340
|
+
meta.contentHash ?? "",
|
|
6341
|
+
meta.symbolCount,
|
|
6342
|
+
meta.lastIndexed
|
|
6343
|
+
);
|
|
5090
6344
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
5091
6345
|
this.db.exec("COMMIT");
|
|
5092
6346
|
} catch (err) {
|
|
@@ -5149,6 +6403,31 @@ var IndexStore = class _IndexStore {
|
|
|
5149
6403
|
findOutgoingCallsByName(symbolName, file, limit = 100) {
|
|
5150
6404
|
return findOutgoingCallsByName((sql) => this.stmt(sql), symbolName, file, limit);
|
|
5151
6405
|
}
|
|
6406
|
+
/**
|
|
6407
|
+
* Transitive incoming-call tree: all symbols that transitively call the
|
|
6408
|
+
* target, to an unbounded depth (cycle-safe via SQL UNION deduplication).
|
|
6409
|
+
* Used by `codebase-incoming-calls` when the caller wants the full call
|
|
6410
|
+
* chain rather than just direct callers.
|
|
6411
|
+
*/
|
|
6412
|
+
findTransitiveIncomingCallsByName(symbolName, file, limit = 200) {
|
|
6413
|
+
return findTransitiveIncomingCallsByName((sql) => this.stmt(sql), symbolName, file, limit);
|
|
6414
|
+
}
|
|
6415
|
+
/**
|
|
6416
|
+
* Transitive outgoing-call tree: all symbols the target transitively calls.
|
|
6417
|
+
* Used by `codebase-outgoing-calls` when the caller wants the full
|
|
6418
|
+
* dependency chain rather than just direct callees.
|
|
6419
|
+
*/
|
|
6420
|
+
findTransitiveOutgoingCallsByName(symbolName, file, limit = 200) {
|
|
6421
|
+
return findTransitiveOutgoingCallsByName((sql) => this.stmt(sql), symbolName, file, limit);
|
|
6422
|
+
}
|
|
6423
|
+
/**
|
|
6424
|
+
* Compute the set of symbol IDs reachable from the given seed IDs using a
|
|
6425
|
+
* native SQLite recursive CTE. Used by dead-code detection to replace the
|
|
6426
|
+
* in-memory BFS.
|
|
6427
|
+
*/
|
|
6428
|
+
findReachableSymbolIds(seedIds) {
|
|
6429
|
+
return findReachableSymbolIds((sql) => this.stmt(sql), seedIds);
|
|
6430
|
+
}
|
|
5152
6431
|
/**
|
|
5153
6432
|
* Find all references TO a given symbol (who calls / uses this symbol?).
|
|
5154
6433
|
*/
|
|
@@ -5256,15 +6535,15 @@ var INDEXABLE_EXTENSION_SET = new Set(INDEXABLE_EXTENSIONS);
|
|
|
5256
6535
|
var MAX_INDEX_FILE_BYTES = 5 * 1024 * 1024;
|
|
5257
6536
|
var MAX_GIT_FILE_LIST_BYTES = 64 * 1024 * 1024;
|
|
5258
6537
|
function isWithinProject(projectRoot, file) {
|
|
5259
|
-
const rel =
|
|
5260
|
-
return rel !== "" && !rel.startsWith(`..${
|
|
6538
|
+
const rel = path12.relative(projectRoot, file);
|
|
6539
|
+
return rel !== "" && !rel.startsWith(`..${path12.sep}`) && rel !== ".." && !path12.isAbsolute(rel);
|
|
5261
6540
|
}
|
|
5262
6541
|
function isMissingPathError(err) {
|
|
5263
6542
|
const code = err?.code;
|
|
5264
6543
|
return code === "ENOENT" || code === "ENOTDIR";
|
|
5265
6544
|
}
|
|
5266
6545
|
function normalizeComparablePath(value) {
|
|
5267
|
-
const resolved =
|
|
6546
|
+
const resolved = path12.resolve(value);
|
|
5268
6547
|
return process.platform === "win32" ? resolved.toLowerCase() : resolved;
|
|
5269
6548
|
}
|
|
5270
6549
|
function gitOutput(projectRoot, args) {
|
|
@@ -5309,24 +6588,24 @@ async function findGitSourceFiles(projectRoot, ignore, signal) {
|
|
|
5309
6588
|
const record = statusRecords[i];
|
|
5310
6589
|
if (!record) continue;
|
|
5311
6590
|
const status = record.slice(0, 2);
|
|
5312
|
-
const changedPath =
|
|
6591
|
+
const changedPath = path12.resolve(projectRoot, record.slice(3));
|
|
5313
6592
|
dirty.add(changedPath);
|
|
5314
6593
|
if (status.includes("D")) deleted.add(changedPath);
|
|
5315
6594
|
if (status.includes("R") || status.includes("C")) {
|
|
5316
6595
|
const source = statusRecords[++i];
|
|
5317
|
-
if (source) dirty.add(
|
|
6596
|
+
if (source) dirty.add(path12.resolve(projectRoot, source));
|
|
5318
6597
|
}
|
|
5319
6598
|
}
|
|
5320
6599
|
const files = [];
|
|
5321
6600
|
for (const relative2 of output.toString("utf8").split("\0")) {
|
|
5322
6601
|
if (!relative2) continue;
|
|
5323
6602
|
const portable = relative2.replace(/\\/g, "/");
|
|
5324
|
-
if (portable.split("/").some((segment) => ignoreSet.has(segment)) || DEFAULT_IGNORE_FILES.has(
|
|
6603
|
+
if (portable.split("/").some((segment) => ignoreSet.has(segment)) || DEFAULT_IGNORE_FILES.has(path12.posix.basename(portable))) {
|
|
5325
6604
|
continue;
|
|
5326
6605
|
}
|
|
5327
|
-
const full =
|
|
6606
|
+
const full = path12.resolve(projectRoot, relative2);
|
|
5328
6607
|
if (deleted.has(full)) continue;
|
|
5329
|
-
const ext =
|
|
6608
|
+
const ext = path12.extname(relative2).toLowerCase();
|
|
5330
6609
|
if (INDEXABLE_EXTENSION_SET.has(ext) || detectLang(full) !== null) files.push(full);
|
|
5331
6610
|
}
|
|
5332
6611
|
return {
|
|
@@ -5361,7 +6640,7 @@ async function findSourceFiles(projectRoot, ignore, isGitIgnored, signal) {
|
|
|
5361
6640
|
}
|
|
5362
6641
|
let entries;
|
|
5363
6642
|
try {
|
|
5364
|
-
entries = await
|
|
6643
|
+
entries = await fs9.readdir(dir, { withFileTypes: true });
|
|
5365
6644
|
} catch (err) {
|
|
5366
6645
|
complete = false;
|
|
5367
6646
|
errors.push(`scan error: ${dir}: ${err instanceof Error ? err.message : String(err)}`);
|
|
@@ -5370,14 +6649,14 @@ async function findSourceFiles(projectRoot, ignore, isGitIgnored, signal) {
|
|
|
5370
6649
|
dirCount++;
|
|
5371
6650
|
for (const e of entries) {
|
|
5372
6651
|
if (ignoreSet.has(e.name)) continue;
|
|
5373
|
-
const full =
|
|
5374
|
-
const rel =
|
|
6652
|
+
const full = path12.join(dir, e.name);
|
|
6653
|
+
const rel = path12.relative(projectRoot, full).replace(/\\/g, "/");
|
|
5375
6654
|
if (e.isDirectory()) {
|
|
5376
6655
|
if (isGitIgnored(rel, true)) continue;
|
|
5377
6656
|
await walk(full);
|
|
5378
6657
|
} else if (e.isFile()) {
|
|
5379
6658
|
if (DEFAULT_IGNORE_FILES.has(e.name) || isGitIgnored(rel, false)) continue;
|
|
5380
|
-
const ext =
|
|
6659
|
+
const ext = path12.extname(e.name).toLowerCase();
|
|
5381
6660
|
if (indexableExts.has(ext) || detectLang(full) !== null) {
|
|
5382
6661
|
results.push(full);
|
|
5383
6662
|
}
|
|
@@ -5451,10 +6730,10 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5451
6730
|
let discoveryComplete = true;
|
|
5452
6731
|
let trustedUnchanged;
|
|
5453
6732
|
if (opts.files && opts.files.length > 0) {
|
|
5454
|
-
files = opts.files.map((f) =>
|
|
6733
|
+
files = opts.files.map((f) => path12.resolve(projectRoot, f)).filter((f) => {
|
|
5455
6734
|
if (!isWithinProject(projectRoot, f)) return false;
|
|
5456
|
-
const rel =
|
|
5457
|
-
return !rel.split("/").some((seg) => DEFAULT_IGNORE.includes(seg)) && !DEFAULT_IGNORE_FILES.has(
|
|
6735
|
+
const rel = path12.relative(projectRoot, f).replace(/\\/g, "/");
|
|
6736
|
+
return !rel.split("/").some((seg) => DEFAULT_IGNORE.includes(seg)) && !DEFAULT_IGNORE_FILES.has(path12.basename(f)) && !isGitIgnored(rel, false);
|
|
5458
6737
|
});
|
|
5459
6738
|
} else {
|
|
5460
6739
|
const discovery = await findSourceFiles(projectRoot, ignore, isGitIgnored, signal);
|
|
@@ -5511,7 +6790,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5511
6790
|
async (file) => {
|
|
5512
6791
|
let stat2;
|
|
5513
6792
|
try {
|
|
5514
|
-
stat2 = await
|
|
6793
|
+
stat2 = await fs9.stat(file, statOpts);
|
|
5515
6794
|
} catch (e) {
|
|
5516
6795
|
if (isAbortError(e)) throw e;
|
|
5517
6796
|
return {
|
|
@@ -5541,7 +6820,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5541
6820
|
}
|
|
5542
6821
|
let content;
|
|
5543
6822
|
try {
|
|
5544
|
-
content = await
|
|
6823
|
+
content = await fs9.readFile(file, { encoding: "utf8", signal });
|
|
5545
6824
|
} catch (e) {
|
|
5546
6825
|
if (isAbortError(e)) throw e;
|
|
5547
6826
|
return {
|
|
@@ -5552,22 +6831,78 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5552
6831
|
error: `read error: ${e instanceof Error ? e.message : String(e)}`
|
|
5553
6832
|
};
|
|
5554
6833
|
}
|
|
5555
|
-
|
|
5556
|
-
|
|
5557
|
-
parsed = await parseFileContent(file, content, lang);
|
|
5558
|
-
} catch (e) {
|
|
6834
|
+
const contentHash = xxhash64String(content);
|
|
6835
|
+
if (!force && meta && meta.contentHash && contentHash === meta.contentHash) {
|
|
5559
6836
|
return {
|
|
5560
6837
|
file,
|
|
5561
6838
|
stat: stat2,
|
|
5562
6839
|
lang,
|
|
5563
6840
|
parsed: null,
|
|
5564
|
-
|
|
6841
|
+
content,
|
|
6842
|
+
contentHash,
|
|
6843
|
+
skippedMeta: { ...meta, mtimeMs: Math.floor(stat2.mtimeMs) }
|
|
5565
6844
|
};
|
|
5566
6845
|
}
|
|
5567
|
-
return { file, stat: stat2, lang, parsed, content };
|
|
6846
|
+
return { file, stat: stat2, lang, parsed: null, content, contentHash };
|
|
5568
6847
|
}
|
|
5569
6848
|
)
|
|
5570
6849
|
);
|
|
6850
|
+
const toParse = [];
|
|
6851
|
+
for (let pi = 0; pi < statReadParse.length; pi++) {
|
|
6852
|
+
const s = statReadParse[pi];
|
|
6853
|
+
if (s.status !== "fulfilled") continue;
|
|
6854
|
+
const r = s.value;
|
|
6855
|
+
if (r.error || r.skippedMeta || !r.lang || r.parsed) continue;
|
|
6856
|
+
if (r.content === void 0) continue;
|
|
6857
|
+
toParse.push({
|
|
6858
|
+
index: pi,
|
|
6859
|
+
file: batchFiles[pi],
|
|
6860
|
+
content: r.content,
|
|
6861
|
+
lang: r.lang
|
|
6862
|
+
});
|
|
6863
|
+
}
|
|
6864
|
+
if (toParse.length > 0) {
|
|
6865
|
+
let pool = toParse.length >= WORKER_POOL_THRESHOLD ? getParserPool() : null;
|
|
6866
|
+
if (pool) {
|
|
6867
|
+
try {
|
|
6868
|
+
await pool.ensureReady();
|
|
6869
|
+
const parsedResults = await pool.parseFiles(
|
|
6870
|
+
toParse.map((p) => ({ file: p.file, content: p.content, lang: p.lang }))
|
|
6871
|
+
);
|
|
6872
|
+
const byFile = new Map(parsedResults.map((r) => [r.file, r]));
|
|
6873
|
+
for (const item of toParse) {
|
|
6874
|
+
const parsed = byFile.get(item.file);
|
|
6875
|
+
const settled = statReadParse[item.index];
|
|
6876
|
+
if (settled.status !== "fulfilled") continue;
|
|
6877
|
+
if (parsed) {
|
|
6878
|
+
settled.value.parsed = parsed;
|
|
6879
|
+
} else {
|
|
6880
|
+
settled.value.error = `parse error: worker returned no result for ${item.file}`;
|
|
6881
|
+
}
|
|
6882
|
+
}
|
|
6883
|
+
} catch {
|
|
6884
|
+
pool = null;
|
|
6885
|
+
}
|
|
6886
|
+
}
|
|
6887
|
+
if (!pool) {
|
|
6888
|
+
await Promise.all(
|
|
6889
|
+
toParse.map(async (item) => {
|
|
6890
|
+
try {
|
|
6891
|
+
const parsed = await parseFileContent(item.file, item.content, item.lang);
|
|
6892
|
+
const settled = statReadParse[item.index];
|
|
6893
|
+
if (settled.status === "fulfilled") {
|
|
6894
|
+
settled.value.parsed = parsed;
|
|
6895
|
+
}
|
|
6896
|
+
} catch (e) {
|
|
6897
|
+
const settled = statReadParse[item.index];
|
|
6898
|
+
if (settled.status === "fulfilled") {
|
|
6899
|
+
settled.value.error = `parse error: ${e instanceof Error ? e.message : String(e)}`;
|
|
6900
|
+
}
|
|
6901
|
+
}
|
|
6902
|
+
})
|
|
6903
|
+
);
|
|
6904
|
+
}
|
|
6905
|
+
}
|
|
5571
6906
|
const batchEntries = [];
|
|
5572
6907
|
const deleteForFiles = [];
|
|
5573
6908
|
for (let fi = 0; fi < statReadParse.length; fi++) {
|
|
@@ -5590,6 +6925,17 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5590
6925
|
langStats[lang] = (langStats[lang] ?? 0) + result.skippedMeta.symbolCount;
|
|
5591
6926
|
symbolsIndexed += result.skippedMeta.symbolCount;
|
|
5592
6927
|
filesIndexed++;
|
|
6928
|
+
const stored = existingMeta.get(file);
|
|
6929
|
+
if (stored && stored.mtimeMs !== result.skippedMeta.mtimeMs) {
|
|
6930
|
+
store.upsertFile({
|
|
6931
|
+
file,
|
|
6932
|
+
lang,
|
|
6933
|
+
mtimeMs: result.skippedMeta.mtimeMs,
|
|
6934
|
+
symbolCount: result.skippedMeta.symbolCount,
|
|
6935
|
+
lastIndexed: Date.now(),
|
|
6936
|
+
contentHash: result.skippedMeta.contentHash
|
|
6937
|
+
});
|
|
6938
|
+
}
|
|
5593
6939
|
continue;
|
|
5594
6940
|
}
|
|
5595
6941
|
if (!lang || !parsed) {
|
|
@@ -5599,7 +6945,8 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5599
6945
|
lang,
|
|
5600
6946
|
mtimeMs: Math.floor(stat2.mtimeMs),
|
|
5601
6947
|
symbolCount: 0,
|
|
5602
|
-
lastIndexed: Date.now()
|
|
6948
|
+
lastIndexed: Date.now(),
|
|
6949
|
+
contentHash: result.contentHash ?? ""
|
|
5603
6950
|
});
|
|
5604
6951
|
filesIndexed++;
|
|
5605
6952
|
}
|
|
@@ -5611,7 +6958,8 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5611
6958
|
lang,
|
|
5612
6959
|
mtimeMs: Math.floor(stat2.mtimeMs),
|
|
5613
6960
|
symbolCount: 0,
|
|
5614
|
-
lastIndexed: Date.now()
|
|
6961
|
+
lastIndexed: Date.now(),
|
|
6962
|
+
contentHash: result.contentHash ?? ""
|
|
5615
6963
|
});
|
|
5616
6964
|
filesIndexed++;
|
|
5617
6965
|
continue;
|
|
@@ -5622,7 +6970,8 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5622
6970
|
symbols: parsed.symbols,
|
|
5623
6971
|
refs: parsed.refs ?? [],
|
|
5624
6972
|
mtimeMs: Math.floor(stat2.mtimeMs),
|
|
5625
|
-
symbolCount: parsed.symbols.length
|
|
6973
|
+
symbolCount: parsed.symbols.length,
|
|
6974
|
+
contentHash: result.contentHash ?? ""
|
|
5626
6975
|
});
|
|
5627
6976
|
deleteForFiles.push(file);
|
|
5628
6977
|
}
|
|
@@ -5659,7 +7008,8 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5659
7008
|
lang: entry.lang,
|
|
5660
7009
|
mtimeMs: entry.mtimeMs,
|
|
5661
7010
|
symbolCount: entry.symbolCount,
|
|
5662
|
-
lastIndexed: Date.now()
|
|
7011
|
+
lastIndexed: Date.now(),
|
|
7012
|
+
contentHash: entry.contentHash
|
|
5663
7013
|
});
|
|
5664
7014
|
} catch (innerErr) {
|
|
5665
7015
|
errors.push(
|
|
@@ -5771,6 +7121,9 @@ function symbolGraphService(args) {
|
|
|
5771
7121
|
function incomingCallsService(args) {
|
|
5772
7122
|
const store = indexStorePool.acquire(args.projectRoot, { indexDir: args.indexDir });
|
|
5773
7123
|
try {
|
|
7124
|
+
if (args.transitive) {
|
|
7125
|
+
return store.findTransitiveIncomingCallsByName(args.symbol, args.file, args.limit ?? 100);
|
|
7126
|
+
}
|
|
5774
7127
|
return store.findIncomingCallsByName(args.symbol, args.file, args.limit ?? 100);
|
|
5775
7128
|
} finally {
|
|
5776
7129
|
indexStorePool.release(store);
|
|
@@ -5779,6 +7132,9 @@ function incomingCallsService(args) {
|
|
|
5779
7132
|
function outgoingCallsService(args) {
|
|
5780
7133
|
const store = indexStorePool.acquire(args.projectRoot, { indexDir: args.indexDir });
|
|
5781
7134
|
try {
|
|
7135
|
+
if (args.transitive) {
|
|
7136
|
+
return store.findTransitiveOutgoingCallsByName(args.symbol, args.file, args.limit ?? 100);
|
|
7137
|
+
}
|
|
5782
7138
|
return store.findOutgoingCallsByName(args.symbol, args.file, args.limit ?? 100);
|
|
5783
7139
|
} finally {
|
|
5784
7140
|
indexStorePool.release(store);
|