@wrongstack/tools 0.302.2 → 0.305.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/builtin.d.ts +6 -0
- package/dist/builtin.js +3213 -686
- package/dist/codebase-index/binary-frame.d.ts +43 -0
- package/dist/codebase-index/codebase-incoming-calls-tool.d.ts +1 -0
- package/dist/codebase-index/codebase-index-tool.d.ts +6 -0
- package/dist/codebase-index/codebase-outgoing-calls-tool.d.ts +1 -0
- package/dist/codebase-index/content-hash.d.ts +66 -0
- package/dist/codebase-index/index.js +1597 -153
- package/dist/codebase-index/indexer.d.ts +6 -0
- package/dist/codebase-index/parser-worker-pool.d.ts +63 -0
- package/dist/codebase-index/parser-worker-script.d.ts +42 -0
- package/dist/codebase-index/project-server-protocol.d.ts +2 -0
- package/dist/codebase-index/project-server.js +1483 -104
- package/dist/codebase-index/schema.d.ts +18 -0
- package/dist/codebase-index/tree-sitter/queries.d.ts +48 -0
- package/dist/codebase-index/tree-sitter/util.d.ts +31 -0
- package/dist/codebase-index/tree-sitter/visitor.d.ts +47 -0
- package/dist/codebase-index/tree-sitter-parser.d.ts +58 -0
- package/dist/codebase-index/vector-search.d.ts +62 -0
- package/dist/codebase-index/worker-protocol.d.ts +2 -0
- package/dist/codebase-index/worker.js +1452 -73
- package/dist/codebase-index/writer-bulk-insert.d.ts +5 -0
- package/dist/codebase-index/writer-graph-reader.d.ts +39 -0
- package/dist/codebase-index/writer-schema.d.ts +9 -2
- package/dist/codebase-index/writer.d.ts +36 -0
- package/dist/index.d.ts +4 -3
- package/dist/index.js +3258 -727
- package/dist/kanban-contract-actions.d.ts +7 -0
- package/dist/kanban-task-inputs.d.ts +16 -2
- package/dist/kanban-tool-schema.d.ts +2 -2
- package/dist/kanban-tool-types.d.ts +39 -2
- package/dist/kanban.js +580 -193
- package/dist/pack.js +3212 -686
- package/dist/plan.d.ts +4 -1
- package/dist/plan.js +2701 -18
- package/dist/read.js +1559 -103
- package/dist/session-kanban.d.ts +94 -1
- package/dist/session-kanban.js +308 -44
- package/dist/task.d.ts +5 -4
- package/dist/task.js +2825 -138
- package/dist/todo.d.ts +10 -1
- package/dist/todo.js +2474 -30
- package/dist/tool-tier.js +3212 -686
- package/package.json +8 -4
package/dist/read.js
CHANGED
|
@@ -2127,8 +2127,579 @@ var init_yaml_parser = __esm({
|
|
|
2127
2127
|
}
|
|
2128
2128
|
});
|
|
2129
2129
|
|
|
2130
|
+
// src/codebase-index/tree-sitter/queries.ts
|
|
2131
|
+
function getQueries(lang) {
|
|
2132
|
+
return LANG_QUERIES[lang] ?? DEFAULT_QUERIES;
|
|
2133
|
+
}
|
|
2134
|
+
function readFirstString(node) {
|
|
2135
|
+
if (!node) return null;
|
|
2136
|
+
if (node.type === "string_literal" || node.type === "alias") {
|
|
2137
|
+
return node.text.replace(/^"|"$/g, "");
|
|
2138
|
+
}
|
|
2139
|
+
const child = node.namedChild(0);
|
|
2140
|
+
return child ? readFirstString(child) : null;
|
|
2141
|
+
}
|
|
2142
|
+
var DEFAULT_QUERIES, LANG_QUERIES;
|
|
2143
|
+
var init_queries = __esm({
|
|
2144
|
+
"src/codebase-index/tree-sitter/queries.ts"() {
|
|
2145
|
+
"use strict";
|
|
2146
|
+
DEFAULT_QUERIES = {
|
|
2147
|
+
declKinds: {}
|
|
2148
|
+
};
|
|
2149
|
+
LANG_QUERIES = {
|
|
2150
|
+
// ─── C family ──────────────────────────────────────────────────────────────
|
|
2151
|
+
c: {
|
|
2152
|
+
declKinds: {
|
|
2153
|
+
function_definition: "function",
|
|
2154
|
+
declaration: "function",
|
|
2155
|
+
// K&R-style `int foo(...)` ambiguous w/ local var; the visitor prefers the function branch when the declarator field is present
|
|
2156
|
+
struct_specifier: "struct",
|
|
2157
|
+
union_specifier: "struct",
|
|
2158
|
+
enum_specifier: "enum",
|
|
2159
|
+
type_definition: "type",
|
|
2160
|
+
// `typedef … X;`
|
|
2161
|
+
preproc_def: "const"
|
|
2162
|
+
// `#define NAME …`
|
|
2163
|
+
},
|
|
2164
|
+
nameField: {
|
|
2165
|
+
function_definition: "declarator",
|
|
2166
|
+
declaration: "declarator",
|
|
2167
|
+
struct_specifier: "name",
|
|
2168
|
+
enum_specifier: "name",
|
|
2169
|
+
type_definition: "declarator",
|
|
2170
|
+
preproc_def: "name"
|
|
2171
|
+
},
|
|
2172
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2173
|
+
"translation_unit",
|
|
2174
|
+
"function_definition",
|
|
2175
|
+
"struct_specifier",
|
|
2176
|
+
"union_specifier",
|
|
2177
|
+
"enum_specifier"
|
|
2178
|
+
])
|
|
2179
|
+
},
|
|
2180
|
+
cpp: {
|
|
2181
|
+
declKinds: {
|
|
2182
|
+
function_definition: "function",
|
|
2183
|
+
template_declaration: "function",
|
|
2184
|
+
// `template<typename T> …`
|
|
2185
|
+
class_specifier: "class",
|
|
2186
|
+
struct_specifier: "struct",
|
|
2187
|
+
union_specifier: "struct",
|
|
2188
|
+
enum_specifier: "enum",
|
|
2189
|
+
namespace_definition: "namespace",
|
|
2190
|
+
type_definition: "type"
|
|
2191
|
+
},
|
|
2192
|
+
nameField: {
|
|
2193
|
+
function_definition: "declarator",
|
|
2194
|
+
template_declaration: "name",
|
|
2195
|
+
class_specifier: "name",
|
|
2196
|
+
struct_specifier: "name",
|
|
2197
|
+
enum_specifier: "name",
|
|
2198
|
+
namespace_definition: "name",
|
|
2199
|
+
type_definition: "declarator"
|
|
2200
|
+
},
|
|
2201
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2202
|
+
"translation_unit",
|
|
2203
|
+
"function_definition",
|
|
2204
|
+
"class_specifier",
|
|
2205
|
+
"struct_specifier",
|
|
2206
|
+
"union_specifier",
|
|
2207
|
+
"enum_specifier",
|
|
2208
|
+
"namespace_definition"
|
|
2209
|
+
])
|
|
2210
|
+
},
|
|
2211
|
+
java: {
|
|
2212
|
+
declKinds: {
|
|
2213
|
+
class_declaration: "class",
|
|
2214
|
+
interface_declaration: "interface",
|
|
2215
|
+
enum_declaration: "enum",
|
|
2216
|
+
record_declaration: "class",
|
|
2217
|
+
annotation_type_declaration: "interface",
|
|
2218
|
+
method_declaration: "method",
|
|
2219
|
+
constructor_declaration: "method",
|
|
2220
|
+
field_declaration: "property"
|
|
2221
|
+
},
|
|
2222
|
+
nameField: {
|
|
2223
|
+
class_declaration: "name",
|
|
2224
|
+
interface_declaration: "name",
|
|
2225
|
+
enum_declaration: "name",
|
|
2226
|
+
record_declaration: "name",
|
|
2227
|
+
annotation_type_declaration: "name",
|
|
2228
|
+
method_declaration: "name",
|
|
2229
|
+
constructor_declaration: "name"
|
|
2230
|
+
},
|
|
2231
|
+
// `field_declaration` has no single `name` field — it carries a list of
|
|
2232
|
+
// variable declarators. We emit one Symbol per node using the first
|
|
2233
|
+
// identifier-shaped named child (see `extractName` fallback in
|
|
2234
|
+
// `visitor.ts`). `int a, b, c;` therefore indexes only `a` — splitting
|
|
2235
|
+
// multi-declarator fields into separate Symbols is a separate refactor
|
|
2236
|
+
// that needs the visitor to know it has multiple names per node, and no
|
|
2237
|
+
// current test relies on it.
|
|
2238
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2239
|
+
"program",
|
|
2240
|
+
"class_declaration",
|
|
2241
|
+
"interface_declaration",
|
|
2242
|
+
"enum_declaration",
|
|
2243
|
+
"record_declaration"
|
|
2244
|
+
])
|
|
2245
|
+
},
|
|
2246
|
+
csharp: {
|
|
2247
|
+
// C# 10+ `namespace Foo.Bar;` produces this node type. The legacy block
|
|
2248
|
+
// form `namespace Foo.Bar { ... }` produces `namespace_declaration`. Both
|
|
2249
|
+
// carry a `qualified_name` child whose text already includes the dots.
|
|
2250
|
+
// `using_directive` is intentionally not a declaration. Imports are
|
|
2251
|
+
// extracted separately; indexing a using directive as a namespace makes
|
|
2252
|
+
// the resolver bind it to its own source file before the real declaration.
|
|
2253
|
+
declKinds: {
|
|
2254
|
+
file_scoped_namespace_declaration: "namespace",
|
|
2255
|
+
class_declaration: "class",
|
|
2256
|
+
interface_declaration: "interface",
|
|
2257
|
+
struct_declaration: "struct",
|
|
2258
|
+
enum_declaration: "enum",
|
|
2259
|
+
record_declaration: "class",
|
|
2260
|
+
method_declaration: "method",
|
|
2261
|
+
constructor_declaration: "method",
|
|
2262
|
+
property_declaration: "property",
|
|
2263
|
+
field_declaration: "property",
|
|
2264
|
+
namespace_declaration: "namespace"
|
|
2265
|
+
},
|
|
2266
|
+
// Custom name extractor: take the full dotted name verbatim.
|
|
2267
|
+
nameExtractor: (node) => {
|
|
2268
|
+
const inner = node.namedChild(0);
|
|
2269
|
+
if (inner && (inner.type === "qualified_name" || inner.type === "name")) {
|
|
2270
|
+
return inner.text;
|
|
2271
|
+
}
|
|
2272
|
+
return null;
|
|
2273
|
+
},
|
|
2274
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2275
|
+
"compilation_unit",
|
|
2276
|
+
"namespace_declaration",
|
|
2277
|
+
"class_declaration",
|
|
2278
|
+
"interface_declaration",
|
|
2279
|
+
"struct_declaration",
|
|
2280
|
+
"enum_declaration",
|
|
2281
|
+
"record_declaration"
|
|
2282
|
+
])
|
|
2283
|
+
},
|
|
2284
|
+
php: {
|
|
2285
|
+
declKinds: {
|
|
2286
|
+
function_definition: "function",
|
|
2287
|
+
method_declaration: "method",
|
|
2288
|
+
class_declaration: "class",
|
|
2289
|
+
interface_declaration: "interface",
|
|
2290
|
+
trait_declaration: "class",
|
|
2291
|
+
enum_declaration: "enum",
|
|
2292
|
+
namespace_definition: "namespace"
|
|
2293
|
+
},
|
|
2294
|
+
nameField: {
|
|
2295
|
+
function_definition: "name",
|
|
2296
|
+
method_declaration: "name",
|
|
2297
|
+
class_declaration: "name",
|
|
2298
|
+
interface_declaration: "name",
|
|
2299
|
+
trait_declaration: "name",
|
|
2300
|
+
enum_declaration: "name",
|
|
2301
|
+
namespace_declaration: "name"
|
|
2302
|
+
},
|
|
2303
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2304
|
+
"program",
|
|
2305
|
+
"namespace_definition",
|
|
2306
|
+
"class_declaration",
|
|
2307
|
+
"interface_declaration",
|
|
2308
|
+
"trait_declaration",
|
|
2309
|
+
"enum_declaration"
|
|
2310
|
+
])
|
|
2311
|
+
},
|
|
2312
|
+
// ─── Scripting / mobile ────────────────────────────────────────────────────
|
|
2313
|
+
ruby: {
|
|
2314
|
+
declKinds: {
|
|
2315
|
+
method: "function",
|
|
2316
|
+
singleton_method: "method",
|
|
2317
|
+
class: "class",
|
|
2318
|
+
module: "namespace",
|
|
2319
|
+
constant: "const"
|
|
2320
|
+
},
|
|
2321
|
+
nameField: {
|
|
2322
|
+
method: "name",
|
|
2323
|
+
singleton_method: "name",
|
|
2324
|
+
class: "name",
|
|
2325
|
+
module: "name",
|
|
2326
|
+
constant: "name"
|
|
2327
|
+
},
|
|
2328
|
+
scopeNodes: /* @__PURE__ */ new Set(["program", "class", "module", "singleton_method", "method"])
|
|
2329
|
+
},
|
|
2330
|
+
swift: {
|
|
2331
|
+
declKinds: {
|
|
2332
|
+
function_declaration: "function",
|
|
2333
|
+
class_declaration: "class",
|
|
2334
|
+
struct_declaration: "struct",
|
|
2335
|
+
enum_declaration: "enum",
|
|
2336
|
+
protocol_declaration: "interface",
|
|
2337
|
+
actor_declaration: "class",
|
|
2338
|
+
extension_declaration: "class",
|
|
2339
|
+
initializer: "method",
|
|
2340
|
+
property_declaration: "property"
|
|
2341
|
+
},
|
|
2342
|
+
nameField: {
|
|
2343
|
+
function_declaration: "name",
|
|
2344
|
+
class_declaration: "name",
|
|
2345
|
+
struct_declaration: "name",
|
|
2346
|
+
enum_declaration: "name",
|
|
2347
|
+
protocol_declaration: "name",
|
|
2348
|
+
actor_declaration: "name",
|
|
2349
|
+
extension_declaration: "name",
|
|
2350
|
+
initializer: "name",
|
|
2351
|
+
property_declaration: "name"
|
|
2352
|
+
},
|
|
2353
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2354
|
+
"source_file",
|
|
2355
|
+
"class_declaration",
|
|
2356
|
+
"struct_declaration",
|
|
2357
|
+
"enum_declaration",
|
|
2358
|
+
"protocol_declaration",
|
|
2359
|
+
"actor_declaration",
|
|
2360
|
+
"extension_declaration"
|
|
2361
|
+
])
|
|
2362
|
+
},
|
|
2363
|
+
kotlin: {
|
|
2364
|
+
declKinds: {
|
|
2365
|
+
class_declaration: "class",
|
|
2366
|
+
object_declaration: "class",
|
|
2367
|
+
interface_declaration: "interface",
|
|
2368
|
+
function_declaration: "function",
|
|
2369
|
+
property_declaration: "property",
|
|
2370
|
+
type_alias: "type"
|
|
2371
|
+
},
|
|
2372
|
+
nameField: {
|
|
2373
|
+
class_declaration: "name",
|
|
2374
|
+
object_declaration: "name",
|
|
2375
|
+
interface_declaration: "name",
|
|
2376
|
+
function_declaration: "name",
|
|
2377
|
+
property_declaration: "name",
|
|
2378
|
+
type_alias: "name"
|
|
2379
|
+
},
|
|
2380
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2381
|
+
"source_file",
|
|
2382
|
+
"class_declaration",
|
|
2383
|
+
"object_declaration",
|
|
2384
|
+
"interface_declaration",
|
|
2385
|
+
"function_declaration"
|
|
2386
|
+
])
|
|
2387
|
+
},
|
|
2388
|
+
elixir: {
|
|
2389
|
+
declKinds: {
|
|
2390
|
+
// `def foo`, `defp foo`, `defmacro foo`, `macrop foo` all surface as
|
|
2391
|
+
// `call` nodes in the tree-sitter grammar — there is no
|
|
2392
|
+
// `function_definition`. The `nameExtractor` walks the call's
|
|
2393
|
+
// children to pick the right sibling identifier.
|
|
2394
|
+
call: "function",
|
|
2395
|
+
module: "namespace"
|
|
2396
|
+
},
|
|
2397
|
+
nameExtractor: (node) => {
|
|
2398
|
+
if (node.type === "module") {
|
|
2399
|
+
const aliasNode = node.childForFieldName("alias");
|
|
2400
|
+
return readFirstString(aliasNode) ?? null;
|
|
2401
|
+
}
|
|
2402
|
+
if (node.type !== "call") return null;
|
|
2403
|
+
const first = node.namedChild(0);
|
|
2404
|
+
if (!first) return null;
|
|
2405
|
+
const target = first.text;
|
|
2406
|
+
if (target !== "def" && target !== "defp" && target !== "defmacro" && target !== "defp_macro" && target !== "macrop" && target !== "defprotocol" && target !== "defguard" && target !== "defguardp") {
|
|
2407
|
+
return null;
|
|
2408
|
+
}
|
|
2409
|
+
const nameNode = node.namedChild(1);
|
|
2410
|
+
return nameNode?.text ?? null;
|
|
2411
|
+
},
|
|
2412
|
+
scopeNodes: /* @__PURE__ */ new Set(["source", "module"])
|
|
2413
|
+
},
|
|
2414
|
+
shell: {
|
|
2415
|
+
declKinds: {
|
|
2416
|
+
function_definition: "function"
|
|
2417
|
+
},
|
|
2418
|
+
nameField: { function_definition: "name" },
|
|
2419
|
+
scopeNodes: /* @__PURE__ */ new Set(["program", "function_definition"])
|
|
2420
|
+
}
|
|
2421
|
+
};
|
|
2422
|
+
}
|
|
2423
|
+
});
|
|
2424
|
+
|
|
2425
|
+
// src/codebase-index/tree-sitter/util.ts
|
|
2426
|
+
function lineColAt2(offsets, index) {
|
|
2427
|
+
let low = 0;
|
|
2428
|
+
let high = offsets.length;
|
|
2429
|
+
while (low < high) {
|
|
2430
|
+
const mid = low + high >>> 1;
|
|
2431
|
+
if ((offsets[mid] ?? 0) < index) low = mid + 1;
|
|
2432
|
+
else high = mid;
|
|
2433
|
+
}
|
|
2434
|
+
const lastNl = low > 0 ? offsets[low - 1] ?? -1 : -1;
|
|
2435
|
+
return { line: low + 1, col: index - lastNl };
|
|
2436
|
+
}
|
|
2437
|
+
function newlineOffsets3(content) {
|
|
2438
|
+
const offsets = [];
|
|
2439
|
+
for (let i = 0; i < content.length; i++) {
|
|
2440
|
+
if (content.charCodeAt(i) === 10) offsets.push(i);
|
|
2441
|
+
}
|
|
2442
|
+
return offsets;
|
|
2443
|
+
}
|
|
2444
|
+
var TREE_SITTER_MAX_FILE_CHARS, TREE_SITTER_MAX_SYMBOLS;
|
|
2445
|
+
var init_util = __esm({
|
|
2446
|
+
"src/codebase-index/tree-sitter/util.ts"() {
|
|
2447
|
+
"use strict";
|
|
2448
|
+
TREE_SITTER_MAX_FILE_CHARS = 512 * 1024;
|
|
2449
|
+
TREE_SITTER_MAX_SYMBOLS = 500;
|
|
2450
|
+
}
|
|
2451
|
+
});
|
|
2452
|
+
|
|
2453
|
+
// src/codebase-index/tree-sitter/visitor.ts
|
|
2454
|
+
function visitTree(tree, content, file, lang, queries) {
|
|
2455
|
+
const boundedContent = content.length > TREE_SITTER_MAX_FILE_CHARS ? content.slice(0, TREE_SITTER_MAX_FILE_CHARS) : content;
|
|
2456
|
+
const nlOffsets = newlineOffsets3(boundedContent);
|
|
2457
|
+
const symbols = [];
|
|
2458
|
+
const scopeStack = [];
|
|
2459
|
+
function visit(node, depth) {
|
|
2460
|
+
if (symbols.length >= TREE_SITTER_MAX_SYMBOLS) return;
|
|
2461
|
+
if (node.isMissing || node.isError) {
|
|
2462
|
+
} else {
|
|
2463
|
+
const kind = queries.declKinds[node.type];
|
|
2464
|
+
if (kind) {
|
|
2465
|
+
const emitted = emitSymbol(
|
|
2466
|
+
node,
|
|
2467
|
+
kind,
|
|
2468
|
+
file,
|
|
2469
|
+
lang,
|
|
2470
|
+
scopeStack,
|
|
2471
|
+
boundedContent,
|
|
2472
|
+
nlOffsets,
|
|
2473
|
+
queries
|
|
2474
|
+
);
|
|
2475
|
+
if (emitted) symbols.push(emitted);
|
|
2476
|
+
}
|
|
2477
|
+
}
|
|
2478
|
+
const pushesScope = queries.scopeNodes?.has(node.type) ?? false;
|
|
2479
|
+
const pushIdx = pushesScope ? pushScope(scopeStack, node, queries) : -1;
|
|
2480
|
+
if (queries.skipNamedChildren) {
|
|
2481
|
+
if (pushIdx !== -1) scopeStack.pop();
|
|
2482
|
+
return;
|
|
2483
|
+
}
|
|
2484
|
+
for (const child of node.namedChildren) {
|
|
2485
|
+
visit(child, depth + 1);
|
|
2486
|
+
if (symbols.length >= TREE_SITTER_MAX_SYMBOLS) {
|
|
2487
|
+
if (pushIdx !== -1) scopeStack.pop();
|
|
2488
|
+
return;
|
|
2489
|
+
}
|
|
2490
|
+
}
|
|
2491
|
+
if (pushIdx !== -1) scopeStack.pop();
|
|
2492
|
+
}
|
|
2493
|
+
visit(tree.rootNode, 0);
|
|
2494
|
+
return { symbols };
|
|
2495
|
+
}
|
|
2496
|
+
function pushScope(scopeStack, node, queries) {
|
|
2497
|
+
const name = extractName(node, queries);
|
|
2498
|
+
if (!name) return -1;
|
|
2499
|
+
scopeStack.push(name);
|
|
2500
|
+
return scopeStack.length - 1;
|
|
2501
|
+
}
|
|
2502
|
+
function extractName(node, queries) {
|
|
2503
|
+
if (queries.nameExtractor) {
|
|
2504
|
+
const extracted = queries.nameExtractor(node);
|
|
2505
|
+
if (extracted) return extracted;
|
|
2506
|
+
}
|
|
2507
|
+
const fieldName = queries.nameField?.[node.type] ?? "name";
|
|
2508
|
+
const field = node.childForFieldName(fieldName);
|
|
2509
|
+
if (field) {
|
|
2510
|
+
if (IDENTIFIER_NODE_TYPES.has(field.type)) {
|
|
2511
|
+
return field.text;
|
|
2512
|
+
}
|
|
2513
|
+
const inner = field.childForFieldName("name") ?? field.namedChild(0);
|
|
2514
|
+
if (inner && IDENTIFIER_NODE_TYPES.has(inner.type)) {
|
|
2515
|
+
return inner.text;
|
|
2516
|
+
}
|
|
2517
|
+
}
|
|
2518
|
+
for (let i = 0; i < node.namedChildCount; i++) {
|
|
2519
|
+
const child = node.namedChild(i);
|
|
2520
|
+
if (child && IDENTIFIER_NODE_TYPES.has(child.type)) {
|
|
2521
|
+
return child.text;
|
|
2522
|
+
}
|
|
2523
|
+
}
|
|
2524
|
+
return null;
|
|
2525
|
+
}
|
|
2526
|
+
function emitSymbol(node, kind, file, lang, scopeStack, content, nlOffsets, queries) {
|
|
2527
|
+
const name = extractName(node, queries);
|
|
2528
|
+
if (!name) return null;
|
|
2529
|
+
const pos = node.startIndex;
|
|
2530
|
+
const { line, col } = lineColAt2(nlOffsets, pos);
|
|
2531
|
+
const end = Math.min(node.endIndex, content.length);
|
|
2532
|
+
const signature = content.slice(pos, end).replace(/\s+/g, " ").trim().slice(0, 500);
|
|
2533
|
+
const scope = scopeStack.join(".");
|
|
2534
|
+
const text = [name, signature].filter(Boolean).join(" | ").trim().slice(0, 1e3);
|
|
2535
|
+
return {
|
|
2536
|
+
id: 0,
|
|
2537
|
+
// caller assigns during bulk insertion
|
|
2538
|
+
lang,
|
|
2539
|
+
kind,
|
|
2540
|
+
name: name.slice(0, 200),
|
|
2541
|
+
file,
|
|
2542
|
+
line,
|
|
2543
|
+
col,
|
|
2544
|
+
signature,
|
|
2545
|
+
docComment: "",
|
|
2546
|
+
// doc-comment extraction lands with ref emission on Day 4
|
|
2547
|
+
scope,
|
|
2548
|
+
text
|
|
2549
|
+
};
|
|
2550
|
+
}
|
|
2551
|
+
var IDENTIFIER_NODE_TYPES;
|
|
2552
|
+
var init_visitor = __esm({
|
|
2553
|
+
"src/codebase-index/tree-sitter/visitor.ts"() {
|
|
2554
|
+
"use strict";
|
|
2555
|
+
init_util();
|
|
2556
|
+
IDENTIFIER_NODE_TYPES = /* @__PURE__ */ new Set([
|
|
2557
|
+
"identifier",
|
|
2558
|
+
"simple_identifier",
|
|
2559
|
+
"type_identifier",
|
|
2560
|
+
"field_identifier",
|
|
2561
|
+
"property_identifier",
|
|
2562
|
+
"name",
|
|
2563
|
+
"word",
|
|
2564
|
+
"variable_name",
|
|
2565
|
+
"constant",
|
|
2566
|
+
"sym"
|
|
2567
|
+
]);
|
|
2568
|
+
}
|
|
2569
|
+
});
|
|
2570
|
+
|
|
2571
|
+
// src/codebase-index/tree-sitter-parser.ts
|
|
2572
|
+
var tree_sitter_parser_exports = {};
|
|
2573
|
+
__export(tree_sitter_parser_exports, {
|
|
2574
|
+
__smokeRootType: () => __smokeRootType,
|
|
2575
|
+
getGrammarWasmPath: () => getGrammarWasmPath,
|
|
2576
|
+
isTreeSitterSupported: () => isTreeSitterSupported,
|
|
2577
|
+
loadTreeSitterLanguage: () => loadTreeSitterLanguage,
|
|
2578
|
+
parseSymbols: () => parseSymbols8
|
|
2579
|
+
});
|
|
2580
|
+
import * as path10 from "node:path";
|
|
2581
|
+
import { fileURLToPath } from "node:url";
|
|
2582
|
+
function optInEnabled(env) {
|
|
2583
|
+
return process.env[env] === "1" || process.env[env] === "true";
|
|
2584
|
+
}
|
|
2585
|
+
function getRuntime() {
|
|
2586
|
+
if (!runtimePromise) {
|
|
2587
|
+
runtimePromise = (async () => {
|
|
2588
|
+
const mod = await import("web-tree-sitter");
|
|
2589
|
+
const init = () => mod.Parser.init({ locateFile: () => RUNTIME_WASM });
|
|
2590
|
+
return { Parser: mod.Parser, Language: mod.Language, init };
|
|
2591
|
+
})();
|
|
2592
|
+
}
|
|
2593
|
+
return runtimePromise;
|
|
2594
|
+
}
|
|
2595
|
+
async function loadLanguage(lang) {
|
|
2596
|
+
const existing = languageCache.get(lang);
|
|
2597
|
+
if (existing) return existing;
|
|
2598
|
+
const promise = (async () => {
|
|
2599
|
+
const grammarName = resolveGrammarName(lang);
|
|
2600
|
+
if (!grammarName) {
|
|
2601
|
+
throw new Error(`tree-sitter: no grammar registered for lang "${lang}"`);
|
|
2602
|
+
}
|
|
2603
|
+
const wasmPath = path10.join(WASM_DIR, grammarName, `tree-sitter-${grammarName}.wasm`);
|
|
2604
|
+
const { Language, init } = await getRuntime();
|
|
2605
|
+
await init();
|
|
2606
|
+
const languageObj = await Language.load(wasmPath);
|
|
2607
|
+
return { lang, Language: languageObj };
|
|
2608
|
+
})();
|
|
2609
|
+
languageCache.set(lang, promise);
|
|
2610
|
+
return promise;
|
|
2611
|
+
}
|
|
2612
|
+
function resolveGrammarName(lang) {
|
|
2613
|
+
if (lang === "go" && optInEnabled(GO_OPT_IN)) return "go";
|
|
2614
|
+
if (lang === "py" && optInEnabled(PY_OPT_IN)) return "python";
|
|
2615
|
+
if (lang === "rs" && optInEnabled(RS_OPT_IN)) return "rust";
|
|
2616
|
+
return LANG_TO_GRAMMAR[lang];
|
|
2617
|
+
}
|
|
2618
|
+
function isTreeSitterSupported(lang) {
|
|
2619
|
+
return resolveGrammarName(lang) !== void 0;
|
|
2620
|
+
}
|
|
2621
|
+
function getGrammarWasmPath(lang) {
|
|
2622
|
+
const name = resolveGrammarName(lang);
|
|
2623
|
+
if (!name) return void 0;
|
|
2624
|
+
return path10.join(WASM_DIR, name, `tree-sitter-${name}.wasm`);
|
|
2625
|
+
}
|
|
2626
|
+
async function parseSymbols8(opts) {
|
|
2627
|
+
const { file, content, lang } = opts;
|
|
2628
|
+
if (!isTreeSitterSupported(lang)) {
|
|
2629
|
+
return { file, lang, symbols: [], mtimeMs: Date.now() };
|
|
2630
|
+
}
|
|
2631
|
+
try {
|
|
2632
|
+
const { Parser } = await getRuntime();
|
|
2633
|
+
const cached = await loadLanguage(lang);
|
|
2634
|
+
const parser = new Parser();
|
|
2635
|
+
parser.setLanguage(cached.Language);
|
|
2636
|
+
const tree = parser.parse(content);
|
|
2637
|
+
if (!tree) {
|
|
2638
|
+
return { file, lang, symbols: [], mtimeMs: Date.now() };
|
|
2639
|
+
}
|
|
2640
|
+
const { symbols } = visitTree(tree, content, file, lang, getQueries(lang));
|
|
2641
|
+
parser.delete();
|
|
2642
|
+
tree.delete();
|
|
2643
|
+
return { file, lang, symbols, refs: [], mtimeMs: Date.now() };
|
|
2644
|
+
} catch {
|
|
2645
|
+
return { file, lang, symbols: [], mtimeMs: Date.now() };
|
|
2646
|
+
}
|
|
2647
|
+
}
|
|
2648
|
+
async function loadTreeSitterLanguage(lang) {
|
|
2649
|
+
const cached = await loadLanguage(lang);
|
|
2650
|
+
return cached.Language;
|
|
2651
|
+
}
|
|
2652
|
+
async function __smokeRootType(opts) {
|
|
2653
|
+
if (!isTreeSitterSupported(opts.lang)) {
|
|
2654
|
+
throw new Error(`tree-sitter: no grammar registered for lang "${opts.lang}"`);
|
|
2655
|
+
}
|
|
2656
|
+
const { Parser } = await getRuntime();
|
|
2657
|
+
const cached = await loadLanguage(opts.lang);
|
|
2658
|
+
const parser = new Parser();
|
|
2659
|
+
parser.setLanguage(cached.Language);
|
|
2660
|
+
let tree = null;
|
|
2661
|
+
try {
|
|
2662
|
+
tree = parser.parse(opts.content);
|
|
2663
|
+
if (!tree) throw new Error("tree-sitter: parser.parse returned null");
|
|
2664
|
+
return tree.rootNode.type;
|
|
2665
|
+
} finally {
|
|
2666
|
+
tree?.delete();
|
|
2667
|
+
parser.delete();
|
|
2668
|
+
}
|
|
2669
|
+
}
|
|
2670
|
+
var WASM_DIR, RUNTIME_WASM, LANG_TO_GRAMMAR, GO_OPT_IN, PY_OPT_IN, RS_OPT_IN, runtimePromise, languageCache;
|
|
2671
|
+
var init_tree_sitter_parser = __esm({
|
|
2672
|
+
"src/codebase-index/tree-sitter-parser.ts"() {
|
|
2673
|
+
"use strict";
|
|
2674
|
+
init_queries();
|
|
2675
|
+
init_visitor();
|
|
2676
|
+
WASM_DIR = fileURLToPath(new URL("./wasm/", import.meta.url));
|
|
2677
|
+
RUNTIME_WASM = path10.join(WASM_DIR, "tree-sitter-runtime.wasm");
|
|
2678
|
+
LANG_TO_GRAMMAR = {
|
|
2679
|
+
c: "c",
|
|
2680
|
+
cpp: "cpp",
|
|
2681
|
+
java: "java",
|
|
2682
|
+
csharp: "c_sharp",
|
|
2683
|
+
// tree-sitter directory uses underscore
|
|
2684
|
+
php: "php",
|
|
2685
|
+
ruby: "ruby",
|
|
2686
|
+
swift: "swift",
|
|
2687
|
+
kotlin: "kotlin",
|
|
2688
|
+
shell: "bash",
|
|
2689
|
+
// we treat `.sh` / `.bash` / `.zsh` via the bash grammar
|
|
2690
|
+
// Go/Python/Rust are opt-in only (see goOptIn / pyOptIn / rsOptIn below)
|
|
2691
|
+
elixir: "elixir"
|
|
2692
|
+
};
|
|
2693
|
+
GO_OPT_IN = "WRONGSTACK_USE_TS_GO";
|
|
2694
|
+
PY_OPT_IN = "WRONGSTACK_USE_TS_PY";
|
|
2695
|
+
RS_OPT_IN = "WRONGSTACK_USE_TS_RS";
|
|
2696
|
+
runtimePromise = null;
|
|
2697
|
+
languageCache = /* @__PURE__ */ new Map();
|
|
2698
|
+
}
|
|
2699
|
+
});
|
|
2700
|
+
|
|
2130
2701
|
// src/read.ts
|
|
2131
|
-
import * as
|
|
2702
|
+
import * as fs13 from "node:fs/promises";
|
|
2132
2703
|
import { FsError, ToolValidationError } from "@wrongstack/core/types";
|
|
2133
2704
|
import { toErrorMessage as toErrorMessage2 } from "@wrongstack/core/utils";
|
|
2134
2705
|
|
|
@@ -2203,9 +2774,9 @@ function isBinaryBuffer(buf) {
|
|
|
2203
2774
|
}
|
|
2204
2775
|
|
|
2205
2776
|
// src/codebase-index/background-indexer.ts
|
|
2206
|
-
import * as
|
|
2207
|
-
import { fileURLToPath as
|
|
2208
|
-
import { Worker } from "node:worker_threads";
|
|
2777
|
+
import * as fs12 from "node:fs";
|
|
2778
|
+
import { fileURLToPath as fileURLToPath5 } from "node:url";
|
|
2779
|
+
import { Worker as Worker2 } from "node:worker_threads";
|
|
2209
2780
|
|
|
2210
2781
|
// src/codebase-index/circuit-breaker.ts
|
|
2211
2782
|
var IndexTimeoutError = class extends Error {
|
|
@@ -2287,15 +2858,95 @@ var indexCircuitBreaker = new IndexCircuitBreaker();
|
|
|
2287
2858
|
// src/codebase-index/indexer.ts
|
|
2288
2859
|
import { expectDefined as expectDefined5 } from "@wrongstack/core/utils";
|
|
2289
2860
|
import { execFile } from "node:child_process";
|
|
2290
|
-
import * as
|
|
2861
|
+
import * as fs9 from "node:fs/promises";
|
|
2291
2862
|
import { availableParallelism } from "node:os";
|
|
2292
|
-
import * as
|
|
2863
|
+
import * as path13 from "node:path";
|
|
2293
2864
|
import {
|
|
2294
2865
|
DEFAULT_WALK_IGNORE_DIRS,
|
|
2295
2866
|
indexParallelBatchSize,
|
|
2296
2867
|
isFrugalPerf
|
|
2297
2868
|
} from "@wrongstack/core/utils";
|
|
2298
2869
|
|
|
2870
|
+
// src/codebase-index/content-hash.ts
|
|
2871
|
+
var PRIME64_1 = 0x9e3779b185ebca87n;
|
|
2872
|
+
var PRIME64_2 = 0xc2b2ae3d27d4eb4fn;
|
|
2873
|
+
var PRIME64_3 = 0x165667b19e3779f9n;
|
|
2874
|
+
var PRIME64_4 = 0x85ebca77c2b2ae63n;
|
|
2875
|
+
var PRIME64_5 = 0x27d4eb2f165667c5n;
|
|
2876
|
+
var MASK64 = 0xffffffffffffffffn;
|
|
2877
|
+
function mul64(a, b) {
|
|
2878
|
+
return (a & MASK64) * (b & MASK64) & MASK64;
|
|
2879
|
+
}
|
|
2880
|
+
function rotl64(x, n) {
|
|
2881
|
+
const v = x & MASK64;
|
|
2882
|
+
return (v << BigInt(n) | v >> BigInt(64 - n)) & MASK64;
|
|
2883
|
+
}
|
|
2884
|
+
function readU64LE(buf, off) {
|
|
2885
|
+
let v = 0n;
|
|
2886
|
+
for (let i = 7; i >= 0; i--) {
|
|
2887
|
+
v = v << 8n | BigInt(buf[off + i] ?? 0);
|
|
2888
|
+
}
|
|
2889
|
+
return v & MASK64;
|
|
2890
|
+
}
|
|
2891
|
+
function readU32LE(buf, off) {
|
|
2892
|
+
return (BigInt(buf[off] ?? 0) | BigInt(buf[off + 1] ?? 0) << 8n | BigInt(buf[off + 2] ?? 0) << 16n | BigInt(buf[off + 3] ?? 0) << 24n) & MASK64;
|
|
2893
|
+
}
|
|
2894
|
+
function xxh64Round(acc, lane) {
|
|
2895
|
+
return mul64(rotl64(acc + mul64(lane, PRIME64_2) & MASK64, 31), PRIME64_1);
|
|
2896
|
+
}
|
|
2897
|
+
function xxh64MergeRound(acc, val) {
|
|
2898
|
+
return mul64(acc ^ xxh64Round(0n, val), PRIME64_1) + PRIME64_4 & MASK64;
|
|
2899
|
+
}
|
|
2900
|
+
function xxhash64Hex(buf, explicitLen) {
|
|
2901
|
+
const length = explicitLen ?? buf.length;
|
|
2902
|
+
let h;
|
|
2903
|
+
let off = 0;
|
|
2904
|
+
if (length >= 32) {
|
|
2905
|
+
let v1 = PRIME64_1 + PRIME64_2 & MASK64;
|
|
2906
|
+
let v2 = PRIME64_2;
|
|
2907
|
+
let v3 = 0n;
|
|
2908
|
+
let v4 = 0n - PRIME64_1 & MASK64;
|
|
2909
|
+
const end32 = length - 32;
|
|
2910
|
+
while (off <= end32) {
|
|
2911
|
+
v1 = xxh64Round(v1, readU64LE(buf, off));
|
|
2912
|
+
v2 = xxh64Round(v2, readU64LE(buf, off + 8));
|
|
2913
|
+
v3 = xxh64Round(v3, readU64LE(buf, off + 16));
|
|
2914
|
+
v4 = xxh64Round(v4, readU64LE(buf, off + 24));
|
|
2915
|
+
off += 32;
|
|
2916
|
+
}
|
|
2917
|
+
h = rotl64(v1, 1) + rotl64(v2, 7) + rotl64(v3, 12) + rotl64(v4, 18) & MASK64;
|
|
2918
|
+
h = xxh64MergeRound(h, v1);
|
|
2919
|
+
h = xxh64MergeRound(h, v2);
|
|
2920
|
+
h = xxh64MergeRound(h, v3);
|
|
2921
|
+
h = xxh64MergeRound(h, v4);
|
|
2922
|
+
} else {
|
|
2923
|
+
h = PRIME64_5;
|
|
2924
|
+
}
|
|
2925
|
+
h = h + BigInt(length) & MASK64;
|
|
2926
|
+
while (off + 8 <= length) {
|
|
2927
|
+
const k1 = xxh64Round(0n, readU64LE(buf, off));
|
|
2928
|
+
h = mul64(rotl64(h ^ k1, 27), PRIME64_1) + PRIME64_4 & MASK64;
|
|
2929
|
+
off += 8;
|
|
2930
|
+
}
|
|
2931
|
+
if (off + 4 <= length) {
|
|
2932
|
+
h = mul64(rotl64(h ^ mul64(readU32LE(buf, off), PRIME64_1), 23), PRIME64_2) + PRIME64_3 & MASK64;
|
|
2933
|
+
off += 4;
|
|
2934
|
+
}
|
|
2935
|
+
while (off < length) {
|
|
2936
|
+
h = mul64(rotl64(h ^ mul64(BigInt(buf[off] ?? 0), PRIME64_5), 11), PRIME64_1) & MASK64;
|
|
2937
|
+
off += 1;
|
|
2938
|
+
}
|
|
2939
|
+
h = (h ^ h >> 33n) & MASK64;
|
|
2940
|
+
h = mul64(h, PRIME64_2);
|
|
2941
|
+
h = (h ^ h >> 29n) & MASK64;
|
|
2942
|
+
h = mul64(h, PRIME64_3);
|
|
2943
|
+
h = (h ^ h >> 32n) & MASK64;
|
|
2944
|
+
return h.toString(16).padStart(16, "0");
|
|
2945
|
+
}
|
|
2946
|
+
function xxhash64String(content) {
|
|
2947
|
+
return xxhash64Hex(new TextEncoder().encode(content));
|
|
2948
|
+
}
|
|
2949
|
+
|
|
2299
2950
|
// src/codebase-index/gitignore.ts
|
|
2300
2951
|
import * as fs from "node:fs/promises";
|
|
2301
2952
|
import * as path2 from "node:path";
|
|
@@ -3091,32 +3742,56 @@ async function dispatch(file, content, lang) {
|
|
|
3091
3742
|
case "tsx":
|
|
3092
3743
|
case "js":
|
|
3093
3744
|
case "jsx": {
|
|
3094
|
-
const { parseSymbols:
|
|
3095
|
-
return
|
|
3745
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_ts_parser(), ts_parser_exports));
|
|
3746
|
+
return parseSymbols9({ file, content, lang });
|
|
3096
3747
|
}
|
|
3097
3748
|
case "go": {
|
|
3098
|
-
const { parseSymbols:
|
|
3099
|
-
return
|
|
3749
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_go_parser(), go_parser_exports));
|
|
3750
|
+
return parseSymbols9({ file, content, lang: "go" });
|
|
3100
3751
|
}
|
|
3101
3752
|
case "py": {
|
|
3102
|
-
const { parseSymbols:
|
|
3103
|
-
return
|
|
3753
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_py_parser(), py_parser_exports));
|
|
3754
|
+
return parseSymbols9({ file, content, lang: "py" });
|
|
3104
3755
|
}
|
|
3105
3756
|
case "rs": {
|
|
3106
|
-
const { parseSymbols:
|
|
3107
|
-
return
|
|
3757
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_rs_parser(), rs_parser_exports));
|
|
3758
|
+
return parseSymbols9({ file, content, lang: "rs" });
|
|
3108
3759
|
}
|
|
3109
3760
|
case "json": {
|
|
3110
|
-
const { parseSymbols:
|
|
3111
|
-
return
|
|
3761
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_json_parser(), json_parser_exports));
|
|
3762
|
+
return parseSymbols9({ file, content, lang: "json" });
|
|
3112
3763
|
}
|
|
3113
3764
|
case "yaml": {
|
|
3114
|
-
const { parseSymbols:
|
|
3115
|
-
return
|
|
3765
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_yaml_parser(), yaml_parser_exports));
|
|
3766
|
+
return parseSymbols9({ file, content, lang: "yaml" });
|
|
3767
|
+
}
|
|
3768
|
+
// Phase 1: ten languages now route through the Tree-Sitter WASM
|
|
3769
|
+
// universal parser (`tree-sitter-parser.ts`). The dispatch falls back to
|
|
3770
|
+
// the regex extractor in `generic-parser.ts` whenever WASM loading fails
|
|
3771
|
+
// or the parser returns zero symbols — preserving the indexable-file
|
|
3772
|
+
// contract that "missing a parser must never mean skipping the file".
|
|
3773
|
+
case "c":
|
|
3774
|
+
case "cpp":
|
|
3775
|
+
case "java":
|
|
3776
|
+
case "csharp":
|
|
3777
|
+
case "php":
|
|
3778
|
+
case "ruby":
|
|
3779
|
+
case "swift":
|
|
3780
|
+
case "kotlin":
|
|
3781
|
+
case "shell":
|
|
3782
|
+
case "elixir": {
|
|
3783
|
+
try {
|
|
3784
|
+
const { parseSymbols: parseSymbols10 } = await Promise.resolve().then(() => (init_tree_sitter_parser(), tree_sitter_parser_exports));
|
|
3785
|
+
const parsed = await parseSymbols10({ file, content, lang });
|
|
3786
|
+
if (parsed.symbols.length > 0) return parsed;
|
|
3787
|
+
} catch {
|
|
3788
|
+
}
|
|
3789
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_generic_parser(), generic_parser_exports));
|
|
3790
|
+
return parseSymbols9({ file, content, lang });
|
|
3116
3791
|
}
|
|
3117
3792
|
default: {
|
|
3118
|
-
const { parseSymbols:
|
|
3119
|
-
return
|
|
3793
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_generic_parser(), generic_parser_exports));
|
|
3794
|
+
return parseSymbols9({ file, content, lang });
|
|
3120
3795
|
}
|
|
3121
3796
|
}
|
|
3122
3797
|
}
|
|
@@ -3128,10 +3803,185 @@ function withRelations(parsed, content, lang) {
|
|
|
3128
3803
|
return { ...parsed, refs: refs.map((ref) => ref.lang ? ref : { ...ref, lang }) };
|
|
3129
3804
|
}
|
|
3130
3805
|
|
|
3806
|
+
// src/codebase-index/parser-worker-pool.ts
|
|
3807
|
+
import { Worker } from "node:worker_threads";
|
|
3808
|
+
import { fileURLToPath as fileURLToPath2 } from "node:url";
|
|
3809
|
+
import * as fs6 from "node:fs";
|
|
3810
|
+
var WORKER_POOL_THRESHOLD = 500;
|
|
3811
|
+
var ParserWorkerPool = class {
|
|
3812
|
+
constructor(maxWorkers = defaultWorkerCount()) {
|
|
3813
|
+
this.maxWorkers = maxWorkers;
|
|
3814
|
+
}
|
|
3815
|
+
maxWorkers;
|
|
3816
|
+
workers = [];
|
|
3817
|
+
nextBatchId = 1;
|
|
3818
|
+
pending = /* @__PURE__ */ new Map();
|
|
3819
|
+
creating = false;
|
|
3820
|
+
unavailable = false;
|
|
3821
|
+
/**
|
|
3822
|
+
* True if the pool is available for use. Returns false when:
|
|
3823
|
+
* - Worker threads aren't supported (sandbox, exotic runtime)
|
|
3824
|
+
* - The built worker script can't be found
|
|
3825
|
+
* - Pool creation was attempted and failed
|
|
3826
|
+
*/
|
|
3827
|
+
isAvailable() {
|
|
3828
|
+
return !this.unavailable && this.workers.length > 0;
|
|
3829
|
+
}
|
|
3830
|
+
/**
|
|
3831
|
+
* Lazily create the worker pool. Returns true if the pool is ready, false
|
|
3832
|
+
* if it's unavailable (caller should fall back to inline parsing).
|
|
3833
|
+
*/
|
|
3834
|
+
async ensureReady() {
|
|
3835
|
+
if (this.isAvailable()) return true;
|
|
3836
|
+
if (this.unavailable) return false;
|
|
3837
|
+
if (this.creating) {
|
|
3838
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
3839
|
+
return this.isAvailable();
|
|
3840
|
+
}
|
|
3841
|
+
this.creating = true;
|
|
3842
|
+
try {
|
|
3843
|
+
const url = resolveWorkerScriptUrl();
|
|
3844
|
+
if (!url) {
|
|
3845
|
+
this.unavailable = true;
|
|
3846
|
+
return false;
|
|
3847
|
+
}
|
|
3848
|
+
for (let i = 0; i < this.maxWorkers; i++) {
|
|
3849
|
+
try {
|
|
3850
|
+
const w = new Worker(url, { name: `wstack-parser-${i}` });
|
|
3851
|
+
w.unref();
|
|
3852
|
+
w.on("message", (msg) => this.handleMessage(msg));
|
|
3853
|
+
w.on("error", (err) => this.handleError(err, w));
|
|
3854
|
+
this.workers.push({ worker: w, busy: false });
|
|
3855
|
+
} catch {
|
|
3856
|
+
if (this.workers.length === 0) {
|
|
3857
|
+
this.unavailable = true;
|
|
3858
|
+
return false;
|
|
3859
|
+
}
|
|
3860
|
+
break;
|
|
3861
|
+
}
|
|
3862
|
+
}
|
|
3863
|
+
return this.workers.length > 0;
|
|
3864
|
+
} finally {
|
|
3865
|
+
this.creating = false;
|
|
3866
|
+
}
|
|
3867
|
+
}
|
|
3868
|
+
/**
|
|
3869
|
+
* Parse files in parallel across the worker pool. Returns a flat
|
|
3870
|
+
* `FileSymbols[]` in completion order (caller sorts if needed).
|
|
3871
|
+
*
|
|
3872
|
+
* Content is pre-read by the main thread (for the content-hash check)
|
|
3873
|
+
* and passed to workers to avoid a second disk read. Files are
|
|
3874
|
+
* distributed round-robin across workers.
|
|
3875
|
+
*/
|
|
3876
|
+
async parseFiles(files) {
|
|
3877
|
+
if (!this.isAvailable()) {
|
|
3878
|
+
throw new Error("ParserWorkerPool.parseFiles called before ensureReady() succeeded");
|
|
3879
|
+
}
|
|
3880
|
+
if (files.length === 0) return [];
|
|
3881
|
+
const batchId = this.nextBatchId++;
|
|
3882
|
+
const workerCount = Math.min(this.workers.length, files.length);
|
|
3883
|
+
const chunks = Array.from(
|
|
3884
|
+
{ length: workerCount },
|
|
3885
|
+
() => []
|
|
3886
|
+
);
|
|
3887
|
+
for (let i = 0; i < files.length; i++) {
|
|
3888
|
+
chunks[i % workerCount].push(files[i]);
|
|
3889
|
+
}
|
|
3890
|
+
return new Promise((resolve4, reject) => {
|
|
3891
|
+
this.pending.set(batchId, {
|
|
3892
|
+
resolve: resolve4,
|
|
3893
|
+
reject,
|
|
3894
|
+
accumulated: [],
|
|
3895
|
+
expectedWorkers: workerCount,
|
|
3896
|
+
completedWorkers: 0
|
|
3897
|
+
});
|
|
3898
|
+
for (let i = 0; i < workerCount; i++) {
|
|
3899
|
+
const pw = this.workers[i];
|
|
3900
|
+
pw.busy = true;
|
|
3901
|
+
pw.worker.postMessage({
|
|
3902
|
+
type: "parse",
|
|
3903
|
+
id: batchId,
|
|
3904
|
+
files: chunks[i]
|
|
3905
|
+
});
|
|
3906
|
+
}
|
|
3907
|
+
});
|
|
3908
|
+
}
|
|
3909
|
+
/** Shut down all workers. Safe to call multiple times. */
|
|
3910
|
+
async shutdown() {
|
|
3911
|
+
const workers = this.workers.map((w) => w.worker);
|
|
3912
|
+
this.workers = [];
|
|
3913
|
+
this.unavailable = false;
|
|
3914
|
+
for (const w of workers) {
|
|
3915
|
+
try {
|
|
3916
|
+
w.postMessage({ type: "shutdown" });
|
|
3917
|
+
} catch {
|
|
3918
|
+
}
|
|
3919
|
+
}
|
|
3920
|
+
await Promise.allSettled(
|
|
3921
|
+
workers.map(
|
|
3922
|
+
(w) => Promise.race([
|
|
3923
|
+
new Promise((resolve4) => {
|
|
3924
|
+
w.once("exit", () => resolve4());
|
|
3925
|
+
}),
|
|
3926
|
+
new Promise((resolve4) => setTimeout(() => resolve4(), 2e3))
|
|
3927
|
+
]).then(() => {
|
|
3928
|
+
if (!w.threadId) return;
|
|
3929
|
+
return w.terminate().catch(() => {
|
|
3930
|
+
});
|
|
3931
|
+
})
|
|
3932
|
+
)
|
|
3933
|
+
);
|
|
3934
|
+
for (const [, p] of this.pending) p.reject(new Error("ParserWorkerPool shut down"));
|
|
3935
|
+
this.pending.clear();
|
|
3936
|
+
}
|
|
3937
|
+
handleMessage(msg) {
|
|
3938
|
+
const batch = this.pending.get(msg.id);
|
|
3939
|
+
if (!batch) return;
|
|
3940
|
+
batch.accumulated.push(...msg.results);
|
|
3941
|
+
batch.completedWorkers++;
|
|
3942
|
+
const freeWorker = this.workers.find((w) => w.busy);
|
|
3943
|
+
if (freeWorker) freeWorker.busy = false;
|
|
3944
|
+
if (batch.completedWorkers >= batch.expectedWorkers) {
|
|
3945
|
+
this.pending.delete(msg.id);
|
|
3946
|
+
batch.resolve(batch.accumulated);
|
|
3947
|
+
}
|
|
3948
|
+
}
|
|
3949
|
+
handleError(err, source) {
|
|
3950
|
+
this.workers = this.workers.filter((w) => w.worker !== source);
|
|
3951
|
+
if (this.workers.length === 0) {
|
|
3952
|
+
for (const [, p] of this.pending) p.reject(err);
|
|
3953
|
+
this.pending.clear();
|
|
3954
|
+
this.unavailable = true;
|
|
3955
|
+
}
|
|
3956
|
+
}
|
|
3957
|
+
};
|
|
3958
|
+
function defaultWorkerCount() {
|
|
3959
|
+
const cores = globalThis.navigator?.hardwareConcurrency ?? 4;
|
|
3960
|
+
return Math.max(1, Math.min(4, cores - 1));
|
|
3961
|
+
}
|
|
3962
|
+
function resolveWorkerScriptUrl() {
|
|
3963
|
+
for (const rel of [
|
|
3964
|
+
"./parser-worker-script.js",
|
|
3965
|
+
"./codebase-index/parser-worker-script.js"
|
|
3966
|
+
]) {
|
|
3967
|
+
try {
|
|
3968
|
+
const url = new URL(rel, import.meta.url);
|
|
3969
|
+
if (url.protocol === "file:" && fs6.existsSync(fileURLToPath2(url))) return url;
|
|
3970
|
+
} catch {
|
|
3971
|
+
}
|
|
3972
|
+
}
|
|
3973
|
+
return null;
|
|
3974
|
+
}
|
|
3975
|
+
var _pool = null;
|
|
3976
|
+
function getParserPool() {
|
|
3977
|
+
_pool ??= new ParserWorkerPool();
|
|
3978
|
+
return _pool;
|
|
3979
|
+
}
|
|
3980
|
+
|
|
3131
3981
|
// src/codebase-index/writer.ts
|
|
3132
3982
|
import { expectDefined as expectDefined4 } from "@wrongstack/core/utils";
|
|
3133
|
-
import * as
|
|
3134
|
-
import * as
|
|
3983
|
+
import * as fs8 from "node:fs";
|
|
3984
|
+
import * as path12 from "node:path";
|
|
3135
3985
|
|
|
3136
3986
|
// src/codebase-index/bm25.ts
|
|
3137
3987
|
var K1 = 1.5;
|
|
@@ -3329,8 +4179,8 @@ function runSqliteWithRetry(fn) {
|
|
|
3329
4179
|
}
|
|
3330
4180
|
|
|
3331
4181
|
// src/codebase-index/writer-admin.ts
|
|
3332
|
-
import * as
|
|
3333
|
-
import * as
|
|
4182
|
+
import * as fs7 from "node:fs";
|
|
4183
|
+
import * as path11 from "node:path";
|
|
3334
4184
|
var DB_FILE = "index.db";
|
|
3335
4185
|
function getAllIndexableWithStatement(stmt) {
|
|
3336
4186
|
return stmt("SELECT id, text FROM symbols").all().map(({ id, text }) => ({ id, text }));
|
|
@@ -3366,7 +4216,7 @@ function getMetadataWithStatement(stmt, key) {
|
|
|
3366
4216
|
}
|
|
3367
4217
|
function getFileMetaWithStatement(stmt, file) {
|
|
3368
4218
|
const rows = stmt(
|
|
3369
|
-
"SELECT file, lang, mtime_ms, symbol_count, last_indexed FROM files WHERE file = ?"
|
|
4219
|
+
"SELECT file, lang, mtime_ms, symbol_count, last_indexed, content_hash FROM files WHERE file = ?"
|
|
3370
4220
|
).all(file);
|
|
3371
4221
|
const r = rows[0];
|
|
3372
4222
|
if (!r) return null;
|
|
@@ -3375,21 +4225,25 @@ function getFileMetaWithStatement(stmt, file) {
|
|
|
3375
4225
|
lang: r.lang,
|
|
3376
4226
|
mtimeMs: r.mtime_ms,
|
|
3377
4227
|
symbolCount: r.symbol_count,
|
|
3378
|
-
lastIndexed: r.last_indexed
|
|
4228
|
+
lastIndexed: r.last_indexed,
|
|
4229
|
+
contentHash: r.content_hash
|
|
3379
4230
|
};
|
|
3380
4231
|
}
|
|
3381
4232
|
function getAllFileMetasWithStatement(stmt) {
|
|
3382
|
-
return stmt(
|
|
4233
|
+
return stmt(
|
|
4234
|
+
"SELECT file, lang, mtime_ms, symbol_count, last_indexed, content_hash FROM files"
|
|
4235
|
+
).all().map((r) => ({
|
|
3383
4236
|
file: r.file,
|
|
3384
4237
|
lang: r.lang,
|
|
3385
4238
|
mtimeMs: r.mtime_ms,
|
|
3386
4239
|
symbolCount: r.symbol_count,
|
|
3387
|
-
lastIndexed: r.last_indexed
|
|
4240
|
+
lastIndexed: r.last_indexed,
|
|
4241
|
+
contentHash: r.content_hash
|
|
3388
4242
|
}));
|
|
3389
4243
|
}
|
|
3390
4244
|
function getIndexDbSizeBytes(indexDir) {
|
|
3391
4245
|
try {
|
|
3392
|
-
return
|
|
4246
|
+
return fs7.statSync(path11.join(indexDir, DB_FILE)).size;
|
|
3393
4247
|
} catch {
|
|
3394
4248
|
return 0;
|
|
3395
4249
|
}
|
|
@@ -3438,6 +4292,18 @@ function bulkInsertFtsWithStatement(stmt, maxSqlVars, ftsAvailable, rows) {
|
|
|
3438
4292
|
insert.run(...binds);
|
|
3439
4293
|
}
|
|
3440
4294
|
}
|
|
4295
|
+
function bulkInsertVectorsWithStatement(stmt, maxSqlVars, rows) {
|
|
4296
|
+
if (rows.length === 0) return;
|
|
4297
|
+
const chunkSize = Math.max(1, Math.floor(maxSqlVars / 2));
|
|
4298
|
+
for (let i = 0; i < rows.length; i += chunkSize) {
|
|
4299
|
+
const chunk = rows.slice(i, i + chunkSize);
|
|
4300
|
+
const placeholders = chunk.map(() => "(?, ?)").join(", ");
|
|
4301
|
+
const insert = stmt(`INSERT INTO symbol_vectors(symbol_id, vector) VALUES ${placeholders}`);
|
|
4302
|
+
const binds = [];
|
|
4303
|
+
for (const r of chunk) binds.push(r.id, r.vector);
|
|
4304
|
+
insert.run(...binds);
|
|
4305
|
+
}
|
|
4306
|
+
}
|
|
3441
4307
|
function bulkInsertRefsWithStatement(stmt, maxSqlVars, refs) {
|
|
3442
4308
|
if (refs.length === 0) return;
|
|
3443
4309
|
const chunkSize = Math.max(1, Math.floor(maxSqlVars / 8));
|
|
@@ -3742,6 +4608,171 @@ function findOutgoingCallsByName(stmt, symbolName, file, limit) {
|
|
|
3742
4608
|
const calls = rows.map(mapCallSiteRow).slice(0, limit);
|
|
3743
4609
|
return { calls, symbolFound: true, unresolvedCount, totalMatches: rows.length };
|
|
3744
4610
|
}
|
|
4611
|
+
function runCteWithSeeds(stmt, seedIds, buildSql) {
|
|
4612
|
+
if (seedIds.length <= 900) {
|
|
4613
|
+
const ph = seedIds.map(() => "?").join(",");
|
|
4614
|
+
return stmt(buildSql(ph)).all(...seedIds);
|
|
4615
|
+
}
|
|
4616
|
+
stmt("DROP TABLE IF EXISTS _cte_seeds").run();
|
|
4617
|
+
try {
|
|
4618
|
+
stmt("CREATE TEMP TABLE _cte_seeds (id INTEGER PRIMARY KEY)").run();
|
|
4619
|
+
for (let i = 0; i < seedIds.length; i += 500) {
|
|
4620
|
+
const chunk = seedIds.slice(i, i + 500);
|
|
4621
|
+
const ph = chunk.map(() => "(?)").join(",");
|
|
4622
|
+
stmt(`INSERT OR IGNORE INTO _cte_seeds (id) VALUES ${ph}`).run(...chunk);
|
|
4623
|
+
}
|
|
4624
|
+
return stmt(buildSql("SELECT id FROM _cte_seeds")).all();
|
|
4625
|
+
} finally {
|
|
4626
|
+
stmt("DROP TABLE IF EXISTS _cte_seeds").run();
|
|
4627
|
+
}
|
|
4628
|
+
}
|
|
4629
|
+
function findTransitiveIncomingCallsByName(stmt, symbolName, file, limit) {
|
|
4630
|
+
const targetIds = resolveSymbolIds(stmt, symbolName, file);
|
|
4631
|
+
if (targetIds.length === 0)
|
|
4632
|
+
return { calls: [], symbolFound: false, ambiguous: false, totalMatches: 0 };
|
|
4633
|
+
let matchIds = targetIds;
|
|
4634
|
+
let ambiguous = false;
|
|
4635
|
+
if (file !== void 0) {
|
|
4636
|
+
const allNamedIds = resolveSymbolIds(stmt, symbolName, void 0);
|
|
4637
|
+
if (allNamedIds.length > targetIds.length) {
|
|
4638
|
+
matchIds = allNamedIds;
|
|
4639
|
+
ambiguous = true;
|
|
4640
|
+
}
|
|
4641
|
+
}
|
|
4642
|
+
const cteSql = (seedSource) => `WITH RECURSIVE incoming_tree(from_id) AS (
|
|
4643
|
+
SELECT r.from_id
|
|
4644
|
+
FROM refs r
|
|
4645
|
+
WHERE r.to_id IN (${seedSource})
|
|
4646
|
+
|
|
4647
|
+
UNION
|
|
4648
|
+
|
|
4649
|
+
SELECT r.from_id
|
|
4650
|
+
FROM refs r
|
|
4651
|
+
JOIN incoming_tree it ON r.to_id = it.from_id
|
|
4652
|
+
)
|
|
4653
|
+
SELECT
|
|
4654
|
+
s.id AS sym_id,
|
|
4655
|
+
s.name AS sym_name,
|
|
4656
|
+
s.kind AS sym_kind,
|
|
4657
|
+
s.lang AS sym_lang,
|
|
4658
|
+
s.file AS sym_file,
|
|
4659
|
+
s.line AS sym_line,
|
|
4660
|
+
s.signature AS sym_signature,
|
|
4661
|
+
'' AS call_type,
|
|
4662
|
+
0 AS ref_line
|
|
4663
|
+
FROM incoming_tree it
|
|
4664
|
+
JOIN symbols s ON s.id = it.from_id
|
|
4665
|
+
GROUP BY s.id
|
|
4666
|
+
ORDER BY s.file, s.line`;
|
|
4667
|
+
const rows = runCteWithSeeds(stmt, matchIds, cteSql);
|
|
4668
|
+
if (!file) {
|
|
4669
|
+
const fallbackRows = stmt(
|
|
4670
|
+
`SELECT
|
|
4671
|
+
s.id AS sym_id,
|
|
4672
|
+
s.name AS sym_name,
|
|
4673
|
+
s.kind AS sym_kind,
|
|
4674
|
+
s.lang AS sym_lang,
|
|
4675
|
+
s.file AS sym_file,
|
|
4676
|
+
s.line AS sym_line,
|
|
4677
|
+
s.signature AS sym_signature,
|
|
4678
|
+
r.call_type,
|
|
4679
|
+
r.line AS ref_line
|
|
4680
|
+
FROM refs r
|
|
4681
|
+
JOIN symbols s ON s.id = r.from_id
|
|
4682
|
+
WHERE r.to_id IS NULL AND r.to_name = ?
|
|
4683
|
+
ORDER BY r.line, r.id`
|
|
4684
|
+
).all(symbolName);
|
|
4685
|
+
rows.push(...fallbackRows);
|
|
4686
|
+
}
|
|
4687
|
+
rows.sort((a, b) => a.ref_line - b.ref_line || a.sym_id - b.sym_id);
|
|
4688
|
+
const allCalls = rows.map(mapCallSiteRow);
|
|
4689
|
+
return {
|
|
4690
|
+
calls: allCalls.slice(0, limit),
|
|
4691
|
+
symbolFound: true,
|
|
4692
|
+
ambiguous,
|
|
4693
|
+
totalMatches: allCalls.length
|
|
4694
|
+
};
|
|
4695
|
+
}
|
|
4696
|
+
function findTransitiveOutgoingCallsByName(stmt, symbolName, file, limit) {
|
|
4697
|
+
const sourceIds = resolveSymbolIds(stmt, symbolName, file);
|
|
4698
|
+
if (sourceIds.length === 0)
|
|
4699
|
+
return { calls: [], symbolFound: false, unresolvedCount: 0, totalMatches: 0 };
|
|
4700
|
+
const unresolvedCount = chunkedIdScalar(
|
|
4701
|
+
stmt,
|
|
4702
|
+
sourceIds,
|
|
4703
|
+
(ph) => `SELECT COUNT(*) AS n FROM refs WHERE from_id IN (${ph}) AND to_id IS NULL`
|
|
4704
|
+
);
|
|
4705
|
+
const cteSql = (seedSource) => `WITH RECURSIVE outgoing_tree(to_id) AS (
|
|
4706
|
+
SELECT r.to_id
|
|
4707
|
+
FROM refs r
|
|
4708
|
+
WHERE r.from_id IN (${seedSource}) AND r.to_id IS NOT NULL
|
|
4709
|
+
|
|
4710
|
+
UNION
|
|
4711
|
+
|
|
4712
|
+
SELECT r.to_id
|
|
4713
|
+
FROM refs r
|
|
4714
|
+
JOIN outgoing_tree ot ON r.from_id = ot.to_id
|
|
4715
|
+
WHERE r.to_id IS NOT NULL
|
|
4716
|
+
)
|
|
4717
|
+
SELECT
|
|
4718
|
+
s.id AS sym_id,
|
|
4719
|
+
s.name AS sym_name,
|
|
4720
|
+
s.kind AS sym_kind,
|
|
4721
|
+
s.lang AS sym_lang,
|
|
4722
|
+
s.file AS sym_file,
|
|
4723
|
+
s.line AS sym_line,
|
|
4724
|
+
s.signature AS sym_signature,
|
|
4725
|
+
'' AS call_type,
|
|
4726
|
+
0 AS ref_line
|
|
4727
|
+
FROM outgoing_tree ot
|
|
4728
|
+
JOIN symbols s ON s.id = ot.to_id
|
|
4729
|
+
GROUP BY s.id
|
|
4730
|
+
ORDER BY s.file, s.line`;
|
|
4731
|
+
const rows = runCteWithSeeds(stmt, sourceIds, cteSql);
|
|
4732
|
+
const calls = rows.map(mapCallSiteRow).slice(0, limit);
|
|
4733
|
+
return { calls, symbolFound: true, unresolvedCount, totalMatches: rows.length };
|
|
4734
|
+
}
|
|
4735
|
+
function findReachableSymbolIds(stmt, seedIds) {
|
|
4736
|
+
if (seedIds.length === 0) return /* @__PURE__ */ new Set();
|
|
4737
|
+
if (seedIds.length > 900) {
|
|
4738
|
+
stmt("DROP TABLE IF EXISTS _seeds").run();
|
|
4739
|
+
try {
|
|
4740
|
+
stmt("CREATE TEMP TABLE _seeds (id INTEGER PRIMARY KEY)").run();
|
|
4741
|
+
for (let i = 0; i < seedIds.length; i += 500) {
|
|
4742
|
+
const chunk = seedIds.slice(i, i + 500);
|
|
4743
|
+
const ph2 = chunk.map(() => "(?)").join(",");
|
|
4744
|
+
stmt(`INSERT OR IGNORE INTO _seeds (id) VALUES ${ph2}`).run(...chunk);
|
|
4745
|
+
}
|
|
4746
|
+
const rows2 = stmt(
|
|
4747
|
+
`WITH RECURSIVE reachable(id) AS (
|
|
4748
|
+
SELECT id FROM _seeds
|
|
4749
|
+
UNION
|
|
4750
|
+
SELECT r.to_id
|
|
4751
|
+
FROM refs r
|
|
4752
|
+
JOIN reachable ON r.from_id = reachable.id
|
|
4753
|
+
WHERE r.to_id IS NOT NULL
|
|
4754
|
+
)
|
|
4755
|
+
SELECT DISTINCT id FROM reachable`
|
|
4756
|
+
).all();
|
|
4757
|
+
return new Set(rows2.map((r) => r.id));
|
|
4758
|
+
} finally {
|
|
4759
|
+
stmt("DROP TABLE IF EXISTS _seeds").run();
|
|
4760
|
+
}
|
|
4761
|
+
}
|
|
4762
|
+
const ph = seedIds.map(() => "?").join(",");
|
|
4763
|
+
const rows = stmt(
|
|
4764
|
+
`WITH RECURSIVE reachable(id) AS (
|
|
4765
|
+
SELECT id FROM symbols WHERE id IN (${ph})
|
|
4766
|
+
UNION
|
|
4767
|
+
SELECT r.to_id
|
|
4768
|
+
FROM refs r
|
|
4769
|
+
JOIN reachable ON r.from_id = reachable.id
|
|
4770
|
+
WHERE r.to_id IS NOT NULL
|
|
4771
|
+
)
|
|
4772
|
+
SELECT DISTINCT id FROM reachable`
|
|
4773
|
+
).all(...seedIds);
|
|
4774
|
+
return new Set(rows.map((r) => r.id));
|
|
4775
|
+
}
|
|
3745
4776
|
function findRefsToWithStatement(stmt, symbolId) {
|
|
3746
4777
|
return stmt(
|
|
3747
4778
|
"SELECT id, from_id, to_name, to_id, call_type, line FROM refs WHERE to_id = ? OR to_name = (SELECT name FROM symbols WHERE id = ?)"
|
|
@@ -3985,6 +5016,12 @@ var CORE_TABLES_SQL = `
|
|
|
3985
5016
|
file TEXT PRIMARY KEY,
|
|
3986
5017
|
lang TEXT NOT NULL,
|
|
3987
5018
|
mtime_ms INTEGER NOT NULL,
|
|
5019
|
+
-- Phase 2: xxHash64 of the file's UTF-8 bytes. Empty string when the
|
|
5020
|
+
-- indexer hasn't populated it yet (legacy rows, schema repaired by
|
|
5021
|
+
-- repairMissingColumns). Compared on incremental re-index so that a
|
|
5022
|
+
-- touch or branch-switch that leaves content byte-identical skips the
|
|
5023
|
+
-- expensive parse phase entirely (refactoring proposal Phase 2).
|
|
5024
|
+
content_hash TEXT NOT NULL DEFAULT '',
|
|
3988
5025
|
symbol_count INTEGER NOT NULL DEFAULT 0,
|
|
3989
5026
|
last_indexed INTEGER NOT NULL,
|
|
3990
5027
|
-- Code Atlas grouping label, computed at index time from the ecosystem's
|
|
@@ -4052,7 +5089,14 @@ var LANG_FAMILY_TABLE_SQL = `
|
|
|
4052
5089
|
);
|
|
4053
5090
|
`;
|
|
4054
5091
|
var LANG_FAMILY_WILDCARD = "*";
|
|
4055
|
-
var SYMBOLS_FTS_SQL = "CREATE VIRTUAL TABLE IF NOT EXISTS symbols_fts USING fts5(text, tokenize = '
|
|
5092
|
+
var SYMBOLS_FTS_SQL = "CREATE VIRTUAL TABLE IF NOT EXISTS symbols_fts USING fts5(text, tokenize = 'trigram')";
|
|
5093
|
+
var SYMBOL_VECTORS_TABLE_SQL = `
|
|
5094
|
+
CREATE TABLE IF NOT EXISTS symbol_vectors (
|
|
5095
|
+
symbol_id INTEGER PRIMARY KEY,
|
|
5096
|
+
vector BLOB NOT NULL,
|
|
5097
|
+
FOREIGN KEY (symbol_id) REFERENCES symbols(id) ON DELETE CASCADE
|
|
5098
|
+
);
|
|
5099
|
+
`;
|
|
4056
5100
|
|
|
4057
5101
|
// src/codebase-index/writer-search-helpers.ts
|
|
4058
5102
|
var SEARCH_CANDIDATE_SCAN_CAP = 5e3;
|
|
@@ -4193,6 +5237,85 @@ var StorePool = class {
|
|
|
4193
5237
|
}
|
|
4194
5238
|
};
|
|
4195
5239
|
|
|
5240
|
+
// src/codebase-index/vector-search.ts
|
|
5241
|
+
var RRF_K = 60;
|
|
5242
|
+
var VECTOR_DIMENSIONS = 384;
|
|
5243
|
+
var NGRAM_SIZE = 3;
|
|
5244
|
+
function embedText(text) {
|
|
5245
|
+
const vec = new Float32Array(VECTOR_DIMENSIONS);
|
|
5246
|
+
const normalized = text.toLowerCase().trim();
|
|
5247
|
+
if (normalized.length < NGRAM_SIZE) {
|
|
5248
|
+
const padded = ` ${normalized} `.slice(0, Math.max(NGRAM_SIZE, normalized.length + 2));
|
|
5249
|
+
for (let i = 0; i <= padded.length - NGRAM_SIZE; i++) {
|
|
5250
|
+
const ngram = padded.slice(i, i + NGRAM_SIZE);
|
|
5251
|
+
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
5252
|
+
vec[bucket] += 1;
|
|
5253
|
+
}
|
|
5254
|
+
} else {
|
|
5255
|
+
for (let i = 0; i <= normalized.length - NGRAM_SIZE; i++) {
|
|
5256
|
+
const ngram = normalized.slice(i, i + NGRAM_SIZE);
|
|
5257
|
+
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
5258
|
+
vec[bucket] += 1;
|
|
5259
|
+
}
|
|
5260
|
+
}
|
|
5261
|
+
let norm = 0;
|
|
5262
|
+
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
5263
|
+
norm += vec[i] * vec[i];
|
|
5264
|
+
}
|
|
5265
|
+
norm = Math.sqrt(norm);
|
|
5266
|
+
if (norm > 0) {
|
|
5267
|
+
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
5268
|
+
vec[i] /= norm;
|
|
5269
|
+
}
|
|
5270
|
+
}
|
|
5271
|
+
return vec;
|
|
5272
|
+
}
|
|
5273
|
+
function hashNgram(str) {
|
|
5274
|
+
let hash = 2166136261;
|
|
5275
|
+
for (let i = 0; i < str.length; i++) {
|
|
5276
|
+
hash ^= str.charCodeAt(i);
|
|
5277
|
+
hash = Math.imul(hash, 16777619);
|
|
5278
|
+
}
|
|
5279
|
+
return hash >>> 0;
|
|
5280
|
+
}
|
|
5281
|
+
function cosineSimilarity(a, b) {
|
|
5282
|
+
let dot = 0;
|
|
5283
|
+
const len = Math.min(a.length, b.length);
|
|
5284
|
+
for (let i = 0; i < len; i++) {
|
|
5285
|
+
dot += a[i] * b[i];
|
|
5286
|
+
}
|
|
5287
|
+
return dot;
|
|
5288
|
+
}
|
|
5289
|
+
function encodeVector(vec) {
|
|
5290
|
+
return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
|
|
5291
|
+
}
|
|
5292
|
+
function decodeVector(buf) {
|
|
5293
|
+
const view = new DataView(
|
|
5294
|
+
buf.buffer,
|
|
5295
|
+
buf.byteOffset,
|
|
5296
|
+
buf.byteLength
|
|
5297
|
+
);
|
|
5298
|
+
const copy = new Float32Array(buf.byteLength / 4);
|
|
5299
|
+
for (let i = 0; i < copy.length; i++) {
|
|
5300
|
+
copy[i] = view.getFloat32(i * 4, true);
|
|
5301
|
+
}
|
|
5302
|
+
return copy;
|
|
5303
|
+
}
|
|
5304
|
+
function reciprocalRankFusion(bm25Ranks, vectorRanks, k = RRF_K) {
|
|
5305
|
+
const allIds = /* @__PURE__ */ new Set([...bm25Ranks.keys(), ...vectorRanks.keys()]);
|
|
5306
|
+
const scored = [];
|
|
5307
|
+
for (const id of allIds) {
|
|
5308
|
+
const bm25Rank = bm25Ranks.get(id);
|
|
5309
|
+
const vecRank = vectorRanks.get(id);
|
|
5310
|
+
let score = 0;
|
|
5311
|
+
if (bm25Rank !== void 0) score += 1 / (k + bm25Rank);
|
|
5312
|
+
if (vecRank !== void 0) score += 1 / (k + vecRank);
|
|
5313
|
+
scored.push([id, score]);
|
|
5314
|
+
}
|
|
5315
|
+
scored.sort((a, b) => b[1] - a[1]);
|
|
5316
|
+
return scored;
|
|
5317
|
+
}
|
|
5318
|
+
|
|
4196
5319
|
// src/codebase-index/writer.ts
|
|
4197
5320
|
var DB_FILE2 = "index.db";
|
|
4198
5321
|
var MAX_STATEMENT_CACHE = 128;
|
|
@@ -4205,6 +5328,12 @@ var IndexStore = class _IndexStore {
|
|
|
4205
5328
|
* When false, ranked search falls back to the LIKE + in-process BM25 path.
|
|
4206
5329
|
*/
|
|
4207
5330
|
ftsAvailable = false;
|
|
5331
|
+
/**
|
|
5332
|
+
* Phase 3: true when the `symbol_vectors` table was created successfully.
|
|
5333
|
+
* When false, hybrid search skips the vector pass and falls back to FTS5
|
|
5334
|
+
* (or LIKE) only.
|
|
5335
|
+
*/
|
|
5336
|
+
vectorsAvailable = false;
|
|
4208
5337
|
/**
|
|
4209
5338
|
* Cache of prepared statements keyed by their SQL text. `DatabaseSync`
|
|
4210
5339
|
* compiles SQL on every `.prepare()` call; for the fixed-SQL methods
|
|
@@ -4263,9 +5392,9 @@ var IndexStore = class _IndexStore {
|
|
|
4263
5392
|
}
|
|
4264
5393
|
constructor(projectRoot, opts = {}) {
|
|
4265
5394
|
this.indexDir = resolveIndexDir(projectRoot, opts.indexDir);
|
|
4266
|
-
|
|
5395
|
+
fs8.mkdirSync(this.indexDir, { recursive: true });
|
|
4267
5396
|
const Database = loadDatabaseSync();
|
|
4268
|
-
this.db = new Database(
|
|
5397
|
+
this.db = new Database(path12.join(this.indexDir, DB_FILE2));
|
|
4269
5398
|
applyIndexStorePragmas(this.db);
|
|
4270
5399
|
this.initSchema();
|
|
4271
5400
|
}
|
|
@@ -4303,7 +5432,13 @@ var IndexStore = class _IndexStore {
|
|
|
4303
5432
|
*/
|
|
4304
5433
|
repairMissingColumns() {
|
|
4305
5434
|
const expected = [
|
|
4306
|
-
{
|
|
5435
|
+
{
|
|
5436
|
+
table: "files",
|
|
5437
|
+
columns: [
|
|
5438
|
+
["package", "TEXT NOT NULL DEFAULT ''"],
|
|
5439
|
+
["content_hash", "TEXT NOT NULL DEFAULT ''"]
|
|
5440
|
+
]
|
|
5441
|
+
},
|
|
4307
5442
|
{
|
|
4308
5443
|
table: "refs",
|
|
4309
5444
|
columns: [
|
|
@@ -4335,6 +5470,7 @@ var IndexStore = class _IndexStore {
|
|
|
4335
5470
|
DROP TABLE IF EXISTS symbols;
|
|
4336
5471
|
DROP TABLE IF EXISTS files;
|
|
4337
5472
|
DROP TABLE IF EXISTS refs;
|
|
5473
|
+
DROP TABLE IF EXISTS symbol_vectors;
|
|
4338
5474
|
`);
|
|
4339
5475
|
this.db.exec("DROP TABLE IF EXISTS symbols_fts");
|
|
4340
5476
|
this.stmt("UPDATE metadata SET value = ? WHERE key = ?").run(
|
|
@@ -4356,6 +5492,12 @@ var IndexStore = class _IndexStore {
|
|
|
4356
5492
|
this.db.exec(LANG_FAMILY_TABLE_SQL);
|
|
4357
5493
|
this.seedLangFamilies();
|
|
4358
5494
|
try {
|
|
5495
|
+
const ftsSchema = this.stmt(
|
|
5496
|
+
"SELECT sql FROM sqlite_master WHERE type='table' AND name='symbols_fts'"
|
|
5497
|
+
).get();
|
|
5498
|
+
if (ftsSchema?.sql?.includes("unicode61")) {
|
|
5499
|
+
this.db.exec("DROP TABLE IF EXISTS symbols_fts");
|
|
5500
|
+
}
|
|
4359
5501
|
this.db.exec(SYMBOLS_FTS_SQL);
|
|
4360
5502
|
this.ftsAvailable = true;
|
|
4361
5503
|
const symbolCount = Number(
|
|
@@ -4366,6 +5508,7 @@ var IndexStore = class _IndexStore {
|
|
|
4366
5508
|
);
|
|
4367
5509
|
if (symbolCount !== ftsCount) {
|
|
4368
5510
|
this.db.exec("DELETE FROM symbols_fts");
|
|
5511
|
+
if (this.vectorsAvailable) this.db.exec("DELETE FROM symbol_vectors");
|
|
4369
5512
|
const rows = this.stmt(
|
|
4370
5513
|
"SELECT id, name, signature, doc_comment FROM symbols ORDER BY id"
|
|
4371
5514
|
).all();
|
|
@@ -4383,6 +5526,12 @@ var IndexStore = class _IndexStore {
|
|
|
4383
5526
|
} catch {
|
|
4384
5527
|
this.ftsAvailable = false;
|
|
4385
5528
|
}
|
|
5529
|
+
try {
|
|
5530
|
+
this.db.exec(SYMBOL_VECTORS_TABLE_SQL);
|
|
5531
|
+
this.vectorsAvailable = true;
|
|
5532
|
+
} catch {
|
|
5533
|
+
this.vectorsAvailable = false;
|
|
5534
|
+
}
|
|
4386
5535
|
this.ensureNextSymbolIdSeeded();
|
|
4387
5536
|
}
|
|
4388
5537
|
// ─── ID allocation & bulk helpers ────────────────────────────────────────────
|
|
@@ -4492,6 +5641,7 @@ var IndexStore = class _IndexStore {
|
|
|
4492
5641
|
const result = [];
|
|
4493
5642
|
const bulk = [];
|
|
4494
5643
|
const ftsRows = [];
|
|
5644
|
+
const vectorRows = [];
|
|
4495
5645
|
for (const s of symbols) {
|
|
4496
5646
|
const id = nextId++;
|
|
4497
5647
|
bulk.push({
|
|
@@ -4510,6 +5660,10 @@ var IndexStore = class _IndexStore {
|
|
|
4510
5660
|
if (this.ftsAvailable) {
|
|
4511
5661
|
ftsRows.push({ id, text: buildIndexableText(s.name, s.signature, s.docComment) });
|
|
4512
5662
|
}
|
|
5663
|
+
vectorRows.push({
|
|
5664
|
+
id,
|
|
5665
|
+
vector: encodeVector(embedText(s.text || buildIndexableText(s.name, s.signature, s.docComment)))
|
|
5666
|
+
});
|
|
4513
5667
|
result.push({ ...s, id });
|
|
4514
5668
|
}
|
|
4515
5669
|
bulkInsertSymbolsWithStatement((sql) => this.stmt(sql), _IndexStore.MAX_SQL_VARS, bulk);
|
|
@@ -4519,6 +5673,13 @@ var IndexStore = class _IndexStore {
|
|
|
4519
5673
|
this.ftsAvailable,
|
|
4520
5674
|
ftsRows
|
|
4521
5675
|
);
|
|
5676
|
+
if (this.vectorsAvailable) {
|
|
5677
|
+
bulkInsertVectorsWithStatement(
|
|
5678
|
+
(sql) => this.stmt(sql),
|
|
5679
|
+
_IndexStore.MAX_SQL_VARS,
|
|
5680
|
+
vectorRows
|
|
5681
|
+
);
|
|
5682
|
+
}
|
|
4522
5683
|
this.db.exec("COMMIT");
|
|
4523
5684
|
return result;
|
|
4524
5685
|
} catch (err) {
|
|
@@ -4538,6 +5699,11 @@ var IndexStore = class _IndexStore {
|
|
|
4538
5699
|
"DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
4539
5700
|
).run(file);
|
|
4540
5701
|
}
|
|
5702
|
+
if (this.vectorsAvailable) {
|
|
5703
|
+
this.stmt(
|
|
5704
|
+
"DELETE FROM symbol_vectors WHERE symbol_id IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
5705
|
+
).run(file);
|
|
5706
|
+
}
|
|
4541
5707
|
this.stmt("DELETE FROM symbols WHERE file_fk = ?").run(file);
|
|
4542
5708
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
4543
5709
|
this.db.exec("COMMIT");
|
|
@@ -4563,6 +5729,11 @@ var IndexStore = class _IndexStore {
|
|
|
4563
5729
|
"DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
4564
5730
|
).run(file);
|
|
4565
5731
|
}
|
|
5732
|
+
if (this.vectorsAvailable) {
|
|
5733
|
+
this.stmt(
|
|
5734
|
+
"DELETE FROM symbol_vectors WHERE symbol_id IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
5735
|
+
).run(file);
|
|
5736
|
+
}
|
|
4566
5737
|
this.stmt(
|
|
4567
5738
|
"DELETE FROM refs WHERE from_id IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
4568
5739
|
).run(file);
|
|
@@ -4580,14 +5751,22 @@ var IndexStore = class _IndexStore {
|
|
|
4580
5751
|
upsertFile(meta) {
|
|
4581
5752
|
this.runWithRetry(() => {
|
|
4582
5753
|
this.stmt(
|
|
4583
|
-
`INSERT INTO files(file, lang, mtime_ms, symbol_count, last_indexed)
|
|
4584
|
-
VALUES (?, ?, ?, ?, ?)
|
|
5754
|
+
`INSERT INTO files(file, lang, mtime_ms, content_hash, symbol_count, last_indexed)
|
|
5755
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
4585
5756
|
ON CONFLICT(file) DO UPDATE SET
|
|
4586
5757
|
lang = excluded.lang,
|
|
4587
5758
|
mtime_ms = excluded.mtime_ms,
|
|
5759
|
+
content_hash = excluded.content_hash,
|
|
4588
5760
|
symbol_count = excluded.symbol_count,
|
|
4589
5761
|
last_indexed = excluded.last_indexed`
|
|
4590
|
-
).run(
|
|
5762
|
+
).run(
|
|
5763
|
+
meta.file,
|
|
5764
|
+
meta.lang,
|
|
5765
|
+
meta.mtimeMs,
|
|
5766
|
+
meta.contentHash ?? "",
|
|
5767
|
+
meta.symbolCount,
|
|
5768
|
+
meta.lastIndexed
|
|
5769
|
+
);
|
|
4591
5770
|
});
|
|
4592
5771
|
}
|
|
4593
5772
|
getFileMeta(file) {
|
|
@@ -4754,9 +5933,18 @@ var IndexStore = class _IndexStore {
|
|
|
4754
5933
|
if (mapped === null) return { results: [], total: 0 };
|
|
4755
5934
|
effectiveKind = mapped;
|
|
4756
5935
|
}
|
|
4757
|
-
const
|
|
5936
|
+
const longTokens = tokens.filter((t) => t.length >= 3);
|
|
5937
|
+
const shortTokens = tokens.filter((t) => t.length < 3);
|
|
5938
|
+
if (longTokens.length === 0) {
|
|
5939
|
+
return this.searchRankedFallback(query, filter, safeLimit);
|
|
5940
|
+
}
|
|
5941
|
+
const match = longTokens.map((t) => `"${t.replaceAll('"', "")}"`).join(" OR ");
|
|
4758
5942
|
const conditions = ["symbols_fts MATCH ?"];
|
|
4759
5943
|
const values = [match];
|
|
5944
|
+
for (const shortTok of shortTokens) {
|
|
5945
|
+
conditions.push("s.text LIKE ? ESCAPE '\\'");
|
|
5946
|
+
values.push(`%${escapeLike(shortTok)}%`);
|
|
5947
|
+
}
|
|
4760
5948
|
if (effectiveKind) {
|
|
4761
5949
|
conditions.push("s.kind = ?");
|
|
4762
5950
|
values.push(effectiveKind);
|
|
@@ -4775,7 +5963,7 @@ var IndexStore = class _IndexStore {
|
|
|
4775
5963
|
).all(...values);
|
|
4776
5964
|
const total = countRows[0] ? Number(countRows[0].n) : 0;
|
|
4777
5965
|
if (total === 0) return { results: [], total: 0 };
|
|
4778
|
-
const
|
|
5966
|
+
const bm25Rows = this.stmt(
|
|
4779
5967
|
`SELECT s.id, s.lang, s.kind, s.name, s.file, s.line, s.col, s.signature, s.doc_comment,
|
|
4780
5968
|
-bm25(symbols_fts) AS score,
|
|
4781
5969
|
snippet(symbols_fts, 0, '', '', '\u2026', 12) AS snippet
|
|
@@ -4788,8 +5976,39 @@ var IndexStore = class _IndexStore {
|
|
|
4788
5976
|
bm25(symbols_fts), lower(s.name), s.file, s.line, s.col, s.id
|
|
4789
5977
|
LIMIT ?`
|
|
4790
5978
|
).all(...values, query.trim(), `${escapeLike(query.trim())}%`, safeLimit);
|
|
5979
|
+
if (this.vectorsAvailable && bm25Rows.length > 0) {
|
|
5980
|
+
const queryVec = embedText(query);
|
|
5981
|
+
const candidateIds = bm25Rows.map((r) => r.id);
|
|
5982
|
+
const placeholders = candidateIds.map(() => "?").join(",");
|
|
5983
|
+
const vecRows = this.stmt(
|
|
5984
|
+
`SELECT sv.symbol_id, sv.vector FROM symbol_vectors sv WHERE sv.symbol_id IN (${placeholders})`
|
|
5985
|
+
).all(...candidateIds);
|
|
5986
|
+
const vecScores = vecRows.map((r) => ({
|
|
5987
|
+
id: r.symbol_id,
|
|
5988
|
+
sim: cosineSimilarity(queryVec, decodeVector(r.vector))
|
|
5989
|
+
})).sort((a, b) => b.sim - a.sim);
|
|
5990
|
+
const bm25Rank = /* @__PURE__ */ new Map();
|
|
5991
|
+
bm25Rows.forEach((r, i) => {
|
|
5992
|
+
bm25Rank.set(r.id, i);
|
|
5993
|
+
});
|
|
5994
|
+
const vecRank = /* @__PURE__ */ new Map();
|
|
5995
|
+
vecScores.forEach((r, i) => {
|
|
5996
|
+
vecRank.set(r.id, i);
|
|
5997
|
+
});
|
|
5998
|
+
const fused = reciprocalRankFusion(bm25Rank, vecRank, 60);
|
|
5999
|
+
const fusedScore = new Map(fused);
|
|
6000
|
+
const sorted = [...bm25Rows].sort(
|
|
6001
|
+
(a, b) => (fusedScore.get(b.id) ?? 0) - (fusedScore.get(a.id) ?? 0)
|
|
6002
|
+
);
|
|
6003
|
+
return {
|
|
6004
|
+
results: sorted.map(
|
|
6005
|
+
(row) => mapWriterSearchRow(row, filter?.lspKind, Math.max(1e-4, row.score), row.snippet)
|
|
6006
|
+
),
|
|
6007
|
+
total
|
|
6008
|
+
};
|
|
6009
|
+
}
|
|
4791
6010
|
return {
|
|
4792
|
-
results:
|
|
6011
|
+
results: bm25Rows.map(
|
|
4793
6012
|
(row) => mapWriterSearchRow(row, filter?.lspKind, Math.max(1e-4, row.score), row.snippet)
|
|
4794
6013
|
),
|
|
4795
6014
|
total
|
|
@@ -4917,6 +6136,7 @@ var IndexStore = class _IndexStore {
|
|
|
4917
6136
|
this.db.exec("DROP TABLE IF EXISTS files");
|
|
4918
6137
|
this.db.exec("DROP TABLE IF EXISTS metadata");
|
|
4919
6138
|
if (this.ftsAvailable) this.db.exec("DROP TABLE IF EXISTS symbols_fts");
|
|
6139
|
+
this.db.exec("DROP TABLE IF EXISTS symbol_vectors");
|
|
4920
6140
|
this.db.exec("COMMIT");
|
|
4921
6141
|
this.stmtCache.clear();
|
|
4922
6142
|
this.initSchema();
|
|
@@ -5004,6 +6224,11 @@ var IndexStore = class _IndexStore {
|
|
|
5004
6224
|
`DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file IN (${placeholders}))`
|
|
5005
6225
|
).run(...options.deleteForFiles);
|
|
5006
6226
|
}
|
|
6227
|
+
if (this.vectorsAvailable) {
|
|
6228
|
+
this.stmt(
|
|
6229
|
+
`DELETE FROM symbol_vectors WHERE symbol_id IN (SELECT id FROM symbols WHERE file IN (${placeholders}))`
|
|
6230
|
+
).run(...options.deleteForFiles);
|
|
6231
|
+
}
|
|
5007
6232
|
this.stmt(
|
|
5008
6233
|
`DELETE FROM refs WHERE from_id IN (SELECT id FROM symbols WHERE file IN (${placeholders}))`
|
|
5009
6234
|
).run(...options.deleteForFiles);
|
|
@@ -5017,6 +6242,7 @@ var IndexStore = class _IndexStore {
|
|
|
5017
6242
|
const refsToInsert = [];
|
|
5018
6243
|
const bulkSyms = [];
|
|
5019
6244
|
const ftsRows = [];
|
|
6245
|
+
const vectorRows = [];
|
|
5020
6246
|
for (const entry of entries) {
|
|
5021
6247
|
const insertedForEntry = [];
|
|
5022
6248
|
for (const s of entry.symbols) {
|
|
@@ -5040,6 +6266,10 @@ var IndexStore = class _IndexStore {
|
|
|
5040
6266
|
text: buildIndexableText(s.name, s.signature, s.docComment)
|
|
5041
6267
|
});
|
|
5042
6268
|
}
|
|
6269
|
+
vectorRows.push({
|
|
6270
|
+
id,
|
|
6271
|
+
vector: encodeVector(embedText(s.text || buildIndexableText(s.name, s.signature, s.docComment)))
|
|
6272
|
+
});
|
|
5043
6273
|
const inserted = { ...s, id };
|
|
5044
6274
|
allInserted.push(inserted);
|
|
5045
6275
|
insertedForEntry.push(inserted);
|
|
@@ -5053,19 +6283,34 @@ var IndexStore = class _IndexStore {
|
|
|
5053
6283
|
this.ftsAvailable,
|
|
5054
6284
|
ftsRows
|
|
5055
6285
|
);
|
|
6286
|
+
if (this.vectorsAvailable) {
|
|
6287
|
+
bulkInsertVectorsWithStatement(
|
|
6288
|
+
(sql) => this.stmt(sql),
|
|
6289
|
+
_IndexStore.MAX_SQL_VARS,
|
|
6290
|
+
vectorRows
|
|
6291
|
+
);
|
|
6292
|
+
}
|
|
5056
6293
|
bulkInsertRefsWithStatement((sql) => this.stmt(sql), _IndexStore.MAX_SQL_VARS, refsToInsert);
|
|
5057
6294
|
const upsertStmt = this.stmt(
|
|
5058
|
-
`INSERT INTO files(file, lang, mtime_ms, symbol_count, last_indexed)
|
|
5059
|
-
VALUES (?, ?, ?, ?, ?)
|
|
6295
|
+
`INSERT INTO files(file, lang, mtime_ms, content_hash, symbol_count, last_indexed)
|
|
6296
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
5060
6297
|
ON CONFLICT(file) DO UPDATE SET
|
|
5061
6298
|
lang = excluded.lang,
|
|
5062
6299
|
mtime_ms = excluded.mtime_ms,
|
|
6300
|
+
content_hash = excluded.content_hash,
|
|
5063
6301
|
symbol_count = excluded.symbol_count,
|
|
5064
6302
|
last_indexed = excluded.last_indexed`
|
|
5065
6303
|
);
|
|
5066
6304
|
const now = Date.now();
|
|
5067
6305
|
for (const entry of entries) {
|
|
5068
|
-
upsertStmt.run(
|
|
6306
|
+
upsertStmt.run(
|
|
6307
|
+
entry.file,
|
|
6308
|
+
entry.lang,
|
|
6309
|
+
entry.mtimeMs,
|
|
6310
|
+
entry.contentHash ?? "",
|
|
6311
|
+
entry.symbolCount,
|
|
6312
|
+
now
|
|
6313
|
+
);
|
|
5069
6314
|
}
|
|
5070
6315
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
5071
6316
|
this.db.exec("COMMIT");
|
|
@@ -5156,19 +6401,32 @@ var IndexStore = class _IndexStore {
|
|
|
5156
6401
|
"DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
5157
6402
|
).run(meta.file);
|
|
5158
6403
|
}
|
|
6404
|
+
if (this.vectorsAvailable) {
|
|
6405
|
+
this.stmt(
|
|
6406
|
+
"DELETE FROM symbol_vectors WHERE symbol_id IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
6407
|
+
).run(meta.file);
|
|
6408
|
+
}
|
|
5159
6409
|
this.stmt(
|
|
5160
6410
|
"DELETE FROM refs WHERE from_id IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
5161
6411
|
).run(meta.file);
|
|
5162
6412
|
this.stmt("DELETE FROM symbols WHERE file_fk = ?").run(meta.file);
|
|
5163
6413
|
this.stmt(
|
|
5164
|
-
`INSERT INTO files(file, lang, mtime_ms, symbol_count, last_indexed)
|
|
5165
|
-
VALUES (?, ?, ?, ?, ?)
|
|
6414
|
+
`INSERT INTO files(file, lang, mtime_ms, content_hash, symbol_count, last_indexed)
|
|
6415
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
5166
6416
|
ON CONFLICT(file) DO UPDATE SET
|
|
5167
6417
|
lang = excluded.lang,
|
|
5168
6418
|
mtime_ms = excluded.mtime_ms,
|
|
6419
|
+
content_hash = excluded.content_hash,
|
|
5169
6420
|
symbol_count = excluded.symbol_count,
|
|
5170
6421
|
last_indexed = excluded.last_indexed`
|
|
5171
|
-
).run(
|
|
6422
|
+
).run(
|
|
6423
|
+
meta.file,
|
|
6424
|
+
meta.lang,
|
|
6425
|
+
meta.mtimeMs,
|
|
6426
|
+
meta.contentHash ?? "",
|
|
6427
|
+
meta.symbolCount,
|
|
6428
|
+
meta.lastIndexed
|
|
6429
|
+
);
|
|
5172
6430
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
5173
6431
|
this.db.exec("COMMIT");
|
|
5174
6432
|
} catch (err) {
|
|
@@ -5231,6 +6489,31 @@ var IndexStore = class _IndexStore {
|
|
|
5231
6489
|
findOutgoingCallsByName(symbolName, file, limit = 100) {
|
|
5232
6490
|
return findOutgoingCallsByName((sql) => this.stmt(sql), symbolName, file, limit);
|
|
5233
6491
|
}
|
|
6492
|
+
/**
|
|
6493
|
+
* Transitive incoming-call tree: all symbols that transitively call the
|
|
6494
|
+
* target, to an unbounded depth (cycle-safe via SQL UNION deduplication).
|
|
6495
|
+
* Used by `codebase-incoming-calls` when the caller wants the full call
|
|
6496
|
+
* chain rather than just direct callers.
|
|
6497
|
+
*/
|
|
6498
|
+
findTransitiveIncomingCallsByName(symbolName, file, limit = 200) {
|
|
6499
|
+
return findTransitiveIncomingCallsByName((sql) => this.stmt(sql), symbolName, file, limit);
|
|
6500
|
+
}
|
|
6501
|
+
/**
|
|
6502
|
+
* Transitive outgoing-call tree: all symbols the target transitively calls.
|
|
6503
|
+
* Used by `codebase-outgoing-calls` when the caller wants the full
|
|
6504
|
+
* dependency chain rather than just direct callees.
|
|
6505
|
+
*/
|
|
6506
|
+
findTransitiveOutgoingCallsByName(symbolName, file, limit = 200) {
|
|
6507
|
+
return findTransitiveOutgoingCallsByName((sql) => this.stmt(sql), symbolName, file, limit);
|
|
6508
|
+
}
|
|
6509
|
+
/**
|
|
6510
|
+
* Compute the set of symbol IDs reachable from the given seed IDs using a
|
|
6511
|
+
* native SQLite recursive CTE. Used by dead-code detection to replace the
|
|
6512
|
+
* in-memory BFS.
|
|
6513
|
+
*/
|
|
6514
|
+
findReachableSymbolIds(seedIds) {
|
|
6515
|
+
return findReachableSymbolIds((sql) => this.stmt(sql), seedIds);
|
|
6516
|
+
}
|
|
5234
6517
|
/**
|
|
5235
6518
|
* Find all references TO a given symbol (who calls / uses this symbol?).
|
|
5236
6519
|
*/
|
|
@@ -5321,6 +6604,9 @@ var YIELD_EVERY_N = 50;
|
|
|
5321
6604
|
function resolveParallelBatch() {
|
|
5322
6605
|
return indexParallelBatchSize(availableParallelism());
|
|
5323
6606
|
}
|
|
6607
|
+
function shouldUseParserWorkerPool(candidateFileCount, parseBatchCount) {
|
|
6608
|
+
return !isFrugalPerf() && candidateFileCount >= WORKER_POOL_THRESHOLD && parseBatchCount > 1;
|
|
6609
|
+
}
|
|
5324
6610
|
function yieldEventLoop() {
|
|
5325
6611
|
return new Promise((resolve4) => setImmediate(resolve4));
|
|
5326
6612
|
}
|
|
@@ -5338,15 +6624,15 @@ var INDEXABLE_EXTENSION_SET = new Set(INDEXABLE_EXTENSIONS);
|
|
|
5338
6624
|
var MAX_INDEX_FILE_BYTES = 5 * 1024 * 1024;
|
|
5339
6625
|
var MAX_GIT_FILE_LIST_BYTES = 64 * 1024 * 1024;
|
|
5340
6626
|
function isWithinProject(projectRoot, file) {
|
|
5341
|
-
const rel =
|
|
5342
|
-
return rel !== "" && !rel.startsWith(`..${
|
|
6627
|
+
const rel = path13.relative(projectRoot, file);
|
|
6628
|
+
return rel !== "" && !rel.startsWith(`..${path13.sep}`) && rel !== ".." && !path13.isAbsolute(rel);
|
|
5343
6629
|
}
|
|
5344
6630
|
function isMissingPathError(err) {
|
|
5345
6631
|
const code = err?.code;
|
|
5346
6632
|
return code === "ENOENT" || code === "ENOTDIR";
|
|
5347
6633
|
}
|
|
5348
6634
|
function normalizeComparablePath(value) {
|
|
5349
|
-
const resolved =
|
|
6635
|
+
const resolved = path13.resolve(value);
|
|
5350
6636
|
return process.platform === "win32" ? resolved.toLowerCase() : resolved;
|
|
5351
6637
|
}
|
|
5352
6638
|
function gitOutput(projectRoot, args) {
|
|
@@ -5391,24 +6677,24 @@ async function findGitSourceFiles(projectRoot, ignore, signal) {
|
|
|
5391
6677
|
const record = statusRecords[i];
|
|
5392
6678
|
if (!record) continue;
|
|
5393
6679
|
const status = record.slice(0, 2);
|
|
5394
|
-
const changedPath =
|
|
6680
|
+
const changedPath = path13.resolve(projectRoot, record.slice(3));
|
|
5395
6681
|
dirty.add(changedPath);
|
|
5396
6682
|
if (status.includes("D")) deleted.add(changedPath);
|
|
5397
6683
|
if (status.includes("R") || status.includes("C")) {
|
|
5398
6684
|
const source = statusRecords[++i];
|
|
5399
|
-
if (source) dirty.add(
|
|
6685
|
+
if (source) dirty.add(path13.resolve(projectRoot, source));
|
|
5400
6686
|
}
|
|
5401
6687
|
}
|
|
5402
6688
|
const files = [];
|
|
5403
6689
|
for (const relative3 of output.toString("utf8").split("\0")) {
|
|
5404
6690
|
if (!relative3) continue;
|
|
5405
6691
|
const portable = relative3.replace(/\\/g, "/");
|
|
5406
|
-
if (portable.split("/").some((segment) => ignoreSet.has(segment)) || DEFAULT_IGNORE_FILES.has(
|
|
6692
|
+
if (portable.split("/").some((segment) => ignoreSet.has(segment)) || DEFAULT_IGNORE_FILES.has(path13.posix.basename(portable))) {
|
|
5407
6693
|
continue;
|
|
5408
6694
|
}
|
|
5409
|
-
const full =
|
|
6695
|
+
const full = path13.resolve(projectRoot, relative3);
|
|
5410
6696
|
if (deleted.has(full)) continue;
|
|
5411
|
-
const ext =
|
|
6697
|
+
const ext = path13.extname(relative3).toLowerCase();
|
|
5412
6698
|
if (INDEXABLE_EXTENSION_SET.has(ext) || detectLang(full) !== null) files.push(full);
|
|
5413
6699
|
}
|
|
5414
6700
|
return {
|
|
@@ -5443,7 +6729,7 @@ async function findSourceFiles(projectRoot, ignore, isGitIgnored, signal) {
|
|
|
5443
6729
|
}
|
|
5444
6730
|
let entries;
|
|
5445
6731
|
try {
|
|
5446
|
-
entries = await
|
|
6732
|
+
entries = await fs9.readdir(dir, { withFileTypes: true });
|
|
5447
6733
|
} catch (err) {
|
|
5448
6734
|
complete = false;
|
|
5449
6735
|
errors.push(`scan error: ${dir}: ${err instanceof Error ? err.message : String(err)}`);
|
|
@@ -5452,14 +6738,14 @@ async function findSourceFiles(projectRoot, ignore, isGitIgnored, signal) {
|
|
|
5452
6738
|
dirCount++;
|
|
5453
6739
|
for (const e of entries) {
|
|
5454
6740
|
if (ignoreSet.has(e.name)) continue;
|
|
5455
|
-
const full =
|
|
5456
|
-
const rel =
|
|
6741
|
+
const full = path13.join(dir, e.name);
|
|
6742
|
+
const rel = path13.relative(projectRoot, full).replace(/\\/g, "/");
|
|
5457
6743
|
if (e.isDirectory()) {
|
|
5458
6744
|
if (isGitIgnored(rel, true)) continue;
|
|
5459
6745
|
await walk(full);
|
|
5460
6746
|
} else if (e.isFile()) {
|
|
5461
6747
|
if (DEFAULT_IGNORE_FILES.has(e.name) || isGitIgnored(rel, false)) continue;
|
|
5462
|
-
const ext =
|
|
6748
|
+
const ext = path13.extname(e.name).toLowerCase();
|
|
5463
6749
|
if (indexableExts.has(ext) || detectLang(full) !== null) {
|
|
5464
6750
|
results.push(full);
|
|
5465
6751
|
}
|
|
@@ -5497,11 +6783,7 @@ async function resolveProjectRelations(store, projectRoot, opts) {
|
|
|
5497
6783
|
const structure = await detectModuleRoots(projectRoot, indexedFiles);
|
|
5498
6784
|
if (opts.signal?.aborted) return;
|
|
5499
6785
|
store.setFilePackages(assignPackageLabels(structure, indexedFiles));
|
|
5500
|
-
const resolver = new ModuleResolver(
|
|
5501
|
-
structure,
|
|
5502
|
-
indexedFiles,
|
|
5503
|
-
store.getNamespaceDeclarations()
|
|
5504
|
-
);
|
|
6786
|
+
const resolver = new ModuleResolver(structure, indexedFiles, store.getNamespaceDeclarations());
|
|
5505
6787
|
const pending2 = store.getUnresolvedImports(opts.onlyFiles);
|
|
5506
6788
|
const resolutions = [];
|
|
5507
6789
|
for (const entry of pending2) {
|
|
@@ -5526,6 +6808,10 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5526
6808
|
const errors = [];
|
|
5527
6809
|
const langStats = {};
|
|
5528
6810
|
let filesIndexed = 0;
|
|
6811
|
+
let filesParsed = 0;
|
|
6812
|
+
let filesSkipped = 0;
|
|
6813
|
+
let filesEmpty = 0;
|
|
6814
|
+
let filesFailed = 0;
|
|
5529
6815
|
let symbolsIndexed = 0;
|
|
5530
6816
|
const isGitIgnored = await loadGitignoreMatcher(projectRoot);
|
|
5531
6817
|
let files;
|
|
@@ -5533,10 +6819,10 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5533
6819
|
let discoveryComplete = true;
|
|
5534
6820
|
let trustedUnchanged;
|
|
5535
6821
|
if (opts.files && opts.files.length > 0) {
|
|
5536
|
-
files = opts.files.map((f) =>
|
|
6822
|
+
files = opts.files.map((f) => path13.resolve(projectRoot, f)).filter((f) => {
|
|
5537
6823
|
if (!isWithinProject(projectRoot, f)) return false;
|
|
5538
|
-
const rel =
|
|
5539
|
-
return !rel.split("/").some((seg) => DEFAULT_IGNORE.includes(seg)) && !DEFAULT_IGNORE_FILES.has(
|
|
6824
|
+
const rel = path13.relative(projectRoot, f).replace(/\\/g, "/");
|
|
6825
|
+
return !rel.split("/").some((seg) => DEFAULT_IGNORE.includes(seg)) && !DEFAULT_IGNORE_FILES.has(path13.basename(f)) && !isGitIgnored(rel, false);
|
|
5540
6826
|
});
|
|
5541
6827
|
} else {
|
|
5542
6828
|
const discovery = await findSourceFiles(projectRoot, ignore, isGitIgnored, signal);
|
|
@@ -5567,12 +6853,14 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5567
6853
|
langStats[meta.lang] = (langStats[meta.lang] ?? 0) + meta.symbolCount;
|
|
5568
6854
|
symbolsIndexed += meta.symbolCount;
|
|
5569
6855
|
filesIndexed++;
|
|
6856
|
+
filesSkipped++;
|
|
5570
6857
|
filesPreSkipped++;
|
|
5571
6858
|
return false;
|
|
5572
6859
|
});
|
|
5573
6860
|
if (filesPreSkipped > 0) opts.onProgress?.(filesPreSkipped, totalFilesForProgress);
|
|
5574
6861
|
}
|
|
5575
6862
|
const parallelBatch = resolveParallelBatch();
|
|
6863
|
+
const parserPoolCandidateCount = files.length;
|
|
5576
6864
|
let filesSinceLastYield = 0;
|
|
5577
6865
|
for (let batchStart = 0; batchStart < files.length; batchStart += parallelBatch) {
|
|
5578
6866
|
const batchEnd = Math.min(batchStart + parallelBatch, files.length);
|
|
@@ -5593,7 +6881,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5593
6881
|
async (file) => {
|
|
5594
6882
|
let stat3;
|
|
5595
6883
|
try {
|
|
5596
|
-
stat3 = await
|
|
6884
|
+
stat3 = await fs9.stat(file, statOpts);
|
|
5597
6885
|
} catch (e) {
|
|
5598
6886
|
if (isAbortError(e)) throw e;
|
|
5599
6887
|
return {
|
|
@@ -5623,7 +6911,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5623
6911
|
}
|
|
5624
6912
|
let content;
|
|
5625
6913
|
try {
|
|
5626
|
-
content = await
|
|
6914
|
+
content = await fs9.readFile(file, { encoding: "utf8", signal });
|
|
5627
6915
|
} catch (e) {
|
|
5628
6916
|
if (isAbortError(e)) throw e;
|
|
5629
6917
|
return {
|
|
@@ -5634,22 +6922,78 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5634
6922
|
error: `read error: ${e instanceof Error ? e.message : String(e)}`
|
|
5635
6923
|
};
|
|
5636
6924
|
}
|
|
5637
|
-
|
|
5638
|
-
|
|
5639
|
-
parsed = await parseFileContent(file, content, lang);
|
|
5640
|
-
} catch (e) {
|
|
6925
|
+
const contentHash = xxhash64String(content);
|
|
6926
|
+
if (!force && meta && meta.contentHash && contentHash === meta.contentHash) {
|
|
5641
6927
|
return {
|
|
5642
6928
|
file,
|
|
5643
6929
|
stat: stat3,
|
|
5644
6930
|
lang,
|
|
5645
6931
|
parsed: null,
|
|
5646
|
-
|
|
6932
|
+
content,
|
|
6933
|
+
contentHash,
|
|
6934
|
+
skippedMeta: { ...meta, mtimeMs: Math.floor(stat3.mtimeMs) }
|
|
5647
6935
|
};
|
|
5648
6936
|
}
|
|
5649
|
-
return { file, stat: stat3, lang, parsed, content };
|
|
6937
|
+
return { file, stat: stat3, lang, parsed: null, content, contentHash };
|
|
5650
6938
|
}
|
|
5651
6939
|
)
|
|
5652
6940
|
);
|
|
6941
|
+
const toParse = [];
|
|
6942
|
+
for (let pi = 0; pi < statReadParse.length; pi++) {
|
|
6943
|
+
const s = statReadParse[pi];
|
|
6944
|
+
if (s.status !== "fulfilled") continue;
|
|
6945
|
+
const r = s.value;
|
|
6946
|
+
if (r.error || r.skippedMeta || !r.lang || r.parsed) continue;
|
|
6947
|
+
if (r.content === void 0) continue;
|
|
6948
|
+
toParse.push({
|
|
6949
|
+
index: pi,
|
|
6950
|
+
file: batchFiles[pi],
|
|
6951
|
+
content: r.content,
|
|
6952
|
+
lang: r.lang
|
|
6953
|
+
});
|
|
6954
|
+
}
|
|
6955
|
+
if (toParse.length > 0) {
|
|
6956
|
+
let pool = shouldUseParserWorkerPool(parserPoolCandidateCount, toParse.length) ? getParserPool() : null;
|
|
6957
|
+
if (pool) {
|
|
6958
|
+
try {
|
|
6959
|
+
await pool.ensureReady();
|
|
6960
|
+
const parsedResults = await pool.parseFiles(
|
|
6961
|
+
toParse.map((p) => ({ file: p.file, content: p.content, lang: p.lang }))
|
|
6962
|
+
);
|
|
6963
|
+
const byFile = new Map(parsedResults.map((r) => [r.file, r]));
|
|
6964
|
+
for (const item of toParse) {
|
|
6965
|
+
const parsed = byFile.get(item.file);
|
|
6966
|
+
const settled = statReadParse[item.index];
|
|
6967
|
+
if (settled.status !== "fulfilled") continue;
|
|
6968
|
+
if (parsed) {
|
|
6969
|
+
settled.value.parsed = parsed;
|
|
6970
|
+
} else {
|
|
6971
|
+
settled.value.error = `parse error: worker returned no result for ${item.file}`;
|
|
6972
|
+
}
|
|
6973
|
+
}
|
|
6974
|
+
} catch {
|
|
6975
|
+
pool = null;
|
|
6976
|
+
}
|
|
6977
|
+
}
|
|
6978
|
+
if (!pool) {
|
|
6979
|
+
await Promise.all(
|
|
6980
|
+
toParse.map(async (item) => {
|
|
6981
|
+
try {
|
|
6982
|
+
const parsed = await parseFileContent(item.file, item.content, item.lang);
|
|
6983
|
+
const settled = statReadParse[item.index];
|
|
6984
|
+
if (settled.status === "fulfilled") {
|
|
6985
|
+
settled.value.parsed = parsed;
|
|
6986
|
+
}
|
|
6987
|
+
} catch (e) {
|
|
6988
|
+
const settled = statReadParse[item.index];
|
|
6989
|
+
if (settled.status === "fulfilled") {
|
|
6990
|
+
settled.value.error = `parse error: ${e instanceof Error ? e.message : String(e)}`;
|
|
6991
|
+
}
|
|
6992
|
+
}
|
|
6993
|
+
})
|
|
6994
|
+
);
|
|
6995
|
+
}
|
|
6996
|
+
}
|
|
5653
6997
|
const batchEntries = [];
|
|
5654
6998
|
const deleteForFiles = [];
|
|
5655
6999
|
for (let fi = 0; fi < statReadParse.length; fi++) {
|
|
@@ -5659,12 +7003,14 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5659
7003
|
const err = settled.reason;
|
|
5660
7004
|
if (err instanceof Error && isAbortError(err)) throw err;
|
|
5661
7005
|
errors.push(`batch error: ${file}: ${err instanceof Error ? err.message : String(err)}`);
|
|
7006
|
+
filesFailed++;
|
|
5662
7007
|
continue;
|
|
5663
7008
|
}
|
|
5664
7009
|
const result = settled.value;
|
|
5665
7010
|
if (result.error) {
|
|
5666
7011
|
if (result.missing) store.deleteFile(file);
|
|
5667
7012
|
errors.push(`${file}: ${result.error}`);
|
|
7013
|
+
filesFailed++;
|
|
5668
7014
|
continue;
|
|
5669
7015
|
}
|
|
5670
7016
|
const { stat: stat3, lang, parsed } = result;
|
|
@@ -5672,6 +7018,18 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5672
7018
|
langStats[lang] = (langStats[lang] ?? 0) + result.skippedMeta.symbolCount;
|
|
5673
7019
|
symbolsIndexed += result.skippedMeta.symbolCount;
|
|
5674
7020
|
filesIndexed++;
|
|
7021
|
+
filesSkipped++;
|
|
7022
|
+
const stored = existingMeta.get(file);
|
|
7023
|
+
if (stored && stored.mtimeMs !== result.skippedMeta.mtimeMs) {
|
|
7024
|
+
store.upsertFile({
|
|
7025
|
+
file,
|
|
7026
|
+
lang,
|
|
7027
|
+
mtimeMs: result.skippedMeta.mtimeMs,
|
|
7028
|
+
symbolCount: result.skippedMeta.symbolCount,
|
|
7029
|
+
lastIndexed: Date.now(),
|
|
7030
|
+
contentHash: result.skippedMeta.contentHash
|
|
7031
|
+
});
|
|
7032
|
+
}
|
|
5675
7033
|
continue;
|
|
5676
7034
|
}
|
|
5677
7035
|
if (!lang || !parsed) {
|
|
@@ -5681,9 +7039,11 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5681
7039
|
lang,
|
|
5682
7040
|
mtimeMs: Math.floor(stat3.mtimeMs),
|
|
5683
7041
|
symbolCount: 0,
|
|
5684
|
-
lastIndexed: Date.now()
|
|
7042
|
+
lastIndexed: Date.now(),
|
|
7043
|
+
contentHash: result.contentHash ?? ""
|
|
5685
7044
|
});
|
|
5686
7045
|
filesIndexed++;
|
|
7046
|
+
filesEmpty++;
|
|
5687
7047
|
}
|
|
5688
7048
|
continue;
|
|
5689
7049
|
}
|
|
@@ -5693,9 +7053,11 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5693
7053
|
lang,
|
|
5694
7054
|
mtimeMs: Math.floor(stat3.mtimeMs),
|
|
5695
7055
|
symbolCount: 0,
|
|
5696
|
-
lastIndexed: Date.now()
|
|
7056
|
+
lastIndexed: Date.now(),
|
|
7057
|
+
contentHash: result.contentHash ?? ""
|
|
5697
7058
|
});
|
|
5698
7059
|
filesIndexed++;
|
|
7060
|
+
filesEmpty++;
|
|
5699
7061
|
continue;
|
|
5700
7062
|
}
|
|
5701
7063
|
batchEntries.push({
|
|
@@ -5704,7 +7066,8 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5704
7066
|
symbols: parsed.symbols,
|
|
5705
7067
|
refs: parsed.refs ?? [],
|
|
5706
7068
|
mtimeMs: Math.floor(stat3.mtimeMs),
|
|
5707
|
-
symbolCount: parsed.symbols.length
|
|
7069
|
+
symbolCount: parsed.symbols.length,
|
|
7070
|
+
contentHash: result.contentHash ?? ""
|
|
5708
7071
|
});
|
|
5709
7072
|
deleteForFiles.push(file);
|
|
5710
7073
|
}
|
|
@@ -5716,6 +7079,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5716
7079
|
symbolsIndexed += count;
|
|
5717
7080
|
langStats[entry.lang] = (langStats[entry.lang] ?? 0) + count;
|
|
5718
7081
|
filesIndexed++;
|
|
7082
|
+
filesParsed++;
|
|
5719
7083
|
}
|
|
5720
7084
|
} catch (err) {
|
|
5721
7085
|
const message = err instanceof Error ? err.message : String(err);
|
|
@@ -5728,6 +7092,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5728
7092
|
symbolsIndexed += symbolsWithIds.length;
|
|
5729
7093
|
langStats[entry.lang] = (langStats[entry.lang] ?? 0) + symbolsWithIds.length;
|
|
5730
7094
|
filesIndexed++;
|
|
7095
|
+
filesParsed++;
|
|
5731
7096
|
if (entry.refs.length > 0 && symbolsWithIds.length > 0) {
|
|
5732
7097
|
const fallbackBatch = assignRefsToSymbols2(entry.refs, symbolsWithIds);
|
|
5733
7098
|
if (fallbackBatch.length > 0) store.insertRefsBatch(fallbackBatch);
|
|
@@ -5741,9 +7106,11 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5741
7106
|
lang: entry.lang,
|
|
5742
7107
|
mtimeMs: entry.mtimeMs,
|
|
5743
7108
|
symbolCount: entry.symbolCount,
|
|
5744
|
-
lastIndexed: Date.now()
|
|
7109
|
+
lastIndexed: Date.now(),
|
|
7110
|
+
contentHash: entry.contentHash
|
|
5745
7111
|
});
|
|
5746
7112
|
} catch (innerErr) {
|
|
7113
|
+
filesFailed++;
|
|
5747
7114
|
errors.push(
|
|
5748
7115
|
`fallback write failed: ${entry.file}: ${innerErr instanceof Error ? innerErr.message : String(innerErr)}`
|
|
5749
7116
|
);
|
|
@@ -5776,6 +7143,12 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5776
7143
|
const durationMs = Date.now() - startMs;
|
|
5777
7144
|
return {
|
|
5778
7145
|
filesIndexed,
|
|
7146
|
+
fileOutcomes: {
|
|
7147
|
+
parsed: filesParsed,
|
|
7148
|
+
skipped: filesSkipped,
|
|
7149
|
+
empty: filesEmpty,
|
|
7150
|
+
failed: filesFailed
|
|
7151
|
+
},
|
|
5779
7152
|
symbolsIndexed,
|
|
5780
7153
|
langStats,
|
|
5781
7154
|
durationMs,
|
|
@@ -5853,6 +7226,9 @@ function symbolGraphService(args) {
|
|
|
5853
7226
|
function incomingCallsService(args) {
|
|
5854
7227
|
const store = indexStorePool.acquire(args.projectRoot, { indexDir: args.indexDir });
|
|
5855
7228
|
try {
|
|
7229
|
+
if (args.transitive) {
|
|
7230
|
+
return store.findTransitiveIncomingCallsByName(args.symbol, args.file, args.limit ?? 100);
|
|
7231
|
+
}
|
|
5856
7232
|
return store.findIncomingCallsByName(args.symbol, args.file, args.limit ?? 100);
|
|
5857
7233
|
} finally {
|
|
5858
7234
|
indexStorePool.release(store);
|
|
@@ -5861,6 +7237,9 @@ function incomingCallsService(args) {
|
|
|
5861
7237
|
function outgoingCallsService(args) {
|
|
5862
7238
|
const store = indexStorePool.acquire(args.projectRoot, { indexDir: args.indexDir });
|
|
5863
7239
|
try {
|
|
7240
|
+
if (args.transitive) {
|
|
7241
|
+
return store.findTransitiveOutgoingCallsByName(args.symbol, args.file, args.limit ?? 100);
|
|
7242
|
+
}
|
|
5864
7243
|
return store.findOutgoingCallsByName(args.symbol, args.file, args.limit ?? 100);
|
|
5865
7244
|
} finally {
|
|
5866
7245
|
indexStorePool.release(store);
|
|
@@ -5869,17 +7248,35 @@ function outgoingCallsService(args) {
|
|
|
5869
7248
|
|
|
5870
7249
|
// src/codebase-index/project-server-client.ts
|
|
5871
7250
|
import { spawn as spawn3 } from "node:child_process";
|
|
5872
|
-
import * as
|
|
7251
|
+
import * as fs11 from "node:fs";
|
|
5873
7252
|
import * as net from "node:net";
|
|
5874
|
-
import {
|
|
7253
|
+
import { StringDecoder } from "node:string_decoder";
|
|
7254
|
+
import { fileURLToPath as fileURLToPath4 } from "node:url";
|
|
5875
7255
|
import { checkUnixSocketPath } from "@wrongstack/core/utils";
|
|
5876
7256
|
|
|
7257
|
+
// src/codebase-index/binary-frame.ts
|
|
7258
|
+
import { decode, encode } from "@msgpack/msgpack";
|
|
7259
|
+
var BINARY_FRAME_MAGIC = 87;
|
|
7260
|
+
function isBinaryFrame(firstByte) {
|
|
7261
|
+
return firstByte === BINARY_FRAME_MAGIC;
|
|
7262
|
+
}
|
|
7263
|
+
function encodeBinaryFrame(message) {
|
|
7264
|
+
const payload = encode(message);
|
|
7265
|
+
const header = Buffer.allocUnsafe(5);
|
|
7266
|
+
header[0] = BINARY_FRAME_MAGIC;
|
|
7267
|
+
header.writeUInt32BE(payload.length, 1);
|
|
7268
|
+
return Buffer.concat([header, payload], 5 + payload.length);
|
|
7269
|
+
}
|
|
7270
|
+
function decodeBinaryFrame(payload) {
|
|
7271
|
+
return decode(payload);
|
|
7272
|
+
}
|
|
7273
|
+
|
|
5877
7274
|
// src/codebase-index/project-server-endpoint.ts
|
|
5878
7275
|
import { createHash as createHash2 } from "node:crypto";
|
|
5879
|
-
import * as
|
|
7276
|
+
import * as fs10 from "node:fs";
|
|
5880
7277
|
import * as os3 from "node:os";
|
|
5881
|
-
import * as
|
|
5882
|
-
import { fileURLToPath } from "node:url";
|
|
7278
|
+
import * as path14 from "node:path";
|
|
7279
|
+
import { fileURLToPath as fileURLToPath3 } from "node:url";
|
|
5883
7280
|
import { assertUnixSocketPathWithinLimit } from "@wrongstack/core/utils";
|
|
5884
7281
|
var PROJECT_INDEX_SERVER_PROTOCOL_VERSION = 1;
|
|
5885
7282
|
var PROJECT_INDEX_SERVER_METADATA_FILE = "server.json";
|
|
@@ -5888,21 +7285,21 @@ var buildIdCache;
|
|
|
5888
7285
|
function projectIndexServerBuildId(entrypoint) {
|
|
5889
7286
|
const href = entrypoint instanceof URL ? entrypoint.href : entrypoint;
|
|
5890
7287
|
const cleanHref = href.split(/[?#]/, 1)[0] ?? href;
|
|
5891
|
-
const file = cleanHref.startsWith("file:") ?
|
|
7288
|
+
const file = cleanHref.startsWith("file:") ? fileURLToPath3(cleanHref) : path14.resolve(cleanHref);
|
|
5892
7289
|
try {
|
|
5893
|
-
const stat3 =
|
|
7290
|
+
const stat3 = fs10.statSync(file);
|
|
5894
7291
|
if (buildIdCache?.file === file && buildIdCache.mtimeMs === stat3.mtimeMs && buildIdCache.size === stat3.size) {
|
|
5895
7292
|
return buildIdCache.buildId;
|
|
5896
7293
|
}
|
|
5897
|
-
const buildId = createHash2("sha256").update(
|
|
7294
|
+
const buildId = createHash2("sha256").update(fs10.readFileSync(file)).digest("hex").slice(0, 24);
|
|
5898
7295
|
buildIdCache = { file, mtimeMs: stat3.mtimeMs, size: stat3.size, buildId };
|
|
5899
7296
|
return buildId;
|
|
5900
7297
|
} catch {
|
|
5901
|
-
return `unreadable:${
|
|
7298
|
+
return `unreadable:${path14.basename(file)}`;
|
|
5902
7299
|
}
|
|
5903
7300
|
}
|
|
5904
7301
|
function normalizeLocalPath(value) {
|
|
5905
|
-
const resolved =
|
|
7302
|
+
const resolved = path14.resolve(value);
|
|
5906
7303
|
return process.platform === "win32" ? resolved.toLowerCase() : resolved;
|
|
5907
7304
|
}
|
|
5908
7305
|
function projectIndexServerKey(projectRoot, indexDir) {
|
|
@@ -5914,11 +7311,11 @@ function projectIndexServerEndpoint(projectRoot, indexDir) {
|
|
|
5914
7311
|
if (process.platform === "win32") {
|
|
5915
7312
|
return `\\\\.\\pipe\\wrongstack-codebase-index-v${PROJECT_INDEX_SERVER_PROTOCOL_VERSION}-${key}`;
|
|
5916
7313
|
}
|
|
5917
|
-
return
|
|
7314
|
+
return path14.join(os3.tmpdir(), PROJECT_INDEX_SERVER_SOCKET_DIR, `${key}.sock`);
|
|
5918
7315
|
}
|
|
5919
7316
|
function projectIndexServerMetadataPath(projectRoot, indexDir) {
|
|
5920
|
-
return
|
|
5921
|
-
|
|
7317
|
+
return path14.join(
|
|
7318
|
+
path14.resolve(resolveIndexDir(projectRoot, indexDir)),
|
|
5922
7319
|
PROJECT_INDEX_SERVER_METADATA_FILE
|
|
5923
7320
|
);
|
|
5924
7321
|
}
|
|
@@ -5958,7 +7355,7 @@ function resolveProjectIndexDaemonAvailability(projectRoot, indexDir) {
|
|
|
5958
7355
|
for (const rel of ["./project-server.js", "./codebase-index/project-server.js"]) {
|
|
5959
7356
|
try {
|
|
5960
7357
|
const url = new URL(rel, import.meta.url);
|
|
5961
|
-
if (url.protocol === "file:" &&
|
|
7358
|
+
if (url.protocol === "file:" && fs11.existsSync(fileURLToPath4(url))) {
|
|
5962
7359
|
builtUrl = url;
|
|
5963
7360
|
break;
|
|
5964
7361
|
}
|
|
@@ -6057,6 +7454,12 @@ var ProjectServerConnection = class {
|
|
|
6057
7454
|
endpoint;
|
|
6058
7455
|
socket = null;
|
|
6059
7456
|
buffer = "";
|
|
7457
|
+
/** P6: binary frame buffer — accumulates raw bytes when in binary mode. */
|
|
7458
|
+
binaryBuffer = [];
|
|
7459
|
+
/** P6: StringDecoder for safe UTF-8 multibyte handling in JSON mode. */
|
|
7460
|
+
textDecoder = null;
|
|
7461
|
+
/** P6: true once the server advertises binary support and client accepts. */
|
|
7462
|
+
useBinary = false;
|
|
6060
7463
|
info = null;
|
|
6061
7464
|
activity = null;
|
|
6062
7465
|
health = null;
|
|
@@ -6195,6 +7598,8 @@ var ProjectServerConnection = class {
|
|
|
6195
7598
|
this.info = null;
|
|
6196
7599
|
this.activity = null;
|
|
6197
7600
|
this.health = null;
|
|
7601
|
+
this.useBinary = false;
|
|
7602
|
+
this.binaryBuffer = [];
|
|
6198
7603
|
this.connectReject?.(new Error("codebase-index client disconnected"));
|
|
6199
7604
|
this.connectResolve = null;
|
|
6200
7605
|
this.connectReject = null;
|
|
@@ -6215,7 +7620,7 @@ var ProjectServerConnection = class {
|
|
|
6215
7620
|
currentAuthToken() {
|
|
6216
7621
|
if (this.authToken === void 0) {
|
|
6217
7622
|
try {
|
|
6218
|
-
const raw =
|
|
7623
|
+
const raw = fs11.readFileSync(
|
|
6219
7624
|
projectIndexServerMetadataPath(this.projectRoot, this.indexDir),
|
|
6220
7625
|
"utf8"
|
|
6221
7626
|
);
|
|
@@ -6323,10 +7728,12 @@ var ProjectServerConnection = class {
|
|
|
6323
7728
|
this.activity = null;
|
|
6324
7729
|
this.health = null;
|
|
6325
7730
|
this.buffer = "";
|
|
7731
|
+
this.binaryBuffer = [];
|
|
7732
|
+
this.useBinary = false;
|
|
7733
|
+
this.textDecoder = null;
|
|
6326
7734
|
return new Promise((resolve4, reject) => {
|
|
6327
7735
|
const socket = net.createConnection(this.endpoint);
|
|
6328
7736
|
this.socket = socket;
|
|
6329
|
-
socket.setEncoding("utf8");
|
|
6330
7737
|
const timer = setTimeout(() => {
|
|
6331
7738
|
reject(new Error("codebase-index server handshake timed out"));
|
|
6332
7739
|
socket.destroy();
|
|
@@ -6355,7 +7762,12 @@ var ProjectServerConnection = class {
|
|
|
6355
7762
|
}
|
|
6356
7763
|
onData(socket, chunk) {
|
|
6357
7764
|
if (socket !== this.socket) return;
|
|
6358
|
-
this.
|
|
7765
|
+
if (this.useBinary) {
|
|
7766
|
+
this.onBinaryData(socket, chunk);
|
|
7767
|
+
return;
|
|
7768
|
+
}
|
|
7769
|
+
if (!this.textDecoder) this.textDecoder = new StringDecoder("utf8");
|
|
7770
|
+
this.buffer += this.textDecoder.write(chunk);
|
|
6359
7771
|
while (true) {
|
|
6360
7772
|
const newline = this.buffer.indexOf("\n");
|
|
6361
7773
|
if (newline < 0) {
|
|
@@ -6381,6 +7793,44 @@ var ProjectServerConnection = class {
|
|
|
6381
7793
|
this.onMessage(message);
|
|
6382
7794
|
}
|
|
6383
7795
|
}
|
|
7796
|
+
/**
|
|
7797
|
+
* P6: Parse binary frames from the raw buffer.
|
|
7798
|
+
*
|
|
7799
|
+
* Each frame is: [1 byte magic 0x57] [4 bytes uint32 BE length] [payload].
|
|
7800
|
+
* The magic byte distinguishes binary from JSON — a JSON frame's first byte
|
|
7801
|
+
* is `{` (0x7B), so there is no ambiguity even in a mixed-mode buffer.
|
|
7802
|
+
*/
|
|
7803
|
+
onBinaryData(socket, chunk) {
|
|
7804
|
+
this.binaryBuffer.push(chunk);
|
|
7805
|
+
const all = Buffer.concat(this.binaryBuffer);
|
|
7806
|
+
let offset = 0;
|
|
7807
|
+
while (offset + 5 <= all.length) {
|
|
7808
|
+
if (!isBinaryFrame(all[offset])) {
|
|
7809
|
+
this.useBinary = false;
|
|
7810
|
+
this.buffer += all.subarray(offset).toString("utf8");
|
|
7811
|
+
this.binaryBuffer = [];
|
|
7812
|
+
return;
|
|
7813
|
+
}
|
|
7814
|
+
const frameLen = all.readUInt32BE(offset + 1);
|
|
7815
|
+
if (frameLen > 256 * 1024 * 1024) {
|
|
7816
|
+
socket.destroy();
|
|
7817
|
+
this.transition("offline", { error: "binary frame length exceeds 256MB limit" });
|
|
7818
|
+
return;
|
|
7819
|
+
}
|
|
7820
|
+
const totalLen = 5 + frameLen;
|
|
7821
|
+
if (offset + totalLen > all.length) break;
|
|
7822
|
+
const payload = all.subarray(offset + 5, offset + 5 + frameLen);
|
|
7823
|
+
try {
|
|
7824
|
+
const message = decodeBinaryFrame(payload);
|
|
7825
|
+
this.onMessage(message);
|
|
7826
|
+
} catch {
|
|
7827
|
+
socket.destroy(new Error("invalid binary codebase-index server response"));
|
|
7828
|
+
return;
|
|
7829
|
+
}
|
|
7830
|
+
offset += totalLen;
|
|
7831
|
+
}
|
|
7832
|
+
this.binaryBuffer = offset < all.length ? [all.subarray(offset)] : [];
|
|
7833
|
+
}
|
|
6384
7834
|
onMessage(message) {
|
|
6385
7835
|
if (message.type === "hello") {
|
|
6386
7836
|
if (message.protocolVersion !== PROJECT_INDEX_SERVER_PROTOCOL_VERSION) {
|
|
@@ -6400,6 +7850,7 @@ var ProjectServerConnection = class {
|
|
|
6400
7850
|
}
|
|
6401
7851
|
this.info = message;
|
|
6402
7852
|
this.markResponsive();
|
|
7853
|
+
if (message.binarySupported) this.useBinary = true;
|
|
6403
7854
|
this.transition("connected", { pid: message.pid });
|
|
6404
7855
|
ensureHeartbeatLoop();
|
|
6405
7856
|
this.connectResolve?.();
|
|
@@ -6458,7 +7909,12 @@ var ProjectServerConnection = class {
|
|
|
6458
7909
|
}
|
|
6459
7910
|
write(message) {
|
|
6460
7911
|
const socket = this.socket;
|
|
6461
|
-
if (socket
|
|
7912
|
+
if (!socket || socket.destroyed) return;
|
|
7913
|
+
if (this.useBinary) {
|
|
7914
|
+
socket.write(encodeBinaryFrame(message));
|
|
7915
|
+
} else {
|
|
7916
|
+
socket.write(encodeProjectServerMessage(message));
|
|
7917
|
+
}
|
|
6462
7918
|
}
|
|
6463
7919
|
rejectStaleServer(message, reason) {
|
|
6464
7920
|
const socket = this.socket;
|
|
@@ -6480,11 +7936,11 @@ var ProjectServerConnection = class {
|
|
|
6480
7936
|
if (!url) throw new Error("built codebase-index project server is unavailable");
|
|
6481
7937
|
if (process.platform !== "win32") {
|
|
6482
7938
|
try {
|
|
6483
|
-
|
|
7939
|
+
fs11.rmSync(this.endpoint, { force: true });
|
|
6484
7940
|
} catch {
|
|
6485
7941
|
}
|
|
6486
7942
|
}
|
|
6487
|
-
const args = [
|
|
7943
|
+
const args = [fileURLToPath4(url), "--project-root", this.projectRoot];
|
|
6488
7944
|
if (this.indexDir) args.push("--index-dir", this.indexDir);
|
|
6489
7945
|
const child = spawn3(process.execPath, args, {
|
|
6490
7946
|
detached: true,
|
|
@@ -6504,8 +7960,8 @@ var ProjectServerConnection = class {
|
|
|
6504
7960
|
process.kill(pid);
|
|
6505
7961
|
const metadataPath = projectIndexServerMetadataPath(this.projectRoot, this.indexDir);
|
|
6506
7962
|
try {
|
|
6507
|
-
const metadata = JSON.parse(
|
|
6508
|
-
if (metadata.pid === pid)
|
|
7963
|
+
const metadata = JSON.parse(fs11.readFileSync(metadataPath, "utf8"));
|
|
7964
|
+
if (metadata.pid === pid) fs11.rmSync(metadataPath, { force: true });
|
|
6509
7965
|
} catch {
|
|
6510
7966
|
}
|
|
6511
7967
|
return true;
|
|
@@ -6604,7 +8060,7 @@ function resolveWorkerUrl() {
|
|
|
6604
8060
|
for (const rel of ["./worker.js", "./codebase-index/worker.js"]) {
|
|
6605
8061
|
try {
|
|
6606
8062
|
const url = new URL(rel, import.meta.url);
|
|
6607
|
-
if (url.protocol === "file:" &&
|
|
8063
|
+
if (url.protocol === "file:" && fs12.existsSync(fileURLToPath5(url))) return url;
|
|
6608
8064
|
} catch {
|
|
6609
8065
|
}
|
|
6610
8066
|
}
|
|
@@ -6624,7 +8080,7 @@ function ensureWorker() {
|
|
|
6624
8080
|
return null;
|
|
6625
8081
|
}
|
|
6626
8082
|
try {
|
|
6627
|
-
const w = new
|
|
8083
|
+
const w = new Worker2(url, { name: "wstack-codebase-index" });
|
|
6628
8084
|
w.unref();
|
|
6629
8085
|
w.on("message", (msg) => {
|
|
6630
8086
|
if (msg.type === "progress") {
|
|
@@ -6848,7 +8304,7 @@ var readTool = {
|
|
|
6848
8304
|
const shouldIncludeSymbols = input.includeSymbols === true || input.includeSymbols !== false && ctx.meta[ADVANCED_MODE_META_KEY] === true;
|
|
6849
8305
|
let stat3;
|
|
6850
8306
|
try {
|
|
6851
|
-
stat3 = await
|
|
8307
|
+
stat3 = await fs13.stat(absPath);
|
|
6852
8308
|
} catch (err) {
|
|
6853
8309
|
const code = err.code;
|
|
6854
8310
|
if (code === "ENOENT") {
|
|
@@ -6900,7 +8356,7 @@ var readTool = {
|
|
|
6900
8356
|
...symResult2?.symbols ? { symbols: symResult2.symbols } : {}
|
|
6901
8357
|
};
|
|
6902
8358
|
}
|
|
6903
|
-
const buf = await
|
|
8359
|
+
const buf = await fs13.readFile(absPath);
|
|
6904
8360
|
if (isBinaryBuffer(buf)) {
|
|
6905
8361
|
throw new Error(`read: "${input.path}" appears to be binary`);
|
|
6906
8362
|
}
|