@wrongstack/tools 0.302.0 → 0.303.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/builtin.js +2799 -609
- package/dist/codebase-index/binary-frame.d.ts +43 -0
- package/dist/codebase-index/codebase-incoming-calls-tool.d.ts +1 -0
- package/dist/codebase-index/codebase-outgoing-calls-tool.d.ts +1 -0
- package/dist/codebase-index/content-hash.d.ts +66 -0
- package/dist/codebase-index/index.js +1569 -148
- package/dist/codebase-index/parser-worker-pool.d.ts +63 -0
- package/dist/codebase-index/parser-worker-script.d.ts +42 -0
- package/dist/codebase-index/project-server-protocol.d.ts +2 -0
- package/dist/codebase-index/project-server.js +1455 -99
- package/dist/codebase-index/schema.d.ts +7 -0
- package/dist/codebase-index/tree-sitter/queries.d.ts +48 -0
- package/dist/codebase-index/tree-sitter/util.d.ts +31 -0
- package/dist/codebase-index/tree-sitter/visitor.d.ts +47 -0
- package/dist/codebase-index/tree-sitter-parser.d.ts +58 -0
- package/dist/codebase-index/vector-search.d.ts +62 -0
- package/dist/codebase-index/worker-protocol.d.ts +2 -0
- package/dist/codebase-index/worker.js +1424 -68
- package/dist/codebase-index/writer-bulk-insert.d.ts +5 -0
- package/dist/codebase-index/writer-graph-reader.d.ts +39 -0
- package/dist/codebase-index/writer-schema.d.ts +9 -2
- package/dist/codebase-index/writer.d.ts +36 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +2785 -634
- package/dist/kanban-task-inputs.d.ts +1 -0
- package/dist/kanban-tool-schema.d.ts +2 -2
- package/dist/kanban-tool-types.d.ts +18 -2
- package/dist/kanban.js +392 -126
- package/dist/pack.js +2799 -609
- package/dist/plan.d.ts +4 -1
- package/dist/plan.js +2380 -9
- package/dist/read.js +1531 -98
- package/dist/session-kanban.d.ts +8 -0
- package/dist/session-kanban.js +111 -17
- package/dist/task.d.ts +5 -4
- package/dist/task.js +2418 -43
- package/dist/todo.d.ts +10 -1
- package/dist/todo.js +2152 -20
- package/dist/tool-tier.js +2799 -609
- package/package.json +8 -4
package/dist/read.js
CHANGED
|
@@ -2127,8 +2127,579 @@ var init_yaml_parser = __esm({
|
|
|
2127
2127
|
}
|
|
2128
2128
|
});
|
|
2129
2129
|
|
|
2130
|
+
// src/codebase-index/tree-sitter/queries.ts
|
|
2131
|
+
function getQueries(lang) {
|
|
2132
|
+
return LANG_QUERIES[lang] ?? DEFAULT_QUERIES;
|
|
2133
|
+
}
|
|
2134
|
+
function readFirstString(node) {
|
|
2135
|
+
if (!node) return null;
|
|
2136
|
+
if (node.type === "string_literal" || node.type === "alias") {
|
|
2137
|
+
return node.text.replace(/^"|"$/g, "");
|
|
2138
|
+
}
|
|
2139
|
+
const child = node.namedChild(0);
|
|
2140
|
+
return child ? readFirstString(child) : null;
|
|
2141
|
+
}
|
|
2142
|
+
var DEFAULT_QUERIES, LANG_QUERIES;
|
|
2143
|
+
var init_queries = __esm({
|
|
2144
|
+
"src/codebase-index/tree-sitter/queries.ts"() {
|
|
2145
|
+
"use strict";
|
|
2146
|
+
DEFAULT_QUERIES = {
|
|
2147
|
+
declKinds: {}
|
|
2148
|
+
};
|
|
2149
|
+
LANG_QUERIES = {
|
|
2150
|
+
// ─── C family ──────────────────────────────────────────────────────────────
|
|
2151
|
+
c: {
|
|
2152
|
+
declKinds: {
|
|
2153
|
+
function_definition: "function",
|
|
2154
|
+
declaration: "function",
|
|
2155
|
+
// K&R-style `int foo(...)` ambiguous w/ local var; the visitor prefers the function branch when the declarator field is present
|
|
2156
|
+
struct_specifier: "struct",
|
|
2157
|
+
union_specifier: "struct",
|
|
2158
|
+
enum_specifier: "enum",
|
|
2159
|
+
type_definition: "type",
|
|
2160
|
+
// `typedef … X;`
|
|
2161
|
+
preproc_def: "const"
|
|
2162
|
+
// `#define NAME …`
|
|
2163
|
+
},
|
|
2164
|
+
nameField: {
|
|
2165
|
+
function_definition: "declarator",
|
|
2166
|
+
declaration: "declarator",
|
|
2167
|
+
struct_specifier: "name",
|
|
2168
|
+
enum_specifier: "name",
|
|
2169
|
+
type_definition: "declarator",
|
|
2170
|
+
preproc_def: "name"
|
|
2171
|
+
},
|
|
2172
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2173
|
+
"translation_unit",
|
|
2174
|
+
"function_definition",
|
|
2175
|
+
"struct_specifier",
|
|
2176
|
+
"union_specifier",
|
|
2177
|
+
"enum_specifier"
|
|
2178
|
+
])
|
|
2179
|
+
},
|
|
2180
|
+
cpp: {
|
|
2181
|
+
declKinds: {
|
|
2182
|
+
function_definition: "function",
|
|
2183
|
+
template_declaration: "function",
|
|
2184
|
+
// `template<typename T> …`
|
|
2185
|
+
class_specifier: "class",
|
|
2186
|
+
struct_specifier: "struct",
|
|
2187
|
+
union_specifier: "struct",
|
|
2188
|
+
enum_specifier: "enum",
|
|
2189
|
+
namespace_definition: "namespace",
|
|
2190
|
+
type_definition: "type"
|
|
2191
|
+
},
|
|
2192
|
+
nameField: {
|
|
2193
|
+
function_definition: "declarator",
|
|
2194
|
+
template_declaration: "name",
|
|
2195
|
+
class_specifier: "name",
|
|
2196
|
+
struct_specifier: "name",
|
|
2197
|
+
enum_specifier: "name",
|
|
2198
|
+
namespace_definition: "name",
|
|
2199
|
+
type_definition: "declarator"
|
|
2200
|
+
},
|
|
2201
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2202
|
+
"translation_unit",
|
|
2203
|
+
"function_definition",
|
|
2204
|
+
"class_specifier",
|
|
2205
|
+
"struct_specifier",
|
|
2206
|
+
"union_specifier",
|
|
2207
|
+
"enum_specifier",
|
|
2208
|
+
"namespace_definition"
|
|
2209
|
+
])
|
|
2210
|
+
},
|
|
2211
|
+
java: {
|
|
2212
|
+
declKinds: {
|
|
2213
|
+
class_declaration: "class",
|
|
2214
|
+
interface_declaration: "interface",
|
|
2215
|
+
enum_declaration: "enum",
|
|
2216
|
+
record_declaration: "class",
|
|
2217
|
+
annotation_type_declaration: "interface",
|
|
2218
|
+
method_declaration: "method",
|
|
2219
|
+
constructor_declaration: "method",
|
|
2220
|
+
field_declaration: "property"
|
|
2221
|
+
},
|
|
2222
|
+
nameField: {
|
|
2223
|
+
class_declaration: "name",
|
|
2224
|
+
interface_declaration: "name",
|
|
2225
|
+
enum_declaration: "name",
|
|
2226
|
+
record_declaration: "name",
|
|
2227
|
+
annotation_type_declaration: "name",
|
|
2228
|
+
method_declaration: "name",
|
|
2229
|
+
constructor_declaration: "name"
|
|
2230
|
+
},
|
|
2231
|
+
// `field_declaration` has no single `name` field — it carries a list of
|
|
2232
|
+
// variable declarators. We emit one Symbol per node using the first
|
|
2233
|
+
// identifier-shaped named child (see `extractName` fallback in
|
|
2234
|
+
// `visitor.ts`). `int a, b, c;` therefore indexes only `a` — splitting
|
|
2235
|
+
// multi-declarator fields into separate Symbols is a separate refactor
|
|
2236
|
+
// that needs the visitor to know it has multiple names per node, and no
|
|
2237
|
+
// current test relies on it.
|
|
2238
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2239
|
+
"program",
|
|
2240
|
+
"class_declaration",
|
|
2241
|
+
"interface_declaration",
|
|
2242
|
+
"enum_declaration",
|
|
2243
|
+
"record_declaration"
|
|
2244
|
+
])
|
|
2245
|
+
},
|
|
2246
|
+
csharp: {
|
|
2247
|
+
// C# 10+ `namespace Foo.Bar;` produces this node type. The legacy block
|
|
2248
|
+
// form `namespace Foo.Bar { ... }` produces `namespace_declaration`. Both
|
|
2249
|
+
// carry a `qualified_name` child whose text already includes the dots.
|
|
2250
|
+
// `using_directive` is intentionally not a declaration. Imports are
|
|
2251
|
+
// extracted separately; indexing a using directive as a namespace makes
|
|
2252
|
+
// the resolver bind it to its own source file before the real declaration.
|
|
2253
|
+
declKinds: {
|
|
2254
|
+
file_scoped_namespace_declaration: "namespace",
|
|
2255
|
+
class_declaration: "class",
|
|
2256
|
+
interface_declaration: "interface",
|
|
2257
|
+
struct_declaration: "struct",
|
|
2258
|
+
enum_declaration: "enum",
|
|
2259
|
+
record_declaration: "class",
|
|
2260
|
+
method_declaration: "method",
|
|
2261
|
+
constructor_declaration: "method",
|
|
2262
|
+
property_declaration: "property",
|
|
2263
|
+
field_declaration: "property",
|
|
2264
|
+
namespace_declaration: "namespace"
|
|
2265
|
+
},
|
|
2266
|
+
// Custom name extractor: take the full dotted name verbatim.
|
|
2267
|
+
nameExtractor: (node) => {
|
|
2268
|
+
const inner = node.namedChild(0);
|
|
2269
|
+
if (inner && (inner.type === "qualified_name" || inner.type === "name")) {
|
|
2270
|
+
return inner.text;
|
|
2271
|
+
}
|
|
2272
|
+
return null;
|
|
2273
|
+
},
|
|
2274
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2275
|
+
"compilation_unit",
|
|
2276
|
+
"namespace_declaration",
|
|
2277
|
+
"class_declaration",
|
|
2278
|
+
"interface_declaration",
|
|
2279
|
+
"struct_declaration",
|
|
2280
|
+
"enum_declaration",
|
|
2281
|
+
"record_declaration"
|
|
2282
|
+
])
|
|
2283
|
+
},
|
|
2284
|
+
php: {
|
|
2285
|
+
declKinds: {
|
|
2286
|
+
function_definition: "function",
|
|
2287
|
+
method_declaration: "method",
|
|
2288
|
+
class_declaration: "class",
|
|
2289
|
+
interface_declaration: "interface",
|
|
2290
|
+
trait_declaration: "class",
|
|
2291
|
+
enum_declaration: "enum",
|
|
2292
|
+
namespace_definition: "namespace"
|
|
2293
|
+
},
|
|
2294
|
+
nameField: {
|
|
2295
|
+
function_definition: "name",
|
|
2296
|
+
method_declaration: "name",
|
|
2297
|
+
class_declaration: "name",
|
|
2298
|
+
interface_declaration: "name",
|
|
2299
|
+
trait_declaration: "name",
|
|
2300
|
+
enum_declaration: "name",
|
|
2301
|
+
namespace_declaration: "name"
|
|
2302
|
+
},
|
|
2303
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2304
|
+
"program",
|
|
2305
|
+
"namespace_definition",
|
|
2306
|
+
"class_declaration",
|
|
2307
|
+
"interface_declaration",
|
|
2308
|
+
"trait_declaration",
|
|
2309
|
+
"enum_declaration"
|
|
2310
|
+
])
|
|
2311
|
+
},
|
|
2312
|
+
// ─── Scripting / mobile ────────────────────────────────────────────────────
|
|
2313
|
+
ruby: {
|
|
2314
|
+
declKinds: {
|
|
2315
|
+
method: "function",
|
|
2316
|
+
singleton_method: "method",
|
|
2317
|
+
class: "class",
|
|
2318
|
+
module: "namespace",
|
|
2319
|
+
constant: "const"
|
|
2320
|
+
},
|
|
2321
|
+
nameField: {
|
|
2322
|
+
method: "name",
|
|
2323
|
+
singleton_method: "name",
|
|
2324
|
+
class: "name",
|
|
2325
|
+
module: "name",
|
|
2326
|
+
constant: "name"
|
|
2327
|
+
},
|
|
2328
|
+
scopeNodes: /* @__PURE__ */ new Set(["program", "class", "module", "singleton_method", "method"])
|
|
2329
|
+
},
|
|
2330
|
+
swift: {
|
|
2331
|
+
declKinds: {
|
|
2332
|
+
function_declaration: "function",
|
|
2333
|
+
class_declaration: "class",
|
|
2334
|
+
struct_declaration: "struct",
|
|
2335
|
+
enum_declaration: "enum",
|
|
2336
|
+
protocol_declaration: "interface",
|
|
2337
|
+
actor_declaration: "class",
|
|
2338
|
+
extension_declaration: "class",
|
|
2339
|
+
initializer: "method",
|
|
2340
|
+
property_declaration: "property"
|
|
2341
|
+
},
|
|
2342
|
+
nameField: {
|
|
2343
|
+
function_declaration: "name",
|
|
2344
|
+
class_declaration: "name",
|
|
2345
|
+
struct_declaration: "name",
|
|
2346
|
+
enum_declaration: "name",
|
|
2347
|
+
protocol_declaration: "name",
|
|
2348
|
+
actor_declaration: "name",
|
|
2349
|
+
extension_declaration: "name",
|
|
2350
|
+
initializer: "name",
|
|
2351
|
+
property_declaration: "name"
|
|
2352
|
+
},
|
|
2353
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2354
|
+
"source_file",
|
|
2355
|
+
"class_declaration",
|
|
2356
|
+
"struct_declaration",
|
|
2357
|
+
"enum_declaration",
|
|
2358
|
+
"protocol_declaration",
|
|
2359
|
+
"actor_declaration",
|
|
2360
|
+
"extension_declaration"
|
|
2361
|
+
])
|
|
2362
|
+
},
|
|
2363
|
+
kotlin: {
|
|
2364
|
+
declKinds: {
|
|
2365
|
+
class_declaration: "class",
|
|
2366
|
+
object_declaration: "class",
|
|
2367
|
+
interface_declaration: "interface",
|
|
2368
|
+
function_declaration: "function",
|
|
2369
|
+
property_declaration: "property",
|
|
2370
|
+
type_alias: "type"
|
|
2371
|
+
},
|
|
2372
|
+
nameField: {
|
|
2373
|
+
class_declaration: "name",
|
|
2374
|
+
object_declaration: "name",
|
|
2375
|
+
interface_declaration: "name",
|
|
2376
|
+
function_declaration: "name",
|
|
2377
|
+
property_declaration: "name",
|
|
2378
|
+
type_alias: "name"
|
|
2379
|
+
},
|
|
2380
|
+
scopeNodes: /* @__PURE__ */ new Set([
|
|
2381
|
+
"source_file",
|
|
2382
|
+
"class_declaration",
|
|
2383
|
+
"object_declaration",
|
|
2384
|
+
"interface_declaration",
|
|
2385
|
+
"function_declaration"
|
|
2386
|
+
])
|
|
2387
|
+
},
|
|
2388
|
+
elixir: {
|
|
2389
|
+
declKinds: {
|
|
2390
|
+
// `def foo`, `defp foo`, `defmacro foo`, `macrop foo` all surface as
|
|
2391
|
+
// `call` nodes in the tree-sitter grammar — there is no
|
|
2392
|
+
// `function_definition`. The `nameExtractor` walks the call's
|
|
2393
|
+
// children to pick the right sibling identifier.
|
|
2394
|
+
call: "function",
|
|
2395
|
+
module: "namespace"
|
|
2396
|
+
},
|
|
2397
|
+
nameExtractor: (node) => {
|
|
2398
|
+
if (node.type === "module") {
|
|
2399
|
+
const aliasNode = node.childForFieldName("alias");
|
|
2400
|
+
return readFirstString(aliasNode) ?? null;
|
|
2401
|
+
}
|
|
2402
|
+
if (node.type !== "call") return null;
|
|
2403
|
+
const first = node.namedChild(0);
|
|
2404
|
+
if (!first) return null;
|
|
2405
|
+
const target = first.text;
|
|
2406
|
+
if (target !== "def" && target !== "defp" && target !== "defmacro" && target !== "defp_macro" && target !== "macrop" && target !== "defprotocol" && target !== "defguard" && target !== "defguardp") {
|
|
2407
|
+
return null;
|
|
2408
|
+
}
|
|
2409
|
+
const nameNode = node.namedChild(1);
|
|
2410
|
+
return nameNode?.text ?? null;
|
|
2411
|
+
},
|
|
2412
|
+
scopeNodes: /* @__PURE__ */ new Set(["source", "module"])
|
|
2413
|
+
},
|
|
2414
|
+
shell: {
|
|
2415
|
+
declKinds: {
|
|
2416
|
+
function_definition: "function"
|
|
2417
|
+
},
|
|
2418
|
+
nameField: { function_definition: "name" },
|
|
2419
|
+
scopeNodes: /* @__PURE__ */ new Set(["program", "function_definition"])
|
|
2420
|
+
}
|
|
2421
|
+
};
|
|
2422
|
+
}
|
|
2423
|
+
});
|
|
2424
|
+
|
|
2425
|
+
// src/codebase-index/tree-sitter/util.ts
|
|
2426
|
+
function lineColAt2(offsets, index) {
|
|
2427
|
+
let low = 0;
|
|
2428
|
+
let high = offsets.length;
|
|
2429
|
+
while (low < high) {
|
|
2430
|
+
const mid = low + high >>> 1;
|
|
2431
|
+
if ((offsets[mid] ?? 0) < index) low = mid + 1;
|
|
2432
|
+
else high = mid;
|
|
2433
|
+
}
|
|
2434
|
+
const lastNl = low > 0 ? offsets[low - 1] ?? -1 : -1;
|
|
2435
|
+
return { line: low + 1, col: index - lastNl };
|
|
2436
|
+
}
|
|
2437
|
+
function newlineOffsets3(content) {
|
|
2438
|
+
const offsets = [];
|
|
2439
|
+
for (let i = 0; i < content.length; i++) {
|
|
2440
|
+
if (content.charCodeAt(i) === 10) offsets.push(i);
|
|
2441
|
+
}
|
|
2442
|
+
return offsets;
|
|
2443
|
+
}
|
|
2444
|
+
var TREE_SITTER_MAX_FILE_CHARS, TREE_SITTER_MAX_SYMBOLS;
|
|
2445
|
+
var init_util = __esm({
|
|
2446
|
+
"src/codebase-index/tree-sitter/util.ts"() {
|
|
2447
|
+
"use strict";
|
|
2448
|
+
TREE_SITTER_MAX_FILE_CHARS = 512 * 1024;
|
|
2449
|
+
TREE_SITTER_MAX_SYMBOLS = 500;
|
|
2450
|
+
}
|
|
2451
|
+
});
|
|
2452
|
+
|
|
2453
|
+
// src/codebase-index/tree-sitter/visitor.ts
|
|
2454
|
+
function visitTree(tree, content, file, lang, queries) {
|
|
2455
|
+
const boundedContent = content.length > TREE_SITTER_MAX_FILE_CHARS ? content.slice(0, TREE_SITTER_MAX_FILE_CHARS) : content;
|
|
2456
|
+
const nlOffsets = newlineOffsets3(boundedContent);
|
|
2457
|
+
const symbols = [];
|
|
2458
|
+
const scopeStack = [];
|
|
2459
|
+
function visit(node, depth) {
|
|
2460
|
+
if (symbols.length >= TREE_SITTER_MAX_SYMBOLS) return;
|
|
2461
|
+
if (node.isMissing || node.isError) {
|
|
2462
|
+
} else {
|
|
2463
|
+
const kind = queries.declKinds[node.type];
|
|
2464
|
+
if (kind) {
|
|
2465
|
+
const emitted = emitSymbol(
|
|
2466
|
+
node,
|
|
2467
|
+
kind,
|
|
2468
|
+
file,
|
|
2469
|
+
lang,
|
|
2470
|
+
scopeStack,
|
|
2471
|
+
boundedContent,
|
|
2472
|
+
nlOffsets,
|
|
2473
|
+
queries
|
|
2474
|
+
);
|
|
2475
|
+
if (emitted) symbols.push(emitted);
|
|
2476
|
+
}
|
|
2477
|
+
}
|
|
2478
|
+
const pushesScope = queries.scopeNodes?.has(node.type) ?? false;
|
|
2479
|
+
const pushIdx = pushesScope ? pushScope(scopeStack, node, queries) : -1;
|
|
2480
|
+
if (queries.skipNamedChildren) {
|
|
2481
|
+
if (pushIdx !== -1) scopeStack.pop();
|
|
2482
|
+
return;
|
|
2483
|
+
}
|
|
2484
|
+
for (const child of node.namedChildren) {
|
|
2485
|
+
visit(child, depth + 1);
|
|
2486
|
+
if (symbols.length >= TREE_SITTER_MAX_SYMBOLS) {
|
|
2487
|
+
if (pushIdx !== -1) scopeStack.pop();
|
|
2488
|
+
return;
|
|
2489
|
+
}
|
|
2490
|
+
}
|
|
2491
|
+
if (pushIdx !== -1) scopeStack.pop();
|
|
2492
|
+
}
|
|
2493
|
+
visit(tree.rootNode, 0);
|
|
2494
|
+
return { symbols };
|
|
2495
|
+
}
|
|
2496
|
+
function pushScope(scopeStack, node, queries) {
|
|
2497
|
+
const name = extractName(node, queries);
|
|
2498
|
+
if (!name) return -1;
|
|
2499
|
+
scopeStack.push(name);
|
|
2500
|
+
return scopeStack.length - 1;
|
|
2501
|
+
}
|
|
2502
|
+
function extractName(node, queries) {
|
|
2503
|
+
if (queries.nameExtractor) {
|
|
2504
|
+
const extracted = queries.nameExtractor(node);
|
|
2505
|
+
if (extracted) return extracted;
|
|
2506
|
+
}
|
|
2507
|
+
const fieldName = queries.nameField?.[node.type] ?? "name";
|
|
2508
|
+
const field = node.childForFieldName(fieldName);
|
|
2509
|
+
if (field) {
|
|
2510
|
+
if (IDENTIFIER_NODE_TYPES.has(field.type)) {
|
|
2511
|
+
return field.text;
|
|
2512
|
+
}
|
|
2513
|
+
const inner = field.childForFieldName("name") ?? field.namedChild(0);
|
|
2514
|
+
if (inner && IDENTIFIER_NODE_TYPES.has(inner.type)) {
|
|
2515
|
+
return inner.text;
|
|
2516
|
+
}
|
|
2517
|
+
}
|
|
2518
|
+
for (let i = 0; i < node.namedChildCount; i++) {
|
|
2519
|
+
const child = node.namedChild(i);
|
|
2520
|
+
if (child && IDENTIFIER_NODE_TYPES.has(child.type)) {
|
|
2521
|
+
return child.text;
|
|
2522
|
+
}
|
|
2523
|
+
}
|
|
2524
|
+
return null;
|
|
2525
|
+
}
|
|
2526
|
+
function emitSymbol(node, kind, file, lang, scopeStack, content, nlOffsets, queries) {
|
|
2527
|
+
const name = extractName(node, queries);
|
|
2528
|
+
if (!name) return null;
|
|
2529
|
+
const pos = node.startIndex;
|
|
2530
|
+
const { line, col } = lineColAt2(nlOffsets, pos);
|
|
2531
|
+
const end = Math.min(node.endIndex, content.length);
|
|
2532
|
+
const signature = content.slice(pos, end).replace(/\s+/g, " ").trim().slice(0, 500);
|
|
2533
|
+
const scope = scopeStack.join(".");
|
|
2534
|
+
const text = [name, signature].filter(Boolean).join(" | ").trim().slice(0, 1e3);
|
|
2535
|
+
return {
|
|
2536
|
+
id: 0,
|
|
2537
|
+
// caller assigns during bulk insertion
|
|
2538
|
+
lang,
|
|
2539
|
+
kind,
|
|
2540
|
+
name: name.slice(0, 200),
|
|
2541
|
+
file,
|
|
2542
|
+
line,
|
|
2543
|
+
col,
|
|
2544
|
+
signature,
|
|
2545
|
+
docComment: "",
|
|
2546
|
+
// doc-comment extraction lands with ref emission on Day 4
|
|
2547
|
+
scope,
|
|
2548
|
+
text
|
|
2549
|
+
};
|
|
2550
|
+
}
|
|
2551
|
+
var IDENTIFIER_NODE_TYPES;
|
|
2552
|
+
var init_visitor = __esm({
|
|
2553
|
+
"src/codebase-index/tree-sitter/visitor.ts"() {
|
|
2554
|
+
"use strict";
|
|
2555
|
+
init_util();
|
|
2556
|
+
IDENTIFIER_NODE_TYPES = /* @__PURE__ */ new Set([
|
|
2557
|
+
"identifier",
|
|
2558
|
+
"simple_identifier",
|
|
2559
|
+
"type_identifier",
|
|
2560
|
+
"field_identifier",
|
|
2561
|
+
"property_identifier",
|
|
2562
|
+
"name",
|
|
2563
|
+
"word",
|
|
2564
|
+
"variable_name",
|
|
2565
|
+
"constant",
|
|
2566
|
+
"sym"
|
|
2567
|
+
]);
|
|
2568
|
+
}
|
|
2569
|
+
});
|
|
2570
|
+
|
|
2571
|
+
// src/codebase-index/tree-sitter-parser.ts
|
|
2572
|
+
var tree_sitter_parser_exports = {};
|
|
2573
|
+
__export(tree_sitter_parser_exports, {
|
|
2574
|
+
__smokeRootType: () => __smokeRootType,
|
|
2575
|
+
getGrammarWasmPath: () => getGrammarWasmPath,
|
|
2576
|
+
isTreeSitterSupported: () => isTreeSitterSupported,
|
|
2577
|
+
loadTreeSitterLanguage: () => loadTreeSitterLanguage,
|
|
2578
|
+
parseSymbols: () => parseSymbols8
|
|
2579
|
+
});
|
|
2580
|
+
import * as path10 from "node:path";
|
|
2581
|
+
import { fileURLToPath } from "node:url";
|
|
2582
|
+
function optInEnabled(env) {
|
|
2583
|
+
return process.env[env] === "1" || process.env[env] === "true";
|
|
2584
|
+
}
|
|
2585
|
+
function getRuntime() {
|
|
2586
|
+
if (!runtimePromise) {
|
|
2587
|
+
runtimePromise = (async () => {
|
|
2588
|
+
const mod = await import("web-tree-sitter");
|
|
2589
|
+
const init = () => mod.Parser.init({ locateFile: () => RUNTIME_WASM });
|
|
2590
|
+
return { Parser: mod.Parser, Language: mod.Language, init };
|
|
2591
|
+
})();
|
|
2592
|
+
}
|
|
2593
|
+
return runtimePromise;
|
|
2594
|
+
}
|
|
2595
|
+
async function loadLanguage(lang) {
|
|
2596
|
+
const existing = languageCache.get(lang);
|
|
2597
|
+
if (existing) return existing;
|
|
2598
|
+
const promise = (async () => {
|
|
2599
|
+
const grammarName = resolveGrammarName(lang);
|
|
2600
|
+
if (!grammarName) {
|
|
2601
|
+
throw new Error(`tree-sitter: no grammar registered for lang "${lang}"`);
|
|
2602
|
+
}
|
|
2603
|
+
const wasmPath = path10.join(WASM_DIR, grammarName, `tree-sitter-${grammarName}.wasm`);
|
|
2604
|
+
const { Language, init } = await getRuntime();
|
|
2605
|
+
await init();
|
|
2606
|
+
const languageObj = await Language.load(wasmPath);
|
|
2607
|
+
return { lang, Language: languageObj };
|
|
2608
|
+
})();
|
|
2609
|
+
languageCache.set(lang, promise);
|
|
2610
|
+
return promise;
|
|
2611
|
+
}
|
|
2612
|
+
function resolveGrammarName(lang) {
|
|
2613
|
+
if (lang === "go" && optInEnabled(GO_OPT_IN)) return "go";
|
|
2614
|
+
if (lang === "py" && optInEnabled(PY_OPT_IN)) return "python";
|
|
2615
|
+
if (lang === "rs" && optInEnabled(RS_OPT_IN)) return "rust";
|
|
2616
|
+
return LANG_TO_GRAMMAR[lang];
|
|
2617
|
+
}
|
|
2618
|
+
function isTreeSitterSupported(lang) {
|
|
2619
|
+
return resolveGrammarName(lang) !== void 0;
|
|
2620
|
+
}
|
|
2621
|
+
function getGrammarWasmPath(lang) {
|
|
2622
|
+
const name = resolveGrammarName(lang);
|
|
2623
|
+
if (!name) return void 0;
|
|
2624
|
+
return path10.join(WASM_DIR, name, `tree-sitter-${name}.wasm`);
|
|
2625
|
+
}
|
|
2626
|
+
async function parseSymbols8(opts) {
|
|
2627
|
+
const { file, content, lang } = opts;
|
|
2628
|
+
if (!isTreeSitterSupported(lang)) {
|
|
2629
|
+
return { file, lang, symbols: [], mtimeMs: Date.now() };
|
|
2630
|
+
}
|
|
2631
|
+
try {
|
|
2632
|
+
const { Parser } = await getRuntime();
|
|
2633
|
+
const cached = await loadLanguage(lang);
|
|
2634
|
+
const parser = new Parser();
|
|
2635
|
+
parser.setLanguage(cached.Language);
|
|
2636
|
+
const tree = parser.parse(content);
|
|
2637
|
+
if (!tree) {
|
|
2638
|
+
return { file, lang, symbols: [], mtimeMs: Date.now() };
|
|
2639
|
+
}
|
|
2640
|
+
const { symbols } = visitTree(tree, content, file, lang, getQueries(lang));
|
|
2641
|
+
parser.delete();
|
|
2642
|
+
tree.delete();
|
|
2643
|
+
return { file, lang, symbols, refs: [], mtimeMs: Date.now() };
|
|
2644
|
+
} catch {
|
|
2645
|
+
return { file, lang, symbols: [], mtimeMs: Date.now() };
|
|
2646
|
+
}
|
|
2647
|
+
}
|
|
2648
|
+
async function loadTreeSitterLanguage(lang) {
|
|
2649
|
+
const cached = await loadLanguage(lang);
|
|
2650
|
+
return cached.Language;
|
|
2651
|
+
}
|
|
2652
|
+
async function __smokeRootType(opts) {
|
|
2653
|
+
if (!isTreeSitterSupported(opts.lang)) {
|
|
2654
|
+
throw new Error(`tree-sitter: no grammar registered for lang "${opts.lang}"`);
|
|
2655
|
+
}
|
|
2656
|
+
const { Parser } = await getRuntime();
|
|
2657
|
+
const cached = await loadLanguage(opts.lang);
|
|
2658
|
+
const parser = new Parser();
|
|
2659
|
+
parser.setLanguage(cached.Language);
|
|
2660
|
+
let tree = null;
|
|
2661
|
+
try {
|
|
2662
|
+
tree = parser.parse(opts.content);
|
|
2663
|
+
if (!tree) throw new Error("tree-sitter: parser.parse returned null");
|
|
2664
|
+
return tree.rootNode.type;
|
|
2665
|
+
} finally {
|
|
2666
|
+
tree?.delete();
|
|
2667
|
+
parser.delete();
|
|
2668
|
+
}
|
|
2669
|
+
}
|
|
2670
|
+
var WASM_DIR, RUNTIME_WASM, LANG_TO_GRAMMAR, GO_OPT_IN, PY_OPT_IN, RS_OPT_IN, runtimePromise, languageCache;
|
|
2671
|
+
var init_tree_sitter_parser = __esm({
|
|
2672
|
+
"src/codebase-index/tree-sitter-parser.ts"() {
|
|
2673
|
+
"use strict";
|
|
2674
|
+
init_queries();
|
|
2675
|
+
init_visitor();
|
|
2676
|
+
WASM_DIR = fileURLToPath(new URL("./wasm/", import.meta.url));
|
|
2677
|
+
RUNTIME_WASM = path10.join(WASM_DIR, "tree-sitter-runtime.wasm");
|
|
2678
|
+
LANG_TO_GRAMMAR = {
|
|
2679
|
+
c: "c",
|
|
2680
|
+
cpp: "cpp",
|
|
2681
|
+
java: "java",
|
|
2682
|
+
csharp: "c_sharp",
|
|
2683
|
+
// tree-sitter directory uses underscore
|
|
2684
|
+
php: "php",
|
|
2685
|
+
ruby: "ruby",
|
|
2686
|
+
swift: "swift",
|
|
2687
|
+
kotlin: "kotlin",
|
|
2688
|
+
shell: "bash",
|
|
2689
|
+
// we treat `.sh` / `.bash` / `.zsh` via the bash grammar
|
|
2690
|
+
// Go/Python/Rust are opt-in only (see goOptIn / pyOptIn / rsOptIn below)
|
|
2691
|
+
elixir: "elixir"
|
|
2692
|
+
};
|
|
2693
|
+
GO_OPT_IN = "WRONGSTACK_USE_TS_GO";
|
|
2694
|
+
PY_OPT_IN = "WRONGSTACK_USE_TS_PY";
|
|
2695
|
+
RS_OPT_IN = "WRONGSTACK_USE_TS_RS";
|
|
2696
|
+
runtimePromise = null;
|
|
2697
|
+
languageCache = /* @__PURE__ */ new Map();
|
|
2698
|
+
}
|
|
2699
|
+
});
|
|
2700
|
+
|
|
2130
2701
|
// src/read.ts
|
|
2131
|
-
import * as
|
|
2702
|
+
import * as fs13 from "node:fs/promises";
|
|
2132
2703
|
import { FsError, ToolValidationError } from "@wrongstack/core/types";
|
|
2133
2704
|
import { toErrorMessage as toErrorMessage2 } from "@wrongstack/core/utils";
|
|
2134
2705
|
|
|
@@ -2203,9 +2774,9 @@ function isBinaryBuffer(buf) {
|
|
|
2203
2774
|
}
|
|
2204
2775
|
|
|
2205
2776
|
// src/codebase-index/background-indexer.ts
|
|
2206
|
-
import * as
|
|
2207
|
-
import { fileURLToPath as
|
|
2208
|
-
import { Worker } from "node:worker_threads";
|
|
2777
|
+
import * as fs12 from "node:fs";
|
|
2778
|
+
import { fileURLToPath as fileURLToPath5 } from "node:url";
|
|
2779
|
+
import { Worker as Worker2 } from "node:worker_threads";
|
|
2209
2780
|
|
|
2210
2781
|
// src/codebase-index/circuit-breaker.ts
|
|
2211
2782
|
var IndexTimeoutError = class extends Error {
|
|
@@ -2287,9 +2858,9 @@ var indexCircuitBreaker = new IndexCircuitBreaker();
|
|
|
2287
2858
|
// src/codebase-index/indexer.ts
|
|
2288
2859
|
import { expectDefined as expectDefined5 } from "@wrongstack/core/utils";
|
|
2289
2860
|
import { execFile } from "node:child_process";
|
|
2290
|
-
import * as
|
|
2861
|
+
import * as fs9 from "node:fs/promises";
|
|
2291
2862
|
import { availableParallelism } from "node:os";
|
|
2292
|
-
import * as
|
|
2863
|
+
import * as path13 from "node:path";
|
|
2293
2864
|
import {
|
|
2294
2865
|
DEFAULT_WALK_IGNORE_DIRS,
|
|
2295
2866
|
indexParallelBatchSize,
|
|
@@ -3091,32 +3662,56 @@ async function dispatch(file, content, lang) {
|
|
|
3091
3662
|
case "tsx":
|
|
3092
3663
|
case "js":
|
|
3093
3664
|
case "jsx": {
|
|
3094
|
-
const { parseSymbols:
|
|
3095
|
-
return
|
|
3665
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_ts_parser(), ts_parser_exports));
|
|
3666
|
+
return parseSymbols9({ file, content, lang });
|
|
3096
3667
|
}
|
|
3097
3668
|
case "go": {
|
|
3098
|
-
const { parseSymbols:
|
|
3099
|
-
return
|
|
3669
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_go_parser(), go_parser_exports));
|
|
3670
|
+
return parseSymbols9({ file, content, lang: "go" });
|
|
3100
3671
|
}
|
|
3101
3672
|
case "py": {
|
|
3102
|
-
const { parseSymbols:
|
|
3103
|
-
return
|
|
3673
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_py_parser(), py_parser_exports));
|
|
3674
|
+
return parseSymbols9({ file, content, lang: "py" });
|
|
3104
3675
|
}
|
|
3105
3676
|
case "rs": {
|
|
3106
|
-
const { parseSymbols:
|
|
3107
|
-
return
|
|
3677
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_rs_parser(), rs_parser_exports));
|
|
3678
|
+
return parseSymbols9({ file, content, lang: "rs" });
|
|
3108
3679
|
}
|
|
3109
3680
|
case "json": {
|
|
3110
|
-
const { parseSymbols:
|
|
3111
|
-
return
|
|
3681
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_json_parser(), json_parser_exports));
|
|
3682
|
+
return parseSymbols9({ file, content, lang: "json" });
|
|
3112
3683
|
}
|
|
3113
3684
|
case "yaml": {
|
|
3114
|
-
const { parseSymbols:
|
|
3115
|
-
return
|
|
3685
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_yaml_parser(), yaml_parser_exports));
|
|
3686
|
+
return parseSymbols9({ file, content, lang: "yaml" });
|
|
3687
|
+
}
|
|
3688
|
+
// Phase 1: ten languages now route through the Tree-Sitter WASM
|
|
3689
|
+
// universal parser (`tree-sitter-parser.ts`). The dispatch falls back to
|
|
3690
|
+
// the regex extractor in `generic-parser.ts` whenever WASM loading fails
|
|
3691
|
+
// or the parser returns zero symbols — preserving the indexable-file
|
|
3692
|
+
// contract that "missing a parser must never mean skipping the file".
|
|
3693
|
+
case "c":
|
|
3694
|
+
case "cpp":
|
|
3695
|
+
case "java":
|
|
3696
|
+
case "csharp":
|
|
3697
|
+
case "php":
|
|
3698
|
+
case "ruby":
|
|
3699
|
+
case "swift":
|
|
3700
|
+
case "kotlin":
|
|
3701
|
+
case "shell":
|
|
3702
|
+
case "elixir": {
|
|
3703
|
+
try {
|
|
3704
|
+
const { parseSymbols: parseSymbols10 } = await Promise.resolve().then(() => (init_tree_sitter_parser(), tree_sitter_parser_exports));
|
|
3705
|
+
const parsed = await parseSymbols10({ file, content, lang });
|
|
3706
|
+
if (parsed.symbols.length > 0) return parsed;
|
|
3707
|
+
} catch {
|
|
3708
|
+
}
|
|
3709
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_generic_parser(), generic_parser_exports));
|
|
3710
|
+
return parseSymbols9({ file, content, lang });
|
|
3116
3711
|
}
|
|
3117
3712
|
default: {
|
|
3118
|
-
const { parseSymbols:
|
|
3119
|
-
return
|
|
3713
|
+
const { parseSymbols: parseSymbols9 } = await Promise.resolve().then(() => (init_generic_parser(), generic_parser_exports));
|
|
3714
|
+
return parseSymbols9({ file, content, lang });
|
|
3120
3715
|
}
|
|
3121
3716
|
}
|
|
3122
3717
|
}
|
|
@@ -3128,10 +3723,265 @@ function withRelations(parsed, content, lang) {
|
|
|
3128
3723
|
return { ...parsed, refs: refs.map((ref) => ref.lang ? ref : { ...ref, lang }) };
|
|
3129
3724
|
}
|
|
3130
3725
|
|
|
3726
|
+
// src/codebase-index/parser-worker-pool.ts
|
|
3727
|
+
import { Worker } from "node:worker_threads";
|
|
3728
|
+
import { fileURLToPath as fileURLToPath2 } from "node:url";
|
|
3729
|
+
import * as fs6 from "node:fs";
|
|
3730
|
+
var WORKER_POOL_THRESHOLD = 500;
|
|
3731
|
+
var ParserWorkerPool = class {
|
|
3732
|
+
constructor(maxWorkers = defaultWorkerCount()) {
|
|
3733
|
+
this.maxWorkers = maxWorkers;
|
|
3734
|
+
}
|
|
3735
|
+
maxWorkers;
|
|
3736
|
+
workers = [];
|
|
3737
|
+
nextBatchId = 1;
|
|
3738
|
+
pending = /* @__PURE__ */ new Map();
|
|
3739
|
+
creating = false;
|
|
3740
|
+
unavailable = false;
|
|
3741
|
+
/**
|
|
3742
|
+
* True if the pool is available for use. Returns false when:
|
|
3743
|
+
* - Worker threads aren't supported (sandbox, exotic runtime)
|
|
3744
|
+
* - The built worker script can't be found
|
|
3745
|
+
* - Pool creation was attempted and failed
|
|
3746
|
+
*/
|
|
3747
|
+
isAvailable() {
|
|
3748
|
+
return !this.unavailable && this.workers.length > 0;
|
|
3749
|
+
}
|
|
3750
|
+
/**
|
|
3751
|
+
* Lazily create the worker pool. Returns true if the pool is ready, false
|
|
3752
|
+
* if it's unavailable (caller should fall back to inline parsing).
|
|
3753
|
+
*/
|
|
3754
|
+
async ensureReady() {
|
|
3755
|
+
if (this.isAvailable()) return true;
|
|
3756
|
+
if (this.unavailable) return false;
|
|
3757
|
+
if (this.creating) {
|
|
3758
|
+
await new Promise((r) => setTimeout(r, 50));
|
|
3759
|
+
return this.isAvailable();
|
|
3760
|
+
}
|
|
3761
|
+
this.creating = true;
|
|
3762
|
+
try {
|
|
3763
|
+
const url = resolveWorkerScriptUrl();
|
|
3764
|
+
if (!url) {
|
|
3765
|
+
this.unavailable = true;
|
|
3766
|
+
return false;
|
|
3767
|
+
}
|
|
3768
|
+
for (let i = 0; i < this.maxWorkers; i++) {
|
|
3769
|
+
try {
|
|
3770
|
+
const w = new Worker(url, { name: `wstack-parser-${i}` });
|
|
3771
|
+
w.unref();
|
|
3772
|
+
w.on("message", (msg) => this.handleMessage(msg));
|
|
3773
|
+
w.on("error", (err) => this.handleError(err, w));
|
|
3774
|
+
this.workers.push({ worker: w, busy: false });
|
|
3775
|
+
} catch {
|
|
3776
|
+
if (this.workers.length === 0) {
|
|
3777
|
+
this.unavailable = true;
|
|
3778
|
+
return false;
|
|
3779
|
+
}
|
|
3780
|
+
break;
|
|
3781
|
+
}
|
|
3782
|
+
}
|
|
3783
|
+
return this.workers.length > 0;
|
|
3784
|
+
} finally {
|
|
3785
|
+
this.creating = false;
|
|
3786
|
+
}
|
|
3787
|
+
}
|
|
3788
|
+
/**
|
|
3789
|
+
* Parse files in parallel across the worker pool. Returns a flat
|
|
3790
|
+
* `FileSymbols[]` in completion order (caller sorts if needed).
|
|
3791
|
+
*
|
|
3792
|
+
* Content is pre-read by the main thread (for the content-hash check)
|
|
3793
|
+
* and passed to workers to avoid a second disk read. Files are
|
|
3794
|
+
* distributed round-robin across workers.
|
|
3795
|
+
*/
|
|
3796
|
+
async parseFiles(files) {
|
|
3797
|
+
if (!this.isAvailable()) {
|
|
3798
|
+
throw new Error("ParserWorkerPool.parseFiles called before ensureReady() succeeded");
|
|
3799
|
+
}
|
|
3800
|
+
if (files.length === 0) return [];
|
|
3801
|
+
const batchId = this.nextBatchId++;
|
|
3802
|
+
const workerCount = Math.min(this.workers.length, files.length);
|
|
3803
|
+
const chunks = Array.from(
|
|
3804
|
+
{ length: workerCount },
|
|
3805
|
+
() => []
|
|
3806
|
+
);
|
|
3807
|
+
for (let i = 0; i < files.length; i++) {
|
|
3808
|
+
chunks[i % workerCount].push(files[i]);
|
|
3809
|
+
}
|
|
3810
|
+
return new Promise((resolve4, reject) => {
|
|
3811
|
+
this.pending.set(batchId, {
|
|
3812
|
+
resolve: resolve4,
|
|
3813
|
+
reject,
|
|
3814
|
+
accumulated: [],
|
|
3815
|
+
expectedWorkers: workerCount,
|
|
3816
|
+
completedWorkers: 0
|
|
3817
|
+
});
|
|
3818
|
+
for (let i = 0; i < workerCount; i++) {
|
|
3819
|
+
const pw = this.workers[i];
|
|
3820
|
+
pw.busy = true;
|
|
3821
|
+
pw.worker.postMessage({
|
|
3822
|
+
type: "parse",
|
|
3823
|
+
id: batchId,
|
|
3824
|
+
files: chunks[i]
|
|
3825
|
+
});
|
|
3826
|
+
}
|
|
3827
|
+
});
|
|
3828
|
+
}
|
|
3829
|
+
/** Shut down all workers. Safe to call multiple times. */
|
|
3830
|
+
async shutdown() {
|
|
3831
|
+
const workers = this.workers.map((w) => w.worker);
|
|
3832
|
+
this.workers = [];
|
|
3833
|
+
this.unavailable = false;
|
|
3834
|
+
for (const w of workers) {
|
|
3835
|
+
try {
|
|
3836
|
+
w.postMessage({ type: "shutdown" });
|
|
3837
|
+
} catch {
|
|
3838
|
+
}
|
|
3839
|
+
}
|
|
3840
|
+
await Promise.allSettled(
|
|
3841
|
+
workers.map(
|
|
3842
|
+
(w) => Promise.race([
|
|
3843
|
+
new Promise((resolve4) => {
|
|
3844
|
+
w.once("exit", () => resolve4());
|
|
3845
|
+
}),
|
|
3846
|
+
new Promise((resolve4) => setTimeout(() => resolve4(), 2e3))
|
|
3847
|
+
]).then(() => {
|
|
3848
|
+
if (!w.threadId) return;
|
|
3849
|
+
return w.terminate().catch(() => {
|
|
3850
|
+
});
|
|
3851
|
+
})
|
|
3852
|
+
)
|
|
3853
|
+
);
|
|
3854
|
+
for (const [, p] of this.pending) p.reject(new Error("ParserWorkerPool shut down"));
|
|
3855
|
+
this.pending.clear();
|
|
3856
|
+
}
|
|
3857
|
+
handleMessage(msg) {
|
|
3858
|
+
const batch = this.pending.get(msg.id);
|
|
3859
|
+
if (!batch) return;
|
|
3860
|
+
batch.accumulated.push(...msg.results);
|
|
3861
|
+
batch.completedWorkers++;
|
|
3862
|
+
const freeWorker = this.workers.find((w) => w.busy);
|
|
3863
|
+
if (freeWorker) freeWorker.busy = false;
|
|
3864
|
+
if (batch.completedWorkers >= batch.expectedWorkers) {
|
|
3865
|
+
this.pending.delete(msg.id);
|
|
3866
|
+
batch.resolve(batch.accumulated);
|
|
3867
|
+
}
|
|
3868
|
+
}
|
|
3869
|
+
handleError(err, source) {
|
|
3870
|
+
this.workers = this.workers.filter((w) => w.worker !== source);
|
|
3871
|
+
if (this.workers.length === 0) {
|
|
3872
|
+
for (const [, p] of this.pending) p.reject(err);
|
|
3873
|
+
this.pending.clear();
|
|
3874
|
+
this.unavailable = true;
|
|
3875
|
+
}
|
|
3876
|
+
}
|
|
3877
|
+
};
|
|
3878
|
+
function defaultWorkerCount() {
|
|
3879
|
+
const cores = globalThis.navigator?.hardwareConcurrency ?? 4;
|
|
3880
|
+
return Math.max(1, Math.min(4, cores - 1));
|
|
3881
|
+
}
|
|
3882
|
+
function resolveWorkerScriptUrl() {
|
|
3883
|
+
for (const rel of [
|
|
3884
|
+
"./parser-worker-script.js",
|
|
3885
|
+
"./codebase-index/parser-worker-script.js"
|
|
3886
|
+
]) {
|
|
3887
|
+
try {
|
|
3888
|
+
const url = new URL(rel, import.meta.url);
|
|
3889
|
+
if (url.protocol === "file:" && fs6.existsSync(fileURLToPath2(url))) return url;
|
|
3890
|
+
} catch {
|
|
3891
|
+
}
|
|
3892
|
+
}
|
|
3893
|
+
return null;
|
|
3894
|
+
}
|
|
3895
|
+
var _pool = null;
|
|
3896
|
+
function getParserPool() {
|
|
3897
|
+
_pool ??= new ParserWorkerPool();
|
|
3898
|
+
return _pool;
|
|
3899
|
+
}
|
|
3900
|
+
|
|
3901
|
+
// src/codebase-index/content-hash.ts
|
|
3902
|
+
var PRIME64_1 = 0x9e3779b185ebca87n;
|
|
3903
|
+
var PRIME64_2 = 0xc2b2ae3d27d4eb4fn;
|
|
3904
|
+
var PRIME64_3 = 0x165667b19e3779f9n;
|
|
3905
|
+
var PRIME64_4 = 0x85ebca77c2b2ae63n;
|
|
3906
|
+
var PRIME64_5 = 0x27d4eb2f165667c5n;
|
|
3907
|
+
var MASK64 = 0xffffffffffffffffn;
|
|
3908
|
+
function mul64(a, b) {
|
|
3909
|
+
return (a & MASK64) * (b & MASK64) & MASK64;
|
|
3910
|
+
}
|
|
3911
|
+
function rotl64(x, n) {
|
|
3912
|
+
const v = x & MASK64;
|
|
3913
|
+
return (v << BigInt(n) | v >> BigInt(64 - n)) & MASK64;
|
|
3914
|
+
}
|
|
3915
|
+
function readU64LE(buf, off) {
|
|
3916
|
+
let v = 0n;
|
|
3917
|
+
for (let i = 7; i >= 0; i--) {
|
|
3918
|
+
v = v << 8n | BigInt(buf[off + i] ?? 0);
|
|
3919
|
+
}
|
|
3920
|
+
return v & MASK64;
|
|
3921
|
+
}
|
|
3922
|
+
function readU32LE(buf, off) {
|
|
3923
|
+
return (BigInt(buf[off] ?? 0) | BigInt(buf[off + 1] ?? 0) << 8n | BigInt(buf[off + 2] ?? 0) << 16n | BigInt(buf[off + 3] ?? 0) << 24n) & MASK64;
|
|
3924
|
+
}
|
|
3925
|
+
function xxh64Round(acc, lane) {
|
|
3926
|
+
return mul64(rotl64(acc + mul64(lane, PRIME64_2) & MASK64, 31), PRIME64_1);
|
|
3927
|
+
}
|
|
3928
|
+
function xxh64MergeRound(acc, val) {
|
|
3929
|
+
return mul64(acc ^ xxh64Round(0n, val), PRIME64_1) + PRIME64_4 & MASK64;
|
|
3930
|
+
}
|
|
3931
|
+
function xxhash64Hex(buf, explicitLen) {
|
|
3932
|
+
const length = explicitLen ?? buf.length;
|
|
3933
|
+
let h;
|
|
3934
|
+
let off = 0;
|
|
3935
|
+
if (length >= 32) {
|
|
3936
|
+
let v1 = PRIME64_1 + PRIME64_2 & MASK64;
|
|
3937
|
+
let v2 = PRIME64_2;
|
|
3938
|
+
let v3 = 0n;
|
|
3939
|
+
let v4 = 0n - PRIME64_1 & MASK64;
|
|
3940
|
+
const end32 = length - 32;
|
|
3941
|
+
while (off <= end32) {
|
|
3942
|
+
v1 = xxh64Round(v1, readU64LE(buf, off));
|
|
3943
|
+
v2 = xxh64Round(v2, readU64LE(buf, off + 8));
|
|
3944
|
+
v3 = xxh64Round(v3, readU64LE(buf, off + 16));
|
|
3945
|
+
v4 = xxh64Round(v4, readU64LE(buf, off + 24));
|
|
3946
|
+
off += 32;
|
|
3947
|
+
}
|
|
3948
|
+
h = rotl64(v1, 1) + rotl64(v2, 7) + rotl64(v3, 12) + rotl64(v4, 18) & MASK64;
|
|
3949
|
+
h = xxh64MergeRound(h, v1);
|
|
3950
|
+
h = xxh64MergeRound(h, v2);
|
|
3951
|
+
h = xxh64MergeRound(h, v3);
|
|
3952
|
+
h = xxh64MergeRound(h, v4);
|
|
3953
|
+
} else {
|
|
3954
|
+
h = PRIME64_5;
|
|
3955
|
+
}
|
|
3956
|
+
h = h + BigInt(length) & MASK64;
|
|
3957
|
+
while (off + 8 <= length) {
|
|
3958
|
+
const k1 = xxh64Round(0n, readU64LE(buf, off));
|
|
3959
|
+
h = mul64(rotl64(h ^ k1, 27), PRIME64_1) + PRIME64_4 & MASK64;
|
|
3960
|
+
off += 8;
|
|
3961
|
+
}
|
|
3962
|
+
if (off + 4 <= length) {
|
|
3963
|
+
h = mul64(rotl64(h ^ mul64(readU32LE(buf, off), PRIME64_1), 23), PRIME64_2) + PRIME64_3 & MASK64;
|
|
3964
|
+
off += 4;
|
|
3965
|
+
}
|
|
3966
|
+
while (off < length) {
|
|
3967
|
+
h = mul64(rotl64(h ^ mul64(BigInt(buf[off] ?? 0), PRIME64_5), 11), PRIME64_1) & MASK64;
|
|
3968
|
+
off += 1;
|
|
3969
|
+
}
|
|
3970
|
+
h = (h ^ h >> 33n) & MASK64;
|
|
3971
|
+
h = mul64(h, PRIME64_2);
|
|
3972
|
+
h = (h ^ h >> 29n) & MASK64;
|
|
3973
|
+
h = mul64(h, PRIME64_3);
|
|
3974
|
+
h = (h ^ h >> 32n) & MASK64;
|
|
3975
|
+
return h.toString(16).padStart(16, "0");
|
|
3976
|
+
}
|
|
3977
|
+
function xxhash64String(content) {
|
|
3978
|
+
return xxhash64Hex(new TextEncoder().encode(content));
|
|
3979
|
+
}
|
|
3980
|
+
|
|
3131
3981
|
// src/codebase-index/writer.ts
|
|
3132
3982
|
import { expectDefined as expectDefined4 } from "@wrongstack/core/utils";
|
|
3133
|
-
import * as
|
|
3134
|
-
import * as
|
|
3983
|
+
import * as fs8 from "node:fs";
|
|
3984
|
+
import * as path12 from "node:path";
|
|
3135
3985
|
|
|
3136
3986
|
// src/codebase-index/bm25.ts
|
|
3137
3987
|
var K1 = 1.5;
|
|
@@ -3329,8 +4179,8 @@ function runSqliteWithRetry(fn) {
|
|
|
3329
4179
|
}
|
|
3330
4180
|
|
|
3331
4181
|
// src/codebase-index/writer-admin.ts
|
|
3332
|
-
import * as
|
|
3333
|
-
import * as
|
|
4182
|
+
import * as fs7 from "node:fs";
|
|
4183
|
+
import * as path11 from "node:path";
|
|
3334
4184
|
var DB_FILE = "index.db";
|
|
3335
4185
|
function getAllIndexableWithStatement(stmt) {
|
|
3336
4186
|
return stmt("SELECT id, text FROM symbols").all().map(({ id, text }) => ({ id, text }));
|
|
@@ -3366,7 +4216,7 @@ function getMetadataWithStatement(stmt, key) {
|
|
|
3366
4216
|
}
|
|
3367
4217
|
function getFileMetaWithStatement(stmt, file) {
|
|
3368
4218
|
const rows = stmt(
|
|
3369
|
-
"SELECT file, lang, mtime_ms, symbol_count, last_indexed FROM files WHERE file = ?"
|
|
4219
|
+
"SELECT file, lang, mtime_ms, symbol_count, last_indexed, content_hash FROM files WHERE file = ?"
|
|
3370
4220
|
).all(file);
|
|
3371
4221
|
const r = rows[0];
|
|
3372
4222
|
if (!r) return null;
|
|
@@ -3375,21 +4225,25 @@ function getFileMetaWithStatement(stmt, file) {
|
|
|
3375
4225
|
lang: r.lang,
|
|
3376
4226
|
mtimeMs: r.mtime_ms,
|
|
3377
4227
|
symbolCount: r.symbol_count,
|
|
3378
|
-
lastIndexed: r.last_indexed
|
|
4228
|
+
lastIndexed: r.last_indexed,
|
|
4229
|
+
contentHash: r.content_hash
|
|
3379
4230
|
};
|
|
3380
4231
|
}
|
|
3381
4232
|
function getAllFileMetasWithStatement(stmt) {
|
|
3382
|
-
return stmt(
|
|
4233
|
+
return stmt(
|
|
4234
|
+
"SELECT file, lang, mtime_ms, symbol_count, last_indexed, content_hash FROM files"
|
|
4235
|
+
).all().map((r) => ({
|
|
3383
4236
|
file: r.file,
|
|
3384
4237
|
lang: r.lang,
|
|
3385
4238
|
mtimeMs: r.mtime_ms,
|
|
3386
4239
|
symbolCount: r.symbol_count,
|
|
3387
|
-
lastIndexed: r.last_indexed
|
|
4240
|
+
lastIndexed: r.last_indexed,
|
|
4241
|
+
contentHash: r.content_hash
|
|
3388
4242
|
}));
|
|
3389
4243
|
}
|
|
3390
4244
|
function getIndexDbSizeBytes(indexDir) {
|
|
3391
4245
|
try {
|
|
3392
|
-
return
|
|
4246
|
+
return fs7.statSync(path11.join(indexDir, DB_FILE)).size;
|
|
3393
4247
|
} catch {
|
|
3394
4248
|
return 0;
|
|
3395
4249
|
}
|
|
@@ -3438,6 +4292,18 @@ function bulkInsertFtsWithStatement(stmt, maxSqlVars, ftsAvailable, rows) {
|
|
|
3438
4292
|
insert.run(...binds);
|
|
3439
4293
|
}
|
|
3440
4294
|
}
|
|
4295
|
+
function bulkInsertVectorsWithStatement(stmt, maxSqlVars, rows) {
|
|
4296
|
+
if (rows.length === 0) return;
|
|
4297
|
+
const chunkSize = Math.max(1, Math.floor(maxSqlVars / 2));
|
|
4298
|
+
for (let i = 0; i < rows.length; i += chunkSize) {
|
|
4299
|
+
const chunk = rows.slice(i, i + chunkSize);
|
|
4300
|
+
const placeholders = chunk.map(() => "(?, ?)").join(", ");
|
|
4301
|
+
const insert = stmt(`INSERT INTO symbol_vectors(symbol_id, vector) VALUES ${placeholders}`);
|
|
4302
|
+
const binds = [];
|
|
4303
|
+
for (const r of chunk) binds.push(r.id, r.vector);
|
|
4304
|
+
insert.run(...binds);
|
|
4305
|
+
}
|
|
4306
|
+
}
|
|
3441
4307
|
function bulkInsertRefsWithStatement(stmt, maxSqlVars, refs) {
|
|
3442
4308
|
if (refs.length === 0) return;
|
|
3443
4309
|
const chunkSize = Math.max(1, Math.floor(maxSqlVars / 8));
|
|
@@ -3742,6 +4608,171 @@ function findOutgoingCallsByName(stmt, symbolName, file, limit) {
|
|
|
3742
4608
|
const calls = rows.map(mapCallSiteRow).slice(0, limit);
|
|
3743
4609
|
return { calls, symbolFound: true, unresolvedCount, totalMatches: rows.length };
|
|
3744
4610
|
}
|
|
4611
|
+
function runCteWithSeeds(stmt, seedIds, buildSql) {
|
|
4612
|
+
if (seedIds.length <= 900) {
|
|
4613
|
+
const ph = seedIds.map(() => "?").join(",");
|
|
4614
|
+
return stmt(buildSql(ph)).all(...seedIds);
|
|
4615
|
+
}
|
|
4616
|
+
stmt("DROP TABLE IF EXISTS _cte_seeds").run();
|
|
4617
|
+
try {
|
|
4618
|
+
stmt("CREATE TEMP TABLE _cte_seeds (id INTEGER PRIMARY KEY)").run();
|
|
4619
|
+
for (let i = 0; i < seedIds.length; i += 500) {
|
|
4620
|
+
const chunk = seedIds.slice(i, i + 500);
|
|
4621
|
+
const ph = chunk.map(() => "(?)").join(",");
|
|
4622
|
+
stmt(`INSERT OR IGNORE INTO _cte_seeds (id) VALUES ${ph}`).run(...chunk);
|
|
4623
|
+
}
|
|
4624
|
+
return stmt(buildSql("SELECT id FROM _cte_seeds")).all();
|
|
4625
|
+
} finally {
|
|
4626
|
+
stmt("DROP TABLE IF EXISTS _cte_seeds").run();
|
|
4627
|
+
}
|
|
4628
|
+
}
|
|
4629
|
+
function findTransitiveIncomingCallsByName(stmt, symbolName, file, limit) {
|
|
4630
|
+
const targetIds = resolveSymbolIds(stmt, symbolName, file);
|
|
4631
|
+
if (targetIds.length === 0)
|
|
4632
|
+
return { calls: [], symbolFound: false, ambiguous: false, totalMatches: 0 };
|
|
4633
|
+
let matchIds = targetIds;
|
|
4634
|
+
let ambiguous = false;
|
|
4635
|
+
if (file !== void 0) {
|
|
4636
|
+
const allNamedIds = resolveSymbolIds(stmt, symbolName, void 0);
|
|
4637
|
+
if (allNamedIds.length > targetIds.length) {
|
|
4638
|
+
matchIds = allNamedIds;
|
|
4639
|
+
ambiguous = true;
|
|
4640
|
+
}
|
|
4641
|
+
}
|
|
4642
|
+
const cteSql = (seedSource) => `WITH RECURSIVE incoming_tree(from_id) AS (
|
|
4643
|
+
SELECT r.from_id
|
|
4644
|
+
FROM refs r
|
|
4645
|
+
WHERE r.to_id IN (${seedSource})
|
|
4646
|
+
|
|
4647
|
+
UNION
|
|
4648
|
+
|
|
4649
|
+
SELECT r.from_id
|
|
4650
|
+
FROM refs r
|
|
4651
|
+
JOIN incoming_tree it ON r.to_id = it.from_id
|
|
4652
|
+
)
|
|
4653
|
+
SELECT
|
|
4654
|
+
s.id AS sym_id,
|
|
4655
|
+
s.name AS sym_name,
|
|
4656
|
+
s.kind AS sym_kind,
|
|
4657
|
+
s.lang AS sym_lang,
|
|
4658
|
+
s.file AS sym_file,
|
|
4659
|
+
s.line AS sym_line,
|
|
4660
|
+
s.signature AS sym_signature,
|
|
4661
|
+
'' AS call_type,
|
|
4662
|
+
0 AS ref_line
|
|
4663
|
+
FROM incoming_tree it
|
|
4664
|
+
JOIN symbols s ON s.id = it.from_id
|
|
4665
|
+
GROUP BY s.id
|
|
4666
|
+
ORDER BY s.file, s.line`;
|
|
4667
|
+
const rows = runCteWithSeeds(stmt, matchIds, cteSql);
|
|
4668
|
+
if (!file) {
|
|
4669
|
+
const fallbackRows = stmt(
|
|
4670
|
+
`SELECT
|
|
4671
|
+
s.id AS sym_id,
|
|
4672
|
+
s.name AS sym_name,
|
|
4673
|
+
s.kind AS sym_kind,
|
|
4674
|
+
s.lang AS sym_lang,
|
|
4675
|
+
s.file AS sym_file,
|
|
4676
|
+
s.line AS sym_line,
|
|
4677
|
+
s.signature AS sym_signature,
|
|
4678
|
+
r.call_type,
|
|
4679
|
+
r.line AS ref_line
|
|
4680
|
+
FROM refs r
|
|
4681
|
+
JOIN symbols s ON s.id = r.from_id
|
|
4682
|
+
WHERE r.to_id IS NULL AND r.to_name = ?
|
|
4683
|
+
ORDER BY r.line, r.id`
|
|
4684
|
+
).all(symbolName);
|
|
4685
|
+
rows.push(...fallbackRows);
|
|
4686
|
+
}
|
|
4687
|
+
rows.sort((a, b) => a.ref_line - b.ref_line || a.sym_id - b.sym_id);
|
|
4688
|
+
const allCalls = rows.map(mapCallSiteRow);
|
|
4689
|
+
return {
|
|
4690
|
+
calls: allCalls.slice(0, limit),
|
|
4691
|
+
symbolFound: true,
|
|
4692
|
+
ambiguous,
|
|
4693
|
+
totalMatches: allCalls.length
|
|
4694
|
+
};
|
|
4695
|
+
}
|
|
4696
|
+
function findTransitiveOutgoingCallsByName(stmt, symbolName, file, limit) {
|
|
4697
|
+
const sourceIds = resolveSymbolIds(stmt, symbolName, file);
|
|
4698
|
+
if (sourceIds.length === 0)
|
|
4699
|
+
return { calls: [], symbolFound: false, unresolvedCount: 0, totalMatches: 0 };
|
|
4700
|
+
const unresolvedCount = chunkedIdScalar(
|
|
4701
|
+
stmt,
|
|
4702
|
+
sourceIds,
|
|
4703
|
+
(ph) => `SELECT COUNT(*) AS n FROM refs WHERE from_id IN (${ph}) AND to_id IS NULL`
|
|
4704
|
+
);
|
|
4705
|
+
const cteSql = (seedSource) => `WITH RECURSIVE outgoing_tree(to_id) AS (
|
|
4706
|
+
SELECT r.to_id
|
|
4707
|
+
FROM refs r
|
|
4708
|
+
WHERE r.from_id IN (${seedSource}) AND r.to_id IS NOT NULL
|
|
4709
|
+
|
|
4710
|
+
UNION
|
|
4711
|
+
|
|
4712
|
+
SELECT r.to_id
|
|
4713
|
+
FROM refs r
|
|
4714
|
+
JOIN outgoing_tree ot ON r.from_id = ot.to_id
|
|
4715
|
+
WHERE r.to_id IS NOT NULL
|
|
4716
|
+
)
|
|
4717
|
+
SELECT
|
|
4718
|
+
s.id AS sym_id,
|
|
4719
|
+
s.name AS sym_name,
|
|
4720
|
+
s.kind AS sym_kind,
|
|
4721
|
+
s.lang AS sym_lang,
|
|
4722
|
+
s.file AS sym_file,
|
|
4723
|
+
s.line AS sym_line,
|
|
4724
|
+
s.signature AS sym_signature,
|
|
4725
|
+
'' AS call_type,
|
|
4726
|
+
0 AS ref_line
|
|
4727
|
+
FROM outgoing_tree ot
|
|
4728
|
+
JOIN symbols s ON s.id = ot.to_id
|
|
4729
|
+
GROUP BY s.id
|
|
4730
|
+
ORDER BY s.file, s.line`;
|
|
4731
|
+
const rows = runCteWithSeeds(stmt, sourceIds, cteSql);
|
|
4732
|
+
const calls = rows.map(mapCallSiteRow).slice(0, limit);
|
|
4733
|
+
return { calls, symbolFound: true, unresolvedCount, totalMatches: rows.length };
|
|
4734
|
+
}
|
|
4735
|
+
function findReachableSymbolIds(stmt, seedIds) {
|
|
4736
|
+
if (seedIds.length === 0) return /* @__PURE__ */ new Set();
|
|
4737
|
+
if (seedIds.length > 900) {
|
|
4738
|
+
stmt("DROP TABLE IF EXISTS _seeds").run();
|
|
4739
|
+
try {
|
|
4740
|
+
stmt("CREATE TEMP TABLE _seeds (id INTEGER PRIMARY KEY)").run();
|
|
4741
|
+
for (let i = 0; i < seedIds.length; i += 500) {
|
|
4742
|
+
const chunk = seedIds.slice(i, i + 500);
|
|
4743
|
+
const ph2 = chunk.map(() => "(?)").join(",");
|
|
4744
|
+
stmt(`INSERT OR IGNORE INTO _seeds (id) VALUES ${ph2}`).run(...chunk);
|
|
4745
|
+
}
|
|
4746
|
+
const rows2 = stmt(
|
|
4747
|
+
`WITH RECURSIVE reachable(id) AS (
|
|
4748
|
+
SELECT id FROM _seeds
|
|
4749
|
+
UNION
|
|
4750
|
+
SELECT r.to_id
|
|
4751
|
+
FROM refs r
|
|
4752
|
+
JOIN reachable ON r.from_id = reachable.id
|
|
4753
|
+
WHERE r.to_id IS NOT NULL
|
|
4754
|
+
)
|
|
4755
|
+
SELECT DISTINCT id FROM reachable`
|
|
4756
|
+
).all();
|
|
4757
|
+
return new Set(rows2.map((r) => r.id));
|
|
4758
|
+
} finally {
|
|
4759
|
+
stmt("DROP TABLE IF EXISTS _seeds").run();
|
|
4760
|
+
}
|
|
4761
|
+
}
|
|
4762
|
+
const ph = seedIds.map(() => "?").join(",");
|
|
4763
|
+
const rows = stmt(
|
|
4764
|
+
`WITH RECURSIVE reachable(id) AS (
|
|
4765
|
+
SELECT id FROM symbols WHERE id IN (${ph})
|
|
4766
|
+
UNION
|
|
4767
|
+
SELECT r.to_id
|
|
4768
|
+
FROM refs r
|
|
4769
|
+
JOIN reachable ON r.from_id = reachable.id
|
|
4770
|
+
WHERE r.to_id IS NOT NULL
|
|
4771
|
+
)
|
|
4772
|
+
SELECT DISTINCT id FROM reachable`
|
|
4773
|
+
).all(...seedIds);
|
|
4774
|
+
return new Set(rows.map((r) => r.id));
|
|
4775
|
+
}
|
|
3745
4776
|
function findRefsToWithStatement(stmt, symbolId) {
|
|
3746
4777
|
return stmt(
|
|
3747
4778
|
"SELECT id, from_id, to_name, to_id, call_type, line FROM refs WHERE to_id = ? OR to_name = (SELECT name FROM symbols WHERE id = ?)"
|
|
@@ -3985,6 +5016,12 @@ var CORE_TABLES_SQL = `
|
|
|
3985
5016
|
file TEXT PRIMARY KEY,
|
|
3986
5017
|
lang TEXT NOT NULL,
|
|
3987
5018
|
mtime_ms INTEGER NOT NULL,
|
|
5019
|
+
-- Phase 2: xxHash64 of the file's UTF-8 bytes. Empty string when the
|
|
5020
|
+
-- indexer hasn't populated it yet (legacy rows, schema repaired by
|
|
5021
|
+
-- repairMissingColumns). Compared on incremental re-index so that a
|
|
5022
|
+
-- touch or branch-switch that leaves content byte-identical skips the
|
|
5023
|
+
-- expensive parse phase entirely (refactoring proposal Phase 2).
|
|
5024
|
+
content_hash TEXT NOT NULL DEFAULT '',
|
|
3988
5025
|
symbol_count INTEGER NOT NULL DEFAULT 0,
|
|
3989
5026
|
last_indexed INTEGER NOT NULL,
|
|
3990
5027
|
-- Code Atlas grouping label, computed at index time from the ecosystem's
|
|
@@ -4052,7 +5089,14 @@ var LANG_FAMILY_TABLE_SQL = `
|
|
|
4052
5089
|
);
|
|
4053
5090
|
`;
|
|
4054
5091
|
var LANG_FAMILY_WILDCARD = "*";
|
|
4055
|
-
var SYMBOLS_FTS_SQL = "CREATE VIRTUAL TABLE IF NOT EXISTS symbols_fts USING fts5(text, tokenize = '
|
|
5092
|
+
var SYMBOLS_FTS_SQL = "CREATE VIRTUAL TABLE IF NOT EXISTS symbols_fts USING fts5(text, tokenize = 'trigram')";
|
|
5093
|
+
var SYMBOL_VECTORS_TABLE_SQL = `
|
|
5094
|
+
CREATE TABLE IF NOT EXISTS symbol_vectors (
|
|
5095
|
+
symbol_id INTEGER PRIMARY KEY,
|
|
5096
|
+
vector BLOB NOT NULL,
|
|
5097
|
+
FOREIGN KEY (symbol_id) REFERENCES symbols(id) ON DELETE CASCADE
|
|
5098
|
+
);
|
|
5099
|
+
`;
|
|
4056
5100
|
|
|
4057
5101
|
// src/codebase-index/writer-search-helpers.ts
|
|
4058
5102
|
var SEARCH_CANDIDATE_SCAN_CAP = 5e3;
|
|
@@ -4193,6 +5237,85 @@ var StorePool = class {
|
|
|
4193
5237
|
}
|
|
4194
5238
|
};
|
|
4195
5239
|
|
|
5240
|
+
// src/codebase-index/vector-search.ts
|
|
5241
|
+
var RRF_K = 60;
|
|
5242
|
+
var VECTOR_DIMENSIONS = 384;
|
|
5243
|
+
var NGRAM_SIZE = 3;
|
|
5244
|
+
function embedText(text) {
|
|
5245
|
+
const vec = new Float32Array(VECTOR_DIMENSIONS);
|
|
5246
|
+
const normalized = text.toLowerCase().trim();
|
|
5247
|
+
if (normalized.length < NGRAM_SIZE) {
|
|
5248
|
+
const padded = ` ${normalized} `.slice(0, Math.max(NGRAM_SIZE, normalized.length + 2));
|
|
5249
|
+
for (let i = 0; i <= padded.length - NGRAM_SIZE; i++) {
|
|
5250
|
+
const ngram = padded.slice(i, i + NGRAM_SIZE);
|
|
5251
|
+
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
5252
|
+
vec[bucket] += 1;
|
|
5253
|
+
}
|
|
5254
|
+
} else {
|
|
5255
|
+
for (let i = 0; i <= normalized.length - NGRAM_SIZE; i++) {
|
|
5256
|
+
const ngram = normalized.slice(i, i + NGRAM_SIZE);
|
|
5257
|
+
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
5258
|
+
vec[bucket] += 1;
|
|
5259
|
+
}
|
|
5260
|
+
}
|
|
5261
|
+
let norm = 0;
|
|
5262
|
+
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
5263
|
+
norm += vec[i] * vec[i];
|
|
5264
|
+
}
|
|
5265
|
+
norm = Math.sqrt(norm);
|
|
5266
|
+
if (norm > 0) {
|
|
5267
|
+
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
5268
|
+
vec[i] /= norm;
|
|
5269
|
+
}
|
|
5270
|
+
}
|
|
5271
|
+
return vec;
|
|
5272
|
+
}
|
|
5273
|
+
function hashNgram(str) {
|
|
5274
|
+
let hash = 2166136261;
|
|
5275
|
+
for (let i = 0; i < str.length; i++) {
|
|
5276
|
+
hash ^= str.charCodeAt(i);
|
|
5277
|
+
hash = Math.imul(hash, 16777619);
|
|
5278
|
+
}
|
|
5279
|
+
return hash >>> 0;
|
|
5280
|
+
}
|
|
5281
|
+
function cosineSimilarity(a, b) {
|
|
5282
|
+
let dot = 0;
|
|
5283
|
+
const len = Math.min(a.length, b.length);
|
|
5284
|
+
for (let i = 0; i < len; i++) {
|
|
5285
|
+
dot += a[i] * b[i];
|
|
5286
|
+
}
|
|
5287
|
+
return dot;
|
|
5288
|
+
}
|
|
5289
|
+
function encodeVector(vec) {
|
|
5290
|
+
return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
|
|
5291
|
+
}
|
|
5292
|
+
function decodeVector(buf) {
|
|
5293
|
+
const view = new DataView(
|
|
5294
|
+
buf.buffer,
|
|
5295
|
+
buf.byteOffset,
|
|
5296
|
+
buf.byteLength
|
|
5297
|
+
);
|
|
5298
|
+
const copy = new Float32Array(buf.byteLength / 4);
|
|
5299
|
+
for (let i = 0; i < copy.length; i++) {
|
|
5300
|
+
copy[i] = view.getFloat32(i * 4, true);
|
|
5301
|
+
}
|
|
5302
|
+
return copy;
|
|
5303
|
+
}
|
|
5304
|
+
function reciprocalRankFusion(bm25Ranks, vectorRanks, k = RRF_K) {
|
|
5305
|
+
const allIds = /* @__PURE__ */ new Set([...bm25Ranks.keys(), ...vectorRanks.keys()]);
|
|
5306
|
+
const scored = [];
|
|
5307
|
+
for (const id of allIds) {
|
|
5308
|
+
const bm25Rank = bm25Ranks.get(id);
|
|
5309
|
+
const vecRank = vectorRanks.get(id);
|
|
5310
|
+
let score = 0;
|
|
5311
|
+
if (bm25Rank !== void 0) score += 1 / (k + bm25Rank);
|
|
5312
|
+
if (vecRank !== void 0) score += 1 / (k + vecRank);
|
|
5313
|
+
scored.push([id, score]);
|
|
5314
|
+
}
|
|
5315
|
+
scored.sort((a, b) => b[1] - a[1]);
|
|
5316
|
+
return scored;
|
|
5317
|
+
}
|
|
5318
|
+
|
|
4196
5319
|
// src/codebase-index/writer.ts
|
|
4197
5320
|
var DB_FILE2 = "index.db";
|
|
4198
5321
|
var MAX_STATEMENT_CACHE = 128;
|
|
@@ -4205,6 +5328,12 @@ var IndexStore = class _IndexStore {
|
|
|
4205
5328
|
* When false, ranked search falls back to the LIKE + in-process BM25 path.
|
|
4206
5329
|
*/
|
|
4207
5330
|
ftsAvailable = false;
|
|
5331
|
+
/**
|
|
5332
|
+
* Phase 3: true when the `symbol_vectors` table was created successfully.
|
|
5333
|
+
* When false, hybrid search skips the vector pass and falls back to FTS5
|
|
5334
|
+
* (or LIKE) only.
|
|
5335
|
+
*/
|
|
5336
|
+
vectorsAvailable = false;
|
|
4208
5337
|
/**
|
|
4209
5338
|
* Cache of prepared statements keyed by their SQL text. `DatabaseSync`
|
|
4210
5339
|
* compiles SQL on every `.prepare()` call; for the fixed-SQL methods
|
|
@@ -4263,9 +5392,9 @@ var IndexStore = class _IndexStore {
|
|
|
4263
5392
|
}
|
|
4264
5393
|
constructor(projectRoot, opts = {}) {
|
|
4265
5394
|
this.indexDir = resolveIndexDir(projectRoot, opts.indexDir);
|
|
4266
|
-
|
|
5395
|
+
fs8.mkdirSync(this.indexDir, { recursive: true });
|
|
4267
5396
|
const Database = loadDatabaseSync();
|
|
4268
|
-
this.db = new Database(
|
|
5397
|
+
this.db = new Database(path12.join(this.indexDir, DB_FILE2));
|
|
4269
5398
|
applyIndexStorePragmas(this.db);
|
|
4270
5399
|
this.initSchema();
|
|
4271
5400
|
}
|
|
@@ -4303,7 +5432,13 @@ var IndexStore = class _IndexStore {
|
|
|
4303
5432
|
*/
|
|
4304
5433
|
repairMissingColumns() {
|
|
4305
5434
|
const expected = [
|
|
4306
|
-
{
|
|
5435
|
+
{
|
|
5436
|
+
table: "files",
|
|
5437
|
+
columns: [
|
|
5438
|
+
["package", "TEXT NOT NULL DEFAULT ''"],
|
|
5439
|
+
["content_hash", "TEXT NOT NULL DEFAULT ''"]
|
|
5440
|
+
]
|
|
5441
|
+
},
|
|
4307
5442
|
{
|
|
4308
5443
|
table: "refs",
|
|
4309
5444
|
columns: [
|
|
@@ -4335,6 +5470,7 @@ var IndexStore = class _IndexStore {
|
|
|
4335
5470
|
DROP TABLE IF EXISTS symbols;
|
|
4336
5471
|
DROP TABLE IF EXISTS files;
|
|
4337
5472
|
DROP TABLE IF EXISTS refs;
|
|
5473
|
+
DROP TABLE IF EXISTS symbol_vectors;
|
|
4338
5474
|
`);
|
|
4339
5475
|
this.db.exec("DROP TABLE IF EXISTS symbols_fts");
|
|
4340
5476
|
this.stmt("UPDATE metadata SET value = ? WHERE key = ?").run(
|
|
@@ -4356,6 +5492,12 @@ var IndexStore = class _IndexStore {
|
|
|
4356
5492
|
this.db.exec(LANG_FAMILY_TABLE_SQL);
|
|
4357
5493
|
this.seedLangFamilies();
|
|
4358
5494
|
try {
|
|
5495
|
+
const ftsSchema = this.stmt(
|
|
5496
|
+
"SELECT sql FROM sqlite_master WHERE type='table' AND name='symbols_fts'"
|
|
5497
|
+
).get();
|
|
5498
|
+
if (ftsSchema?.sql && ftsSchema.sql.includes("unicode61")) {
|
|
5499
|
+
this.db.exec("DROP TABLE IF EXISTS symbols_fts");
|
|
5500
|
+
}
|
|
4359
5501
|
this.db.exec(SYMBOLS_FTS_SQL);
|
|
4360
5502
|
this.ftsAvailable = true;
|
|
4361
5503
|
const symbolCount = Number(
|
|
@@ -4366,6 +5508,7 @@ var IndexStore = class _IndexStore {
|
|
|
4366
5508
|
);
|
|
4367
5509
|
if (symbolCount !== ftsCount) {
|
|
4368
5510
|
this.db.exec("DELETE FROM symbols_fts");
|
|
5511
|
+
if (this.vectorsAvailable) this.db.exec("DELETE FROM symbol_vectors");
|
|
4369
5512
|
const rows = this.stmt(
|
|
4370
5513
|
"SELECT id, name, signature, doc_comment FROM symbols ORDER BY id"
|
|
4371
5514
|
).all();
|
|
@@ -4383,6 +5526,12 @@ var IndexStore = class _IndexStore {
|
|
|
4383
5526
|
} catch {
|
|
4384
5527
|
this.ftsAvailable = false;
|
|
4385
5528
|
}
|
|
5529
|
+
try {
|
|
5530
|
+
this.db.exec(SYMBOL_VECTORS_TABLE_SQL);
|
|
5531
|
+
this.vectorsAvailable = true;
|
|
5532
|
+
} catch {
|
|
5533
|
+
this.vectorsAvailable = false;
|
|
5534
|
+
}
|
|
4386
5535
|
this.ensureNextSymbolIdSeeded();
|
|
4387
5536
|
}
|
|
4388
5537
|
// ─── ID allocation & bulk helpers ────────────────────────────────────────────
|
|
@@ -4492,6 +5641,7 @@ var IndexStore = class _IndexStore {
|
|
|
4492
5641
|
const result = [];
|
|
4493
5642
|
const bulk = [];
|
|
4494
5643
|
const ftsRows = [];
|
|
5644
|
+
const vectorRows = [];
|
|
4495
5645
|
for (const s of symbols) {
|
|
4496
5646
|
const id = nextId++;
|
|
4497
5647
|
bulk.push({
|
|
@@ -4510,6 +5660,10 @@ var IndexStore = class _IndexStore {
|
|
|
4510
5660
|
if (this.ftsAvailable) {
|
|
4511
5661
|
ftsRows.push({ id, text: buildIndexableText(s.name, s.signature, s.docComment) });
|
|
4512
5662
|
}
|
|
5663
|
+
vectorRows.push({
|
|
5664
|
+
id,
|
|
5665
|
+
vector: encodeVector(embedText(s.text || buildIndexableText(s.name, s.signature, s.docComment)))
|
|
5666
|
+
});
|
|
4513
5667
|
result.push({ ...s, id });
|
|
4514
5668
|
}
|
|
4515
5669
|
bulkInsertSymbolsWithStatement((sql) => this.stmt(sql), _IndexStore.MAX_SQL_VARS, bulk);
|
|
@@ -4519,6 +5673,13 @@ var IndexStore = class _IndexStore {
|
|
|
4519
5673
|
this.ftsAvailable,
|
|
4520
5674
|
ftsRows
|
|
4521
5675
|
);
|
|
5676
|
+
if (this.vectorsAvailable) {
|
|
5677
|
+
bulkInsertVectorsWithStatement(
|
|
5678
|
+
(sql) => this.stmt(sql),
|
|
5679
|
+
_IndexStore.MAX_SQL_VARS,
|
|
5680
|
+
vectorRows
|
|
5681
|
+
);
|
|
5682
|
+
}
|
|
4522
5683
|
this.db.exec("COMMIT");
|
|
4523
5684
|
return result;
|
|
4524
5685
|
} catch (err) {
|
|
@@ -4538,6 +5699,11 @@ var IndexStore = class _IndexStore {
|
|
|
4538
5699
|
"DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
4539
5700
|
).run(file);
|
|
4540
5701
|
}
|
|
5702
|
+
if (this.vectorsAvailable) {
|
|
5703
|
+
this.stmt(
|
|
5704
|
+
"DELETE FROM symbol_vectors WHERE symbol_id IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
5705
|
+
).run(file);
|
|
5706
|
+
}
|
|
4541
5707
|
this.stmt("DELETE FROM symbols WHERE file_fk = ?").run(file);
|
|
4542
5708
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
4543
5709
|
this.db.exec("COMMIT");
|
|
@@ -4563,6 +5729,11 @@ var IndexStore = class _IndexStore {
|
|
|
4563
5729
|
"DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
4564
5730
|
).run(file);
|
|
4565
5731
|
}
|
|
5732
|
+
if (this.vectorsAvailable) {
|
|
5733
|
+
this.stmt(
|
|
5734
|
+
"DELETE FROM symbol_vectors WHERE symbol_id IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
5735
|
+
).run(file);
|
|
5736
|
+
}
|
|
4566
5737
|
this.stmt(
|
|
4567
5738
|
"DELETE FROM refs WHERE from_id IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
4568
5739
|
).run(file);
|
|
@@ -4580,14 +5751,22 @@ var IndexStore = class _IndexStore {
|
|
|
4580
5751
|
upsertFile(meta) {
|
|
4581
5752
|
this.runWithRetry(() => {
|
|
4582
5753
|
this.stmt(
|
|
4583
|
-
`INSERT INTO files(file, lang, mtime_ms, symbol_count, last_indexed)
|
|
4584
|
-
VALUES (?, ?, ?, ?, ?)
|
|
5754
|
+
`INSERT INTO files(file, lang, mtime_ms, content_hash, symbol_count, last_indexed)
|
|
5755
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
4585
5756
|
ON CONFLICT(file) DO UPDATE SET
|
|
4586
5757
|
lang = excluded.lang,
|
|
4587
5758
|
mtime_ms = excluded.mtime_ms,
|
|
5759
|
+
content_hash = excluded.content_hash,
|
|
4588
5760
|
symbol_count = excluded.symbol_count,
|
|
4589
5761
|
last_indexed = excluded.last_indexed`
|
|
4590
|
-
).run(
|
|
5762
|
+
).run(
|
|
5763
|
+
meta.file,
|
|
5764
|
+
meta.lang,
|
|
5765
|
+
meta.mtimeMs,
|
|
5766
|
+
meta.contentHash ?? "",
|
|
5767
|
+
meta.symbolCount,
|
|
5768
|
+
meta.lastIndexed
|
|
5769
|
+
);
|
|
4591
5770
|
});
|
|
4592
5771
|
}
|
|
4593
5772
|
getFileMeta(file) {
|
|
@@ -4754,9 +5933,18 @@ var IndexStore = class _IndexStore {
|
|
|
4754
5933
|
if (mapped === null) return { results: [], total: 0 };
|
|
4755
5934
|
effectiveKind = mapped;
|
|
4756
5935
|
}
|
|
4757
|
-
const
|
|
5936
|
+
const longTokens = tokens.filter((t) => t.length >= 3);
|
|
5937
|
+
const shortTokens = tokens.filter((t) => t.length < 3);
|
|
5938
|
+
if (longTokens.length === 0) {
|
|
5939
|
+
return this.searchRankedFallback(query, filter, safeLimit);
|
|
5940
|
+
}
|
|
5941
|
+
const match = longTokens.map((t) => `"${t.replaceAll('"', "")}"`).join(" OR ");
|
|
4758
5942
|
const conditions = ["symbols_fts MATCH ?"];
|
|
4759
5943
|
const values = [match];
|
|
5944
|
+
for (const shortTok of shortTokens) {
|
|
5945
|
+
conditions.push("s.text LIKE ? ESCAPE '\\'");
|
|
5946
|
+
values.push(`%${escapeLike(shortTok)}%`);
|
|
5947
|
+
}
|
|
4760
5948
|
if (effectiveKind) {
|
|
4761
5949
|
conditions.push("s.kind = ?");
|
|
4762
5950
|
values.push(effectiveKind);
|
|
@@ -4775,7 +5963,7 @@ var IndexStore = class _IndexStore {
|
|
|
4775
5963
|
).all(...values);
|
|
4776
5964
|
const total = countRows[0] ? Number(countRows[0].n) : 0;
|
|
4777
5965
|
if (total === 0) return { results: [], total: 0 };
|
|
4778
|
-
const
|
|
5966
|
+
const bm25Rows = this.stmt(
|
|
4779
5967
|
`SELECT s.id, s.lang, s.kind, s.name, s.file, s.line, s.col, s.signature, s.doc_comment,
|
|
4780
5968
|
-bm25(symbols_fts) AS score,
|
|
4781
5969
|
snippet(symbols_fts, 0, '', '', '\u2026', 12) AS snippet
|
|
@@ -4788,8 +5976,35 @@ var IndexStore = class _IndexStore {
|
|
|
4788
5976
|
bm25(symbols_fts), lower(s.name), s.file, s.line, s.col, s.id
|
|
4789
5977
|
LIMIT ?`
|
|
4790
5978
|
).all(...values, query.trim(), `${escapeLike(query.trim())}%`, safeLimit);
|
|
5979
|
+
if (this.vectorsAvailable && bm25Rows.length > 0) {
|
|
5980
|
+
const queryVec = embedText(query);
|
|
5981
|
+
const candidateIds = bm25Rows.map((r) => r.id);
|
|
5982
|
+
const placeholders = candidateIds.map(() => "?").join(",");
|
|
5983
|
+
const vecRows = this.stmt(
|
|
5984
|
+
`SELECT sv.symbol_id, sv.vector FROM symbol_vectors sv WHERE sv.symbol_id IN (${placeholders})`
|
|
5985
|
+
).all(...candidateIds);
|
|
5986
|
+
const vecScores = vecRows.map((r) => ({
|
|
5987
|
+
id: r.symbol_id,
|
|
5988
|
+
sim: cosineSimilarity(queryVec, decodeVector(r.vector))
|
|
5989
|
+
})).sort((a, b) => b.sim - a.sim);
|
|
5990
|
+
const bm25Rank = /* @__PURE__ */ new Map();
|
|
5991
|
+
bm25Rows.forEach((r, i) => bm25Rank.set(r.id, i));
|
|
5992
|
+
const vecRank = /* @__PURE__ */ new Map();
|
|
5993
|
+
vecScores.forEach((r, i) => vecRank.set(r.id, i));
|
|
5994
|
+
const fused = reciprocalRankFusion(bm25Rank, vecRank, 60);
|
|
5995
|
+
const fusedScore = new Map(fused);
|
|
5996
|
+
const sorted = [...bm25Rows].sort(
|
|
5997
|
+
(a, b) => (fusedScore.get(b.id) ?? 0) - (fusedScore.get(a.id) ?? 0)
|
|
5998
|
+
);
|
|
5999
|
+
return {
|
|
6000
|
+
results: sorted.map(
|
|
6001
|
+
(row) => mapWriterSearchRow(row, filter?.lspKind, Math.max(1e-4, row.score), row.snippet)
|
|
6002
|
+
),
|
|
6003
|
+
total
|
|
6004
|
+
};
|
|
6005
|
+
}
|
|
4791
6006
|
return {
|
|
4792
|
-
results:
|
|
6007
|
+
results: bm25Rows.map(
|
|
4793
6008
|
(row) => mapWriterSearchRow(row, filter?.lspKind, Math.max(1e-4, row.score), row.snippet)
|
|
4794
6009
|
),
|
|
4795
6010
|
total
|
|
@@ -4917,6 +6132,7 @@ var IndexStore = class _IndexStore {
|
|
|
4917
6132
|
this.db.exec("DROP TABLE IF EXISTS files");
|
|
4918
6133
|
this.db.exec("DROP TABLE IF EXISTS metadata");
|
|
4919
6134
|
if (this.ftsAvailable) this.db.exec("DROP TABLE IF EXISTS symbols_fts");
|
|
6135
|
+
this.db.exec("DROP TABLE IF EXISTS symbol_vectors");
|
|
4920
6136
|
this.db.exec("COMMIT");
|
|
4921
6137
|
this.stmtCache.clear();
|
|
4922
6138
|
this.initSchema();
|
|
@@ -5004,6 +6220,11 @@ var IndexStore = class _IndexStore {
|
|
|
5004
6220
|
`DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file IN (${placeholders}))`
|
|
5005
6221
|
).run(...options.deleteForFiles);
|
|
5006
6222
|
}
|
|
6223
|
+
if (this.vectorsAvailable) {
|
|
6224
|
+
this.stmt(
|
|
6225
|
+
`DELETE FROM symbol_vectors WHERE symbol_id IN (SELECT id FROM symbols WHERE file IN (${placeholders}))`
|
|
6226
|
+
).run(...options.deleteForFiles);
|
|
6227
|
+
}
|
|
5007
6228
|
this.stmt(
|
|
5008
6229
|
`DELETE FROM refs WHERE from_id IN (SELECT id FROM symbols WHERE file IN (${placeholders}))`
|
|
5009
6230
|
).run(...options.deleteForFiles);
|
|
@@ -5017,6 +6238,7 @@ var IndexStore = class _IndexStore {
|
|
|
5017
6238
|
const refsToInsert = [];
|
|
5018
6239
|
const bulkSyms = [];
|
|
5019
6240
|
const ftsRows = [];
|
|
6241
|
+
const vectorRows = [];
|
|
5020
6242
|
for (const entry of entries) {
|
|
5021
6243
|
const insertedForEntry = [];
|
|
5022
6244
|
for (const s of entry.symbols) {
|
|
@@ -5040,6 +6262,10 @@ var IndexStore = class _IndexStore {
|
|
|
5040
6262
|
text: buildIndexableText(s.name, s.signature, s.docComment)
|
|
5041
6263
|
});
|
|
5042
6264
|
}
|
|
6265
|
+
vectorRows.push({
|
|
6266
|
+
id,
|
|
6267
|
+
vector: encodeVector(embedText(s.text || buildIndexableText(s.name, s.signature, s.docComment)))
|
|
6268
|
+
});
|
|
5043
6269
|
const inserted = { ...s, id };
|
|
5044
6270
|
allInserted.push(inserted);
|
|
5045
6271
|
insertedForEntry.push(inserted);
|
|
@@ -5053,19 +6279,34 @@ var IndexStore = class _IndexStore {
|
|
|
5053
6279
|
this.ftsAvailable,
|
|
5054
6280
|
ftsRows
|
|
5055
6281
|
);
|
|
6282
|
+
if (this.vectorsAvailable) {
|
|
6283
|
+
bulkInsertVectorsWithStatement(
|
|
6284
|
+
(sql) => this.stmt(sql),
|
|
6285
|
+
_IndexStore.MAX_SQL_VARS,
|
|
6286
|
+
vectorRows
|
|
6287
|
+
);
|
|
6288
|
+
}
|
|
5056
6289
|
bulkInsertRefsWithStatement((sql) => this.stmt(sql), _IndexStore.MAX_SQL_VARS, refsToInsert);
|
|
5057
6290
|
const upsertStmt = this.stmt(
|
|
5058
|
-
`INSERT INTO files(file, lang, mtime_ms, symbol_count, last_indexed)
|
|
5059
|
-
VALUES (?, ?, ?, ?, ?)
|
|
6291
|
+
`INSERT INTO files(file, lang, mtime_ms, content_hash, symbol_count, last_indexed)
|
|
6292
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
5060
6293
|
ON CONFLICT(file) DO UPDATE SET
|
|
5061
6294
|
lang = excluded.lang,
|
|
5062
6295
|
mtime_ms = excluded.mtime_ms,
|
|
6296
|
+
content_hash = excluded.content_hash,
|
|
5063
6297
|
symbol_count = excluded.symbol_count,
|
|
5064
6298
|
last_indexed = excluded.last_indexed`
|
|
5065
6299
|
);
|
|
5066
6300
|
const now = Date.now();
|
|
5067
6301
|
for (const entry of entries) {
|
|
5068
|
-
upsertStmt.run(
|
|
6302
|
+
upsertStmt.run(
|
|
6303
|
+
entry.file,
|
|
6304
|
+
entry.lang,
|
|
6305
|
+
entry.mtimeMs,
|
|
6306
|
+
entry.contentHash ?? "",
|
|
6307
|
+
entry.symbolCount,
|
|
6308
|
+
now
|
|
6309
|
+
);
|
|
5069
6310
|
}
|
|
5070
6311
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
5071
6312
|
this.db.exec("COMMIT");
|
|
@@ -5156,19 +6397,32 @@ var IndexStore = class _IndexStore {
|
|
|
5156
6397
|
"DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
5157
6398
|
).run(meta.file);
|
|
5158
6399
|
}
|
|
6400
|
+
if (this.vectorsAvailable) {
|
|
6401
|
+
this.stmt(
|
|
6402
|
+
"DELETE FROM symbol_vectors WHERE symbol_id IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
6403
|
+
).run(meta.file);
|
|
6404
|
+
}
|
|
5159
6405
|
this.stmt(
|
|
5160
6406
|
"DELETE FROM refs WHERE from_id IN (SELECT id FROM symbols WHERE file_fk = ?)"
|
|
5161
6407
|
).run(meta.file);
|
|
5162
6408
|
this.stmt("DELETE FROM symbols WHERE file_fk = ?").run(meta.file);
|
|
5163
6409
|
this.stmt(
|
|
5164
|
-
`INSERT INTO files(file, lang, mtime_ms, symbol_count, last_indexed)
|
|
5165
|
-
VALUES (?, ?, ?, ?, ?)
|
|
6410
|
+
`INSERT INTO files(file, lang, mtime_ms, content_hash, symbol_count, last_indexed)
|
|
6411
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
5166
6412
|
ON CONFLICT(file) DO UPDATE SET
|
|
5167
6413
|
lang = excluded.lang,
|
|
5168
6414
|
mtime_ms = excluded.mtime_ms,
|
|
6415
|
+
content_hash = excluded.content_hash,
|
|
5169
6416
|
symbol_count = excluded.symbol_count,
|
|
5170
6417
|
last_indexed = excluded.last_indexed`
|
|
5171
|
-
).run(
|
|
6418
|
+
).run(
|
|
6419
|
+
meta.file,
|
|
6420
|
+
meta.lang,
|
|
6421
|
+
meta.mtimeMs,
|
|
6422
|
+
meta.contentHash ?? "",
|
|
6423
|
+
meta.symbolCount,
|
|
6424
|
+
meta.lastIndexed
|
|
6425
|
+
);
|
|
5172
6426
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
5173
6427
|
this.db.exec("COMMIT");
|
|
5174
6428
|
} catch (err) {
|
|
@@ -5231,6 +6485,31 @@ var IndexStore = class _IndexStore {
|
|
|
5231
6485
|
findOutgoingCallsByName(symbolName, file, limit = 100) {
|
|
5232
6486
|
return findOutgoingCallsByName((sql) => this.stmt(sql), symbolName, file, limit);
|
|
5233
6487
|
}
|
|
6488
|
+
/**
|
|
6489
|
+
* Transitive incoming-call tree: all symbols that transitively call the
|
|
6490
|
+
* target, to an unbounded depth (cycle-safe via SQL UNION deduplication).
|
|
6491
|
+
* Used by `codebase-incoming-calls` when the caller wants the full call
|
|
6492
|
+
* chain rather than just direct callers.
|
|
6493
|
+
*/
|
|
6494
|
+
findTransitiveIncomingCallsByName(symbolName, file, limit = 200) {
|
|
6495
|
+
return findTransitiveIncomingCallsByName((sql) => this.stmt(sql), symbolName, file, limit);
|
|
6496
|
+
}
|
|
6497
|
+
/**
|
|
6498
|
+
* Transitive outgoing-call tree: all symbols the target transitively calls.
|
|
6499
|
+
* Used by `codebase-outgoing-calls` when the caller wants the full
|
|
6500
|
+
* dependency chain rather than just direct callees.
|
|
6501
|
+
*/
|
|
6502
|
+
findTransitiveOutgoingCallsByName(symbolName, file, limit = 200) {
|
|
6503
|
+
return findTransitiveOutgoingCallsByName((sql) => this.stmt(sql), symbolName, file, limit);
|
|
6504
|
+
}
|
|
6505
|
+
/**
|
|
6506
|
+
* Compute the set of symbol IDs reachable from the given seed IDs using a
|
|
6507
|
+
* native SQLite recursive CTE. Used by dead-code detection to replace the
|
|
6508
|
+
* in-memory BFS.
|
|
6509
|
+
*/
|
|
6510
|
+
findReachableSymbolIds(seedIds) {
|
|
6511
|
+
return findReachableSymbolIds((sql) => this.stmt(sql), seedIds);
|
|
6512
|
+
}
|
|
5234
6513
|
/**
|
|
5235
6514
|
* Find all references TO a given symbol (who calls / uses this symbol?).
|
|
5236
6515
|
*/
|
|
@@ -5338,15 +6617,15 @@ var INDEXABLE_EXTENSION_SET = new Set(INDEXABLE_EXTENSIONS);
|
|
|
5338
6617
|
var MAX_INDEX_FILE_BYTES = 5 * 1024 * 1024;
|
|
5339
6618
|
var MAX_GIT_FILE_LIST_BYTES = 64 * 1024 * 1024;
|
|
5340
6619
|
function isWithinProject(projectRoot, file) {
|
|
5341
|
-
const rel =
|
|
5342
|
-
return rel !== "" && !rel.startsWith(`..${
|
|
6620
|
+
const rel = path13.relative(projectRoot, file);
|
|
6621
|
+
return rel !== "" && !rel.startsWith(`..${path13.sep}`) && rel !== ".." && !path13.isAbsolute(rel);
|
|
5343
6622
|
}
|
|
5344
6623
|
function isMissingPathError(err) {
|
|
5345
6624
|
const code = err?.code;
|
|
5346
6625
|
return code === "ENOENT" || code === "ENOTDIR";
|
|
5347
6626
|
}
|
|
5348
6627
|
function normalizeComparablePath(value) {
|
|
5349
|
-
const resolved =
|
|
6628
|
+
const resolved = path13.resolve(value);
|
|
5350
6629
|
return process.platform === "win32" ? resolved.toLowerCase() : resolved;
|
|
5351
6630
|
}
|
|
5352
6631
|
function gitOutput(projectRoot, args) {
|
|
@@ -5391,24 +6670,24 @@ async function findGitSourceFiles(projectRoot, ignore, signal) {
|
|
|
5391
6670
|
const record = statusRecords[i];
|
|
5392
6671
|
if (!record) continue;
|
|
5393
6672
|
const status = record.slice(0, 2);
|
|
5394
|
-
const changedPath =
|
|
6673
|
+
const changedPath = path13.resolve(projectRoot, record.slice(3));
|
|
5395
6674
|
dirty.add(changedPath);
|
|
5396
6675
|
if (status.includes("D")) deleted.add(changedPath);
|
|
5397
6676
|
if (status.includes("R") || status.includes("C")) {
|
|
5398
6677
|
const source = statusRecords[++i];
|
|
5399
|
-
if (source) dirty.add(
|
|
6678
|
+
if (source) dirty.add(path13.resolve(projectRoot, source));
|
|
5400
6679
|
}
|
|
5401
6680
|
}
|
|
5402
6681
|
const files = [];
|
|
5403
6682
|
for (const relative3 of output.toString("utf8").split("\0")) {
|
|
5404
6683
|
if (!relative3) continue;
|
|
5405
6684
|
const portable = relative3.replace(/\\/g, "/");
|
|
5406
|
-
if (portable.split("/").some((segment) => ignoreSet.has(segment)) || DEFAULT_IGNORE_FILES.has(
|
|
6685
|
+
if (portable.split("/").some((segment) => ignoreSet.has(segment)) || DEFAULT_IGNORE_FILES.has(path13.posix.basename(portable))) {
|
|
5407
6686
|
continue;
|
|
5408
6687
|
}
|
|
5409
|
-
const full =
|
|
6688
|
+
const full = path13.resolve(projectRoot, relative3);
|
|
5410
6689
|
if (deleted.has(full)) continue;
|
|
5411
|
-
const ext =
|
|
6690
|
+
const ext = path13.extname(relative3).toLowerCase();
|
|
5412
6691
|
if (INDEXABLE_EXTENSION_SET.has(ext) || detectLang(full) !== null) files.push(full);
|
|
5413
6692
|
}
|
|
5414
6693
|
return {
|
|
@@ -5443,7 +6722,7 @@ async function findSourceFiles(projectRoot, ignore, isGitIgnored, signal) {
|
|
|
5443
6722
|
}
|
|
5444
6723
|
let entries;
|
|
5445
6724
|
try {
|
|
5446
|
-
entries = await
|
|
6725
|
+
entries = await fs9.readdir(dir, { withFileTypes: true });
|
|
5447
6726
|
} catch (err) {
|
|
5448
6727
|
complete = false;
|
|
5449
6728
|
errors.push(`scan error: ${dir}: ${err instanceof Error ? err.message : String(err)}`);
|
|
@@ -5452,14 +6731,14 @@ async function findSourceFiles(projectRoot, ignore, isGitIgnored, signal) {
|
|
|
5452
6731
|
dirCount++;
|
|
5453
6732
|
for (const e of entries) {
|
|
5454
6733
|
if (ignoreSet.has(e.name)) continue;
|
|
5455
|
-
const full =
|
|
5456
|
-
const rel =
|
|
6734
|
+
const full = path13.join(dir, e.name);
|
|
6735
|
+
const rel = path13.relative(projectRoot, full).replace(/\\/g, "/");
|
|
5457
6736
|
if (e.isDirectory()) {
|
|
5458
6737
|
if (isGitIgnored(rel, true)) continue;
|
|
5459
6738
|
await walk(full);
|
|
5460
6739
|
} else if (e.isFile()) {
|
|
5461
6740
|
if (DEFAULT_IGNORE_FILES.has(e.name) || isGitIgnored(rel, false)) continue;
|
|
5462
|
-
const ext =
|
|
6741
|
+
const ext = path13.extname(e.name).toLowerCase();
|
|
5463
6742
|
if (indexableExts.has(ext) || detectLang(full) !== null) {
|
|
5464
6743
|
results.push(full);
|
|
5465
6744
|
}
|
|
@@ -5533,10 +6812,10 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5533
6812
|
let discoveryComplete = true;
|
|
5534
6813
|
let trustedUnchanged;
|
|
5535
6814
|
if (opts.files && opts.files.length > 0) {
|
|
5536
|
-
files = opts.files.map((f) =>
|
|
6815
|
+
files = opts.files.map((f) => path13.resolve(projectRoot, f)).filter((f) => {
|
|
5537
6816
|
if (!isWithinProject(projectRoot, f)) return false;
|
|
5538
|
-
const rel =
|
|
5539
|
-
return !rel.split("/").some((seg) => DEFAULT_IGNORE.includes(seg)) && !DEFAULT_IGNORE_FILES.has(
|
|
6817
|
+
const rel = path13.relative(projectRoot, f).replace(/\\/g, "/");
|
|
6818
|
+
return !rel.split("/").some((seg) => DEFAULT_IGNORE.includes(seg)) && !DEFAULT_IGNORE_FILES.has(path13.basename(f)) && !isGitIgnored(rel, false);
|
|
5540
6819
|
});
|
|
5541
6820
|
} else {
|
|
5542
6821
|
const discovery = await findSourceFiles(projectRoot, ignore, isGitIgnored, signal);
|
|
@@ -5593,7 +6872,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5593
6872
|
async (file) => {
|
|
5594
6873
|
let stat3;
|
|
5595
6874
|
try {
|
|
5596
|
-
stat3 = await
|
|
6875
|
+
stat3 = await fs9.stat(file, statOpts);
|
|
5597
6876
|
} catch (e) {
|
|
5598
6877
|
if (isAbortError(e)) throw e;
|
|
5599
6878
|
return {
|
|
@@ -5623,7 +6902,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5623
6902
|
}
|
|
5624
6903
|
let content;
|
|
5625
6904
|
try {
|
|
5626
|
-
content = await
|
|
6905
|
+
content = await fs9.readFile(file, { encoding: "utf8", signal });
|
|
5627
6906
|
} catch (e) {
|
|
5628
6907
|
if (isAbortError(e)) throw e;
|
|
5629
6908
|
return {
|
|
@@ -5634,22 +6913,78 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5634
6913
|
error: `read error: ${e instanceof Error ? e.message : String(e)}`
|
|
5635
6914
|
};
|
|
5636
6915
|
}
|
|
5637
|
-
|
|
5638
|
-
|
|
5639
|
-
parsed = await parseFileContent(file, content, lang);
|
|
5640
|
-
} catch (e) {
|
|
6916
|
+
const contentHash = xxhash64String(content);
|
|
6917
|
+
if (!force && meta && meta.contentHash && contentHash === meta.contentHash) {
|
|
5641
6918
|
return {
|
|
5642
6919
|
file,
|
|
5643
6920
|
stat: stat3,
|
|
5644
6921
|
lang,
|
|
5645
6922
|
parsed: null,
|
|
5646
|
-
|
|
6923
|
+
content,
|
|
6924
|
+
contentHash,
|
|
6925
|
+
skippedMeta: { ...meta, mtimeMs: Math.floor(stat3.mtimeMs) }
|
|
5647
6926
|
};
|
|
5648
6927
|
}
|
|
5649
|
-
return { file, stat: stat3, lang, parsed, content };
|
|
6928
|
+
return { file, stat: stat3, lang, parsed: null, content, contentHash };
|
|
5650
6929
|
}
|
|
5651
6930
|
)
|
|
5652
6931
|
);
|
|
6932
|
+
const toParse = [];
|
|
6933
|
+
for (let pi = 0; pi < statReadParse.length; pi++) {
|
|
6934
|
+
const s = statReadParse[pi];
|
|
6935
|
+
if (s.status !== "fulfilled") continue;
|
|
6936
|
+
const r = s.value;
|
|
6937
|
+
if (r.error || r.skippedMeta || !r.lang || r.parsed) continue;
|
|
6938
|
+
if (r.content === void 0) continue;
|
|
6939
|
+
toParse.push({
|
|
6940
|
+
index: pi,
|
|
6941
|
+
file: batchFiles[pi],
|
|
6942
|
+
content: r.content,
|
|
6943
|
+
lang: r.lang
|
|
6944
|
+
});
|
|
6945
|
+
}
|
|
6946
|
+
if (toParse.length > 0) {
|
|
6947
|
+
let pool = toParse.length >= WORKER_POOL_THRESHOLD ? getParserPool() : null;
|
|
6948
|
+
if (pool) {
|
|
6949
|
+
try {
|
|
6950
|
+
await pool.ensureReady();
|
|
6951
|
+
const parsedResults = await pool.parseFiles(
|
|
6952
|
+
toParse.map((p) => ({ file: p.file, content: p.content, lang: p.lang }))
|
|
6953
|
+
);
|
|
6954
|
+
const byFile = new Map(parsedResults.map((r) => [r.file, r]));
|
|
6955
|
+
for (const item of toParse) {
|
|
6956
|
+
const parsed = byFile.get(item.file);
|
|
6957
|
+
const settled = statReadParse[item.index];
|
|
6958
|
+
if (settled.status !== "fulfilled") continue;
|
|
6959
|
+
if (parsed) {
|
|
6960
|
+
settled.value.parsed = parsed;
|
|
6961
|
+
} else {
|
|
6962
|
+
settled.value.error = `parse error: worker returned no result for ${item.file}`;
|
|
6963
|
+
}
|
|
6964
|
+
}
|
|
6965
|
+
} catch {
|
|
6966
|
+
pool = null;
|
|
6967
|
+
}
|
|
6968
|
+
}
|
|
6969
|
+
if (!pool) {
|
|
6970
|
+
await Promise.all(
|
|
6971
|
+
toParse.map(async (item) => {
|
|
6972
|
+
try {
|
|
6973
|
+
const parsed = await parseFileContent(item.file, item.content, item.lang);
|
|
6974
|
+
const settled = statReadParse[item.index];
|
|
6975
|
+
if (settled.status === "fulfilled") {
|
|
6976
|
+
settled.value.parsed = parsed;
|
|
6977
|
+
}
|
|
6978
|
+
} catch (e) {
|
|
6979
|
+
const settled = statReadParse[item.index];
|
|
6980
|
+
if (settled.status === "fulfilled") {
|
|
6981
|
+
settled.value.error = `parse error: ${e instanceof Error ? e.message : String(e)}`;
|
|
6982
|
+
}
|
|
6983
|
+
}
|
|
6984
|
+
})
|
|
6985
|
+
);
|
|
6986
|
+
}
|
|
6987
|
+
}
|
|
5653
6988
|
const batchEntries = [];
|
|
5654
6989
|
const deleteForFiles = [];
|
|
5655
6990
|
for (let fi = 0; fi < statReadParse.length; fi++) {
|
|
@@ -5672,6 +7007,17 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5672
7007
|
langStats[lang] = (langStats[lang] ?? 0) + result.skippedMeta.symbolCount;
|
|
5673
7008
|
symbolsIndexed += result.skippedMeta.symbolCount;
|
|
5674
7009
|
filesIndexed++;
|
|
7010
|
+
const stored = existingMeta.get(file);
|
|
7011
|
+
if (stored && stored.mtimeMs !== result.skippedMeta.mtimeMs) {
|
|
7012
|
+
store.upsertFile({
|
|
7013
|
+
file,
|
|
7014
|
+
lang,
|
|
7015
|
+
mtimeMs: result.skippedMeta.mtimeMs,
|
|
7016
|
+
symbolCount: result.skippedMeta.symbolCount,
|
|
7017
|
+
lastIndexed: Date.now(),
|
|
7018
|
+
contentHash: result.skippedMeta.contentHash
|
|
7019
|
+
});
|
|
7020
|
+
}
|
|
5675
7021
|
continue;
|
|
5676
7022
|
}
|
|
5677
7023
|
if (!lang || !parsed) {
|
|
@@ -5681,7 +7027,8 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5681
7027
|
lang,
|
|
5682
7028
|
mtimeMs: Math.floor(stat3.mtimeMs),
|
|
5683
7029
|
symbolCount: 0,
|
|
5684
|
-
lastIndexed: Date.now()
|
|
7030
|
+
lastIndexed: Date.now(),
|
|
7031
|
+
contentHash: result.contentHash ?? ""
|
|
5685
7032
|
});
|
|
5686
7033
|
filesIndexed++;
|
|
5687
7034
|
}
|
|
@@ -5693,7 +7040,8 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5693
7040
|
lang,
|
|
5694
7041
|
mtimeMs: Math.floor(stat3.mtimeMs),
|
|
5695
7042
|
symbolCount: 0,
|
|
5696
|
-
lastIndexed: Date.now()
|
|
7043
|
+
lastIndexed: Date.now(),
|
|
7044
|
+
contentHash: result.contentHash ?? ""
|
|
5697
7045
|
});
|
|
5698
7046
|
filesIndexed++;
|
|
5699
7047
|
continue;
|
|
@@ -5704,7 +7052,8 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5704
7052
|
symbols: parsed.symbols,
|
|
5705
7053
|
refs: parsed.refs ?? [],
|
|
5706
7054
|
mtimeMs: Math.floor(stat3.mtimeMs),
|
|
5707
|
-
symbolCount: parsed.symbols.length
|
|
7055
|
+
symbolCount: parsed.symbols.length,
|
|
7056
|
+
contentHash: result.contentHash ?? ""
|
|
5708
7057
|
});
|
|
5709
7058
|
deleteForFiles.push(file);
|
|
5710
7059
|
}
|
|
@@ -5741,7 +7090,8 @@ async function runIndexerWithStore(store, opts) {
|
|
|
5741
7090
|
lang: entry.lang,
|
|
5742
7091
|
mtimeMs: entry.mtimeMs,
|
|
5743
7092
|
symbolCount: entry.symbolCount,
|
|
5744
|
-
lastIndexed: Date.now()
|
|
7093
|
+
lastIndexed: Date.now(),
|
|
7094
|
+
contentHash: entry.contentHash
|
|
5745
7095
|
});
|
|
5746
7096
|
} catch (innerErr) {
|
|
5747
7097
|
errors.push(
|
|
@@ -5853,6 +7203,9 @@ function symbolGraphService(args) {
|
|
|
5853
7203
|
function incomingCallsService(args) {
|
|
5854
7204
|
const store = indexStorePool.acquire(args.projectRoot, { indexDir: args.indexDir });
|
|
5855
7205
|
try {
|
|
7206
|
+
if (args.transitive) {
|
|
7207
|
+
return store.findTransitiveIncomingCallsByName(args.symbol, args.file, args.limit ?? 100);
|
|
7208
|
+
}
|
|
5856
7209
|
return store.findIncomingCallsByName(args.symbol, args.file, args.limit ?? 100);
|
|
5857
7210
|
} finally {
|
|
5858
7211
|
indexStorePool.release(store);
|
|
@@ -5861,6 +7214,9 @@ function incomingCallsService(args) {
|
|
|
5861
7214
|
function outgoingCallsService(args) {
|
|
5862
7215
|
const store = indexStorePool.acquire(args.projectRoot, { indexDir: args.indexDir });
|
|
5863
7216
|
try {
|
|
7217
|
+
if (args.transitive) {
|
|
7218
|
+
return store.findTransitiveOutgoingCallsByName(args.symbol, args.file, args.limit ?? 100);
|
|
7219
|
+
}
|
|
5864
7220
|
return store.findOutgoingCallsByName(args.symbol, args.file, args.limit ?? 100);
|
|
5865
7221
|
} finally {
|
|
5866
7222
|
indexStorePool.release(store);
|
|
@@ -5869,17 +7225,35 @@ function outgoingCallsService(args) {
|
|
|
5869
7225
|
|
|
5870
7226
|
// src/codebase-index/project-server-client.ts
|
|
5871
7227
|
import { spawn as spawn3 } from "node:child_process";
|
|
5872
|
-
import * as
|
|
7228
|
+
import * as fs11 from "node:fs";
|
|
5873
7229
|
import * as net from "node:net";
|
|
5874
|
-
import {
|
|
7230
|
+
import { StringDecoder } from "node:string_decoder";
|
|
7231
|
+
import { fileURLToPath as fileURLToPath4 } from "node:url";
|
|
5875
7232
|
import { checkUnixSocketPath } from "@wrongstack/core/utils";
|
|
5876
7233
|
|
|
7234
|
+
// src/codebase-index/binary-frame.ts
|
|
7235
|
+
import { decode, encode } from "@msgpack/msgpack";
|
|
7236
|
+
var BINARY_FRAME_MAGIC = 87;
|
|
7237
|
+
function isBinaryFrame(firstByte) {
|
|
7238
|
+
return firstByte === BINARY_FRAME_MAGIC;
|
|
7239
|
+
}
|
|
7240
|
+
function encodeBinaryFrame(message) {
|
|
7241
|
+
const payload = encode(message);
|
|
7242
|
+
const header = Buffer.allocUnsafe(5);
|
|
7243
|
+
header[0] = BINARY_FRAME_MAGIC;
|
|
7244
|
+
header.writeUInt32BE(payload.length, 1);
|
|
7245
|
+
return Buffer.concat([header, payload], 5 + payload.length);
|
|
7246
|
+
}
|
|
7247
|
+
function decodeBinaryFrame(payload) {
|
|
7248
|
+
return decode(payload);
|
|
7249
|
+
}
|
|
7250
|
+
|
|
5877
7251
|
// src/codebase-index/project-server-endpoint.ts
|
|
5878
7252
|
import { createHash as createHash2 } from "node:crypto";
|
|
5879
|
-
import * as
|
|
7253
|
+
import * as fs10 from "node:fs";
|
|
5880
7254
|
import * as os3 from "node:os";
|
|
5881
|
-
import * as
|
|
5882
|
-
import { fileURLToPath } from "node:url";
|
|
7255
|
+
import * as path14 from "node:path";
|
|
7256
|
+
import { fileURLToPath as fileURLToPath3 } from "node:url";
|
|
5883
7257
|
import { assertUnixSocketPathWithinLimit } from "@wrongstack/core/utils";
|
|
5884
7258
|
var PROJECT_INDEX_SERVER_PROTOCOL_VERSION = 1;
|
|
5885
7259
|
var PROJECT_INDEX_SERVER_METADATA_FILE = "server.json";
|
|
@@ -5888,21 +7262,21 @@ var buildIdCache;
|
|
|
5888
7262
|
function projectIndexServerBuildId(entrypoint) {
|
|
5889
7263
|
const href = entrypoint instanceof URL ? entrypoint.href : entrypoint;
|
|
5890
7264
|
const cleanHref = href.split(/[?#]/, 1)[0] ?? href;
|
|
5891
|
-
const file = cleanHref.startsWith("file:") ?
|
|
7265
|
+
const file = cleanHref.startsWith("file:") ? fileURLToPath3(cleanHref) : path14.resolve(cleanHref);
|
|
5892
7266
|
try {
|
|
5893
|
-
const stat3 =
|
|
7267
|
+
const stat3 = fs10.statSync(file);
|
|
5894
7268
|
if (buildIdCache?.file === file && buildIdCache.mtimeMs === stat3.mtimeMs && buildIdCache.size === stat3.size) {
|
|
5895
7269
|
return buildIdCache.buildId;
|
|
5896
7270
|
}
|
|
5897
|
-
const buildId = createHash2("sha256").update(
|
|
7271
|
+
const buildId = createHash2("sha256").update(fs10.readFileSync(file)).digest("hex").slice(0, 24);
|
|
5898
7272
|
buildIdCache = { file, mtimeMs: stat3.mtimeMs, size: stat3.size, buildId };
|
|
5899
7273
|
return buildId;
|
|
5900
7274
|
} catch {
|
|
5901
|
-
return `unreadable:${
|
|
7275
|
+
return `unreadable:${path14.basename(file)}`;
|
|
5902
7276
|
}
|
|
5903
7277
|
}
|
|
5904
7278
|
function normalizeLocalPath(value) {
|
|
5905
|
-
const resolved =
|
|
7279
|
+
const resolved = path14.resolve(value);
|
|
5906
7280
|
return process.platform === "win32" ? resolved.toLowerCase() : resolved;
|
|
5907
7281
|
}
|
|
5908
7282
|
function projectIndexServerKey(projectRoot, indexDir) {
|
|
@@ -5914,11 +7288,11 @@ function projectIndexServerEndpoint(projectRoot, indexDir) {
|
|
|
5914
7288
|
if (process.platform === "win32") {
|
|
5915
7289
|
return `\\\\.\\pipe\\wrongstack-codebase-index-v${PROJECT_INDEX_SERVER_PROTOCOL_VERSION}-${key}`;
|
|
5916
7290
|
}
|
|
5917
|
-
return
|
|
7291
|
+
return path14.join(os3.tmpdir(), PROJECT_INDEX_SERVER_SOCKET_DIR, `${key}.sock`);
|
|
5918
7292
|
}
|
|
5919
7293
|
function projectIndexServerMetadataPath(projectRoot, indexDir) {
|
|
5920
|
-
return
|
|
5921
|
-
|
|
7294
|
+
return path14.join(
|
|
7295
|
+
path14.resolve(resolveIndexDir(projectRoot, indexDir)),
|
|
5922
7296
|
PROJECT_INDEX_SERVER_METADATA_FILE
|
|
5923
7297
|
);
|
|
5924
7298
|
}
|
|
@@ -5958,7 +7332,7 @@ function resolveProjectIndexDaemonAvailability(projectRoot, indexDir) {
|
|
|
5958
7332
|
for (const rel of ["./project-server.js", "./codebase-index/project-server.js"]) {
|
|
5959
7333
|
try {
|
|
5960
7334
|
const url = new URL(rel, import.meta.url);
|
|
5961
|
-
if (url.protocol === "file:" &&
|
|
7335
|
+
if (url.protocol === "file:" && fs11.existsSync(fileURLToPath4(url))) {
|
|
5962
7336
|
builtUrl = url;
|
|
5963
7337
|
break;
|
|
5964
7338
|
}
|
|
@@ -6057,6 +7431,12 @@ var ProjectServerConnection = class {
|
|
|
6057
7431
|
endpoint;
|
|
6058
7432
|
socket = null;
|
|
6059
7433
|
buffer = "";
|
|
7434
|
+
/** P6: binary frame buffer — accumulates raw bytes when in binary mode. */
|
|
7435
|
+
binaryBuffer = [];
|
|
7436
|
+
/** P6: StringDecoder for safe UTF-8 multibyte handling in JSON mode. */
|
|
7437
|
+
textDecoder = null;
|
|
7438
|
+
/** P6: true once the server advertises binary support and client accepts. */
|
|
7439
|
+
useBinary = false;
|
|
6060
7440
|
info = null;
|
|
6061
7441
|
activity = null;
|
|
6062
7442
|
health = null;
|
|
@@ -6195,6 +7575,8 @@ var ProjectServerConnection = class {
|
|
|
6195
7575
|
this.info = null;
|
|
6196
7576
|
this.activity = null;
|
|
6197
7577
|
this.health = null;
|
|
7578
|
+
this.useBinary = false;
|
|
7579
|
+
this.binaryBuffer = [];
|
|
6198
7580
|
this.connectReject?.(new Error("codebase-index client disconnected"));
|
|
6199
7581
|
this.connectResolve = null;
|
|
6200
7582
|
this.connectReject = null;
|
|
@@ -6215,7 +7597,7 @@ var ProjectServerConnection = class {
|
|
|
6215
7597
|
currentAuthToken() {
|
|
6216
7598
|
if (this.authToken === void 0) {
|
|
6217
7599
|
try {
|
|
6218
|
-
const raw =
|
|
7600
|
+
const raw = fs11.readFileSync(
|
|
6219
7601
|
projectIndexServerMetadataPath(this.projectRoot, this.indexDir),
|
|
6220
7602
|
"utf8"
|
|
6221
7603
|
);
|
|
@@ -6323,10 +7705,12 @@ var ProjectServerConnection = class {
|
|
|
6323
7705
|
this.activity = null;
|
|
6324
7706
|
this.health = null;
|
|
6325
7707
|
this.buffer = "";
|
|
7708
|
+
this.binaryBuffer = [];
|
|
7709
|
+
this.useBinary = false;
|
|
7710
|
+
this.textDecoder = null;
|
|
6326
7711
|
return new Promise((resolve4, reject) => {
|
|
6327
7712
|
const socket = net.createConnection(this.endpoint);
|
|
6328
7713
|
this.socket = socket;
|
|
6329
|
-
socket.setEncoding("utf8");
|
|
6330
7714
|
const timer = setTimeout(() => {
|
|
6331
7715
|
reject(new Error("codebase-index server handshake timed out"));
|
|
6332
7716
|
socket.destroy();
|
|
@@ -6355,7 +7739,12 @@ var ProjectServerConnection = class {
|
|
|
6355
7739
|
}
|
|
6356
7740
|
onData(socket, chunk) {
|
|
6357
7741
|
if (socket !== this.socket) return;
|
|
6358
|
-
this.
|
|
7742
|
+
if (this.useBinary) {
|
|
7743
|
+
this.onBinaryData(socket, chunk);
|
|
7744
|
+
return;
|
|
7745
|
+
}
|
|
7746
|
+
if (!this.textDecoder) this.textDecoder = new StringDecoder("utf8");
|
|
7747
|
+
this.buffer += this.textDecoder.write(chunk);
|
|
6359
7748
|
while (true) {
|
|
6360
7749
|
const newline = this.buffer.indexOf("\n");
|
|
6361
7750
|
if (newline < 0) {
|
|
@@ -6381,6 +7770,44 @@ var ProjectServerConnection = class {
|
|
|
6381
7770
|
this.onMessage(message);
|
|
6382
7771
|
}
|
|
6383
7772
|
}
|
|
7773
|
+
/**
|
|
7774
|
+
* P6: Parse binary frames from the raw buffer.
|
|
7775
|
+
*
|
|
7776
|
+
* Each frame is: [1 byte magic 0x57] [4 bytes uint32 BE length] [payload].
|
|
7777
|
+
* The magic byte distinguishes binary from JSON — a JSON frame's first byte
|
|
7778
|
+
* is `{` (0x7B), so there is no ambiguity even in a mixed-mode buffer.
|
|
7779
|
+
*/
|
|
7780
|
+
onBinaryData(socket, chunk) {
|
|
7781
|
+
this.binaryBuffer.push(chunk);
|
|
7782
|
+
const all = Buffer.concat(this.binaryBuffer);
|
|
7783
|
+
let offset = 0;
|
|
7784
|
+
while (offset + 5 <= all.length) {
|
|
7785
|
+
if (!isBinaryFrame(all[offset])) {
|
|
7786
|
+
this.useBinary = false;
|
|
7787
|
+
this.buffer += all.subarray(offset).toString("utf8");
|
|
7788
|
+
this.binaryBuffer = [];
|
|
7789
|
+
return;
|
|
7790
|
+
}
|
|
7791
|
+
const frameLen = all.readUInt32BE(offset + 1);
|
|
7792
|
+
if (frameLen > 256 * 1024 * 1024) {
|
|
7793
|
+
socket.destroy();
|
|
7794
|
+
this.transition("offline", { error: "binary frame length exceeds 256MB limit" });
|
|
7795
|
+
return;
|
|
7796
|
+
}
|
|
7797
|
+
const totalLen = 5 + frameLen;
|
|
7798
|
+
if (offset + totalLen > all.length) break;
|
|
7799
|
+
const payload = all.subarray(offset + 5, offset + 5 + frameLen);
|
|
7800
|
+
try {
|
|
7801
|
+
const message = decodeBinaryFrame(payload);
|
|
7802
|
+
this.onMessage(message);
|
|
7803
|
+
} catch {
|
|
7804
|
+
socket.destroy(new Error("invalid binary codebase-index server response"));
|
|
7805
|
+
return;
|
|
7806
|
+
}
|
|
7807
|
+
offset += totalLen;
|
|
7808
|
+
}
|
|
7809
|
+
this.binaryBuffer = offset < all.length ? [all.subarray(offset)] : [];
|
|
7810
|
+
}
|
|
6384
7811
|
onMessage(message) {
|
|
6385
7812
|
if (message.type === "hello") {
|
|
6386
7813
|
if (message.protocolVersion !== PROJECT_INDEX_SERVER_PROTOCOL_VERSION) {
|
|
@@ -6400,6 +7827,7 @@ var ProjectServerConnection = class {
|
|
|
6400
7827
|
}
|
|
6401
7828
|
this.info = message;
|
|
6402
7829
|
this.markResponsive();
|
|
7830
|
+
if (message.binarySupported) this.useBinary = true;
|
|
6403
7831
|
this.transition("connected", { pid: message.pid });
|
|
6404
7832
|
ensureHeartbeatLoop();
|
|
6405
7833
|
this.connectResolve?.();
|
|
@@ -6458,7 +7886,12 @@ var ProjectServerConnection = class {
|
|
|
6458
7886
|
}
|
|
6459
7887
|
write(message) {
|
|
6460
7888
|
const socket = this.socket;
|
|
6461
|
-
if (socket
|
|
7889
|
+
if (!socket || socket.destroyed) return;
|
|
7890
|
+
if (this.useBinary) {
|
|
7891
|
+
socket.write(encodeBinaryFrame(message));
|
|
7892
|
+
} else {
|
|
7893
|
+
socket.write(encodeProjectServerMessage(message));
|
|
7894
|
+
}
|
|
6462
7895
|
}
|
|
6463
7896
|
rejectStaleServer(message, reason) {
|
|
6464
7897
|
const socket = this.socket;
|
|
@@ -6480,11 +7913,11 @@ var ProjectServerConnection = class {
|
|
|
6480
7913
|
if (!url) throw new Error("built codebase-index project server is unavailable");
|
|
6481
7914
|
if (process.platform !== "win32") {
|
|
6482
7915
|
try {
|
|
6483
|
-
|
|
7916
|
+
fs11.rmSync(this.endpoint, { force: true });
|
|
6484
7917
|
} catch {
|
|
6485
7918
|
}
|
|
6486
7919
|
}
|
|
6487
|
-
const args = [
|
|
7920
|
+
const args = [fileURLToPath4(url), "--project-root", this.projectRoot];
|
|
6488
7921
|
if (this.indexDir) args.push("--index-dir", this.indexDir);
|
|
6489
7922
|
const child = spawn3(process.execPath, args, {
|
|
6490
7923
|
detached: true,
|
|
@@ -6504,8 +7937,8 @@ var ProjectServerConnection = class {
|
|
|
6504
7937
|
process.kill(pid);
|
|
6505
7938
|
const metadataPath = projectIndexServerMetadataPath(this.projectRoot, this.indexDir);
|
|
6506
7939
|
try {
|
|
6507
|
-
const metadata = JSON.parse(
|
|
6508
|
-
if (metadata.pid === pid)
|
|
7940
|
+
const metadata = JSON.parse(fs11.readFileSync(metadataPath, "utf8"));
|
|
7941
|
+
if (metadata.pid === pid) fs11.rmSync(metadataPath, { force: true });
|
|
6509
7942
|
} catch {
|
|
6510
7943
|
}
|
|
6511
7944
|
return true;
|
|
@@ -6604,7 +8037,7 @@ function resolveWorkerUrl() {
|
|
|
6604
8037
|
for (const rel of ["./worker.js", "./codebase-index/worker.js"]) {
|
|
6605
8038
|
try {
|
|
6606
8039
|
const url = new URL(rel, import.meta.url);
|
|
6607
|
-
if (url.protocol === "file:" &&
|
|
8040
|
+
if (url.protocol === "file:" && fs12.existsSync(fileURLToPath5(url))) return url;
|
|
6608
8041
|
} catch {
|
|
6609
8042
|
}
|
|
6610
8043
|
}
|
|
@@ -6624,7 +8057,7 @@ function ensureWorker() {
|
|
|
6624
8057
|
return null;
|
|
6625
8058
|
}
|
|
6626
8059
|
try {
|
|
6627
|
-
const w = new
|
|
8060
|
+
const w = new Worker2(url, { name: "wstack-codebase-index" });
|
|
6628
8061
|
w.unref();
|
|
6629
8062
|
w.on("message", (msg) => {
|
|
6630
8063
|
if (msg.type === "progress") {
|
|
@@ -6848,7 +8281,7 @@ var readTool = {
|
|
|
6848
8281
|
const shouldIncludeSymbols = input.includeSymbols === true || input.includeSymbols !== false && ctx.meta[ADVANCED_MODE_META_KEY] === true;
|
|
6849
8282
|
let stat3;
|
|
6850
8283
|
try {
|
|
6851
|
-
stat3 = await
|
|
8284
|
+
stat3 = await fs13.stat(absPath);
|
|
6852
8285
|
} catch (err) {
|
|
6853
8286
|
const code = err.code;
|
|
6854
8287
|
if (code === "ENOENT") {
|
|
@@ -6900,7 +8333,7 @@ var readTool = {
|
|
|
6900
8333
|
...symResult2?.symbols ? { symbols: symResult2.symbols } : {}
|
|
6901
8334
|
};
|
|
6902
8335
|
}
|
|
6903
|
-
const buf = await
|
|
8336
|
+
const buf = await fs13.readFile(absPath);
|
|
6904
8337
|
if (isBinaryBuffer(buf)) {
|
|
6905
8338
|
throw new Error(`read: "${input.path}" appears to be binary`);
|
|
6906
8339
|
}
|