@wrongstack/tools 0.306.4 → 0.307.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/audit.js +14 -8
- package/dist/auto-proceed-loop-guard.js +8 -2
- package/dist/bash.js +57 -22
- package/dist/builtin.d.ts +6 -6
- package/dist/builtin.js +11469 -9304
- package/dist/clarify.d.ts +32 -0
- package/dist/codebase-index/ast-invariant-engine.d.ts +102 -0
- package/dist/codebase-index/ast-symbol-mutator.d.ts +28 -0
- package/dist/codebase-index/background-indexer.d.ts +9 -0
- package/dist/codebase-index/codebase-ast-replace-tool.d.ts +31 -0
- package/dist/codebase-index/codebase-impact-analysis-tool.d.ts +42 -0
- package/dist/codebase-index/codebase-invariant-check-tool.d.ts +23 -0
- package/dist/codebase-index/codebase-repo-map-tool.d.ts +22 -0
- package/dist/codebase-index/codebase-skeleton-tool.d.ts +36 -0
- package/dist/codebase-index/codebase-targeted-test-tool.d.ts +30 -0
- package/dist/codebase-index/index.d.ts +11 -1
- package/dist/codebase-index/index.js +4239 -1492
- package/dist/codebase-index/project-server-client-state.d.ts +79 -0
- package/dist/codebase-index/project-server-client.d.ts +3 -71
- package/dist/codebase-index/project-server-protocol.d.ts +1 -0
- package/dist/codebase-index/project-server.js +819 -933
- package/dist/codebase-index/repo-map.d.ts +20 -0
- package/dist/codebase-index/skeleton-extractor.d.ts +64 -0
- package/dist/codebase-index/tree-sitter-parser.d.ts +11 -0
- package/dist/codebase-index/worker.js +790 -914
- package/dist/codebase-index/writer-graph-helpers.d.ts +1 -1
- package/dist/codebase-index/writer-helpers.d.ts +12 -0
- package/dist/codebase-index/writer-mutations.d.ts +26 -0
- package/dist/codebase-index/writer-refs.d.ts +50 -0
- package/dist/codebase-index/writer-search.d.ts +30 -0
- package/dist/codebase-index/writer.d.ts +2 -320
- package/dist/edit.js +8602 -349
- package/dist/exec.js +73 -20
- package/dist/fetch.js +11 -9
- package/dist/format.js +14 -8
- package/dist/glob.js +5 -1
- package/dist/grep.js +19 -8
- package/dist/index.d.ts +3 -1
- package/dist/index.js +6178 -4037
- package/dist/install.js +14 -8
- package/dist/json.js +88 -15
- package/dist/kanban-board-actions.d.ts +4 -0
- package/dist/kanban-lifecycle-actions.d.ts +4 -0
- package/dist/kanban-serializer.d.ts +21 -0
- package/dist/kanban.js +985 -1034
- package/dist/languages/index.js +14 -8
- package/dist/lint.js +14 -8
- package/dist/logs.js +13 -3
- package/dist/next-steps.d.ts +8 -0
- package/dist/next-steps.js +18 -0
- package/dist/outdated.js +14 -8
- package/dist/pack.js +11461 -9304
- package/dist/patch.js +8345 -78
- package/dist/plan.js +1231 -1281
- package/dist/process-registry.js +14 -8
- package/dist/ps-slash.js +65 -30
- package/dist/read.js +822 -937
- package/dist/replace.js +8494 -224
- package/dist/search.js +21 -15
- package/dist/security-ast-scan-tool.d.ts +43 -0
- package/dist/session-kanban-graph.d.ts +9 -0
- package/dist/session-kanban-sync.d.ts +31 -0
- package/dist/session-kanban.d.ts +5 -141
- package/dist/session-kanban.js +367 -360
- package/dist/task.js +1157 -1207
- package/dist/test.js +14 -8
- package/dist/todo.js +2187 -2237
- package/dist/tool-diff.js +6 -1
- package/dist/tool-summary.js +4 -2
- package/dist/tool-tier.js +11469 -9304
- package/dist/typecheck.js +14 -8
- package/dist/write.js +8425 -160
- package/package.json +7 -7
|
@@ -562,17 +562,44 @@ function fallbackParse(filePath, content, lang) {
|
|
|
562
562
|
const col = line.length - trimmed.length + 1;
|
|
563
563
|
const fn = /^func\s+(?:\([^)]*\)\s*)?([A-Za-z_]\w*)\s*\(/.exec(trimmed);
|
|
564
564
|
if (fn?.[1]) {
|
|
565
|
-
addFallbackSymbol(symbols, {
|
|
565
|
+
addFallbackSymbol(symbols, {
|
|
566
|
+
filePath,
|
|
567
|
+
lang,
|
|
568
|
+
kind: trimmed.startsWith("func (") ? "method" : "function",
|
|
569
|
+
name: fn[1],
|
|
570
|
+
line: idx + 1,
|
|
571
|
+
col,
|
|
572
|
+
signature: trimmed,
|
|
573
|
+
scope: packageName ? `${packageName}.${fn[1]}` : fn[1]
|
|
574
|
+
});
|
|
566
575
|
continue;
|
|
567
576
|
}
|
|
568
577
|
const typeDecl = /^type\s+([A-Za-z_]\w*)\b/.exec(trimmed);
|
|
569
578
|
if (typeDecl?.[1]) {
|
|
570
|
-
addFallbackSymbol(symbols, {
|
|
579
|
+
addFallbackSymbol(symbols, {
|
|
580
|
+
filePath,
|
|
581
|
+
lang,
|
|
582
|
+
kind: "type",
|
|
583
|
+
name: typeDecl[1],
|
|
584
|
+
line: idx + 1,
|
|
585
|
+
col,
|
|
586
|
+
signature: trimmed,
|
|
587
|
+
scope: packageName
|
|
588
|
+
});
|
|
571
589
|
continue;
|
|
572
590
|
}
|
|
573
591
|
const valueDecl = /^(const|var)\s+([A-Za-z_]\w*)\b/.exec(trimmed);
|
|
574
592
|
if (valueDecl?.[1] && valueDecl[2]) {
|
|
575
|
-
addFallbackSymbol(symbols, {
|
|
593
|
+
addFallbackSymbol(symbols, {
|
|
594
|
+
filePath,
|
|
595
|
+
lang,
|
|
596
|
+
kind: valueDecl[1],
|
|
597
|
+
name: valueDecl[2],
|
|
598
|
+
line: idx + 1,
|
|
599
|
+
col,
|
|
600
|
+
signature: trimmed,
|
|
601
|
+
scope: packageName
|
|
602
|
+
});
|
|
576
603
|
}
|
|
577
604
|
}
|
|
578
605
|
return { file: filePath, lang, symbols, mtimeMs: Date.now() };
|
|
@@ -1023,7 +1050,10 @@ function parseGeneric(opts) {
|
|
|
1023
1050
|
const seen = /* @__PURE__ */ new Set();
|
|
1024
1051
|
const nlOffsets = newlineOffsets2(content);
|
|
1025
1052
|
for (const pattern of patterns) {
|
|
1026
|
-
const re = new RegExp(
|
|
1053
|
+
const re = new RegExp(
|
|
1054
|
+
pattern.re.source,
|
|
1055
|
+
pattern.re.flags.includes("g") ? pattern.re.flags : `${pattern.re.flags}g`
|
|
1056
|
+
);
|
|
1027
1057
|
re.lastIndex = 0;
|
|
1028
1058
|
for (const match of content.matchAll(re)) {
|
|
1029
1059
|
if (symbols.length >= maxSymbols) break;
|
|
@@ -1130,7 +1160,10 @@ var init_generic_parser = __esm({
|
|
|
1130
1160
|
],
|
|
1131
1161
|
kotlin: [
|
|
1132
1162
|
{ re: /\b(?:fun)\s+([A-Za-z_]\w*)/g, kind: "function" },
|
|
1133
|
-
{
|
|
1163
|
+
{
|
|
1164
|
+
re: /\b(?:class|interface|object|enum\s+class|data\s+class)\s+([A-Za-z_]\w*)/g,
|
|
1165
|
+
kind: "class"
|
|
1166
|
+
}
|
|
1134
1167
|
],
|
|
1135
1168
|
scala: [
|
|
1136
1169
|
{ re: /\b(?:def)\s+([A-Za-z_]\w*)/g, kind: "function" },
|
|
@@ -1141,14 +1174,13 @@ var init_generic_parser = __esm({
|
|
|
1141
1174
|
{ re: /^([A-Za-z_][\w]*)\s*\(\)\s*\{/gm, kind: "function" }
|
|
1142
1175
|
],
|
|
1143
1176
|
sql: [
|
|
1144
|
-
{
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
],
|
|
1149
|
-
toml: [
|
|
1150
|
-
{ re: /^\[([^\]]+)\]/gm, kind: "namespace" }
|
|
1177
|
+
{
|
|
1178
|
+
re: /\bCREATE\s+(?:OR\s+REPLACE\s+)?(?:TABLE|VIEW|INDEX|FUNCTION|PROCEDURE|TRIGGER)\s+(?:IF\s+NOT\s+EXISTS\s+)?([A-Za-z_"][\w."]*)/gi,
|
|
1179
|
+
kind: "type"
|
|
1180
|
+
}
|
|
1151
1181
|
],
|
|
1182
|
+
md: [{ re: /^(#{1,6})\s+(.+)$/gm, kind: "namespace" }],
|
|
1183
|
+
toml: [{ re: /^\[([^\]]+)\]/gm, kind: "namespace" }],
|
|
1152
1184
|
html: [
|
|
1153
1185
|
{ re: /\bid\s*=\s*["']([^"']+)["']/gi, kind: "property" },
|
|
1154
1186
|
{ re: /<(?:script|template|style)\b/gi, kind: "namespace" }
|
|
@@ -1158,11 +1190,17 @@ var init_generic_parser = __esm({
|
|
|
1158
1190
|
{ re: /@(?:keyframes|media|supports)\s+([^{\s]+)/g, kind: "namespace" }
|
|
1159
1191
|
],
|
|
1160
1192
|
vue: [
|
|
1161
|
-
{
|
|
1193
|
+
{
|
|
1194
|
+
re: /\b(?:function|const|let|var|class|export\s+(?:default\s+)?(?:function|class|const))\s+([A-Za-z_]\w*)/g,
|
|
1195
|
+
kind: "function"
|
|
1196
|
+
},
|
|
1162
1197
|
{ re: /<(?:script|template|style)\b/gi, kind: "namespace" }
|
|
1163
1198
|
],
|
|
1164
1199
|
svelte: [
|
|
1165
|
-
{
|
|
1200
|
+
{
|
|
1201
|
+
re: /\b(?:function|const|let|var|class|export\s+(?:default\s+)?(?:function|class|const))\s+([A-Za-z_]\w*)/g,
|
|
1202
|
+
kind: "function"
|
|
1203
|
+
}
|
|
1166
1204
|
],
|
|
1167
1205
|
dart: [
|
|
1168
1206
|
{ re: /\b(?:class|enum|mixin|extension)\s+([A-Za-z_]\w*)/g, kind: "class" },
|
|
@@ -1376,12 +1414,7 @@ async function syncPyParse(filePath, content, lang) {
|
|
|
1376
1414
|
cachedPyBinary ??= resolvePython();
|
|
1377
1415
|
const pyBinary = await cachedPyBinary;
|
|
1378
1416
|
if (!pyBinary) return null;
|
|
1379
|
-
const { code, stdout } = await spawnPyParser(
|
|
1380
|
-
pyBinary,
|
|
1381
|
-
_cachedScriptPath,
|
|
1382
|
-
filePath,
|
|
1383
|
-
content
|
|
1384
|
-
);
|
|
1417
|
+
const { code, stdout } = await spawnPyParser(pyBinary, _cachedScriptPath, filePath, content);
|
|
1385
1418
|
if (code !== 0 || !stdout.trim()) {
|
|
1386
1419
|
return { file: filePath, lang, symbols: [], mtimeMs: Date.now() };
|
|
1387
1420
|
}
|
|
@@ -2575,7 +2608,8 @@ __export(tree_sitter_parser_exports, {
|
|
|
2575
2608
|
getGrammarWasmPath: () => getGrammarWasmPath,
|
|
2576
2609
|
isTreeSitterSupported: () => isTreeSitterSupported,
|
|
2577
2610
|
loadTreeSitterLanguage: () => loadTreeSitterLanguage,
|
|
2578
|
-
parseSymbols: () => parseSymbols8
|
|
2611
|
+
parseSymbols: () => parseSymbols8,
|
|
2612
|
+
parseTreeSitterAst: () => parseTreeSitterAst
|
|
2579
2613
|
});
|
|
2580
2614
|
import * as path9 from "node:path";
|
|
2581
2615
|
import { fileURLToPath } from "node:url";
|
|
@@ -2667,6 +2701,26 @@ async function __smokeRootType(opts) {
|
|
|
2667
2701
|
parser.delete();
|
|
2668
2702
|
}
|
|
2669
2703
|
}
|
|
2704
|
+
async function parseTreeSitterAst(opts) {
|
|
2705
|
+
const grammar = resolveGrammarName(opts.lang) ?? (opts.lang === "go" ? "go" : opts.lang === "py" ? "python" : opts.lang === "rs" ? "rust" : void 0);
|
|
2706
|
+
if (!grammar) return null;
|
|
2707
|
+
try {
|
|
2708
|
+
const { Parser, Language, init } = await getRuntime();
|
|
2709
|
+
await init();
|
|
2710
|
+
const wasmPath = path9.join(WASM_DIR, grammar, `tree-sitter-${grammar}.wasm`);
|
|
2711
|
+
const languageObj = await Language.load(wasmPath);
|
|
2712
|
+
const parser = new Parser();
|
|
2713
|
+
parser.setLanguage(languageObj);
|
|
2714
|
+
const tree = parser.parse(opts.content);
|
|
2715
|
+
if (!tree) {
|
|
2716
|
+
parser.delete();
|
|
2717
|
+
return null;
|
|
2718
|
+
}
|
|
2719
|
+
return { tree, parser };
|
|
2720
|
+
} catch {
|
|
2721
|
+
return null;
|
|
2722
|
+
}
|
|
2723
|
+
}
|
|
2670
2724
|
var WASM_DIR, RUNTIME_WASM, LANG_TO_GRAMMAR, GO_OPT_IN, PY_OPT_IN, RS_OPT_IN, runtimePromise, languageCache;
|
|
2671
2725
|
var init_tree_sitter_parser = __esm({
|
|
2672
2726
|
"src/codebase-index/tree-sitter-parser.ts"() {
|
|
@@ -3467,10 +3521,7 @@ var LANG_IMPORTS = {
|
|
|
3467
3521
|
// quoted path. A stray match elsewhere resolves to no file and is dropped.
|
|
3468
3522
|
{ re: /^[ \t]*(?:[A-Za-z_.]\w*\s+)?"([^"]+)"\s*$/gm }
|
|
3469
3523
|
],
|
|
3470
|
-
py: [
|
|
3471
|
-
{ re: /^[ \t]*import\s+([\w.]+)/gm },
|
|
3472
|
-
{ re: /^[ \t]*from\s+([.\w]+)\s+import\b/gm }
|
|
3473
|
-
],
|
|
3524
|
+
py: [{ re: /^[ \t]*import\s+([\w.]+)/gm }, { re: /^[ \t]*from\s+([.\w]+)\s+import\b/gm }],
|
|
3474
3525
|
rs: [
|
|
3475
3526
|
// use a::b::C; | use a::b::{C, D}; → the path before any brace
|
|
3476
3527
|
{ re: /^[ \t]*(?:pub\s+)?use\s+([\w:]+?)(?:::\{|\s*;|\s+as\b)/gm },
|
|
@@ -3807,10 +3858,7 @@ function defaultWorkerCount() {
|
|
|
3807
3858
|
return Math.max(1, Math.min(4, cores - 1));
|
|
3808
3859
|
}
|
|
3809
3860
|
function resolveWorkerScriptUrl() {
|
|
3810
|
-
for (const rel of [
|
|
3811
|
-
"./parser-worker-script.js",
|
|
3812
|
-
"./codebase-index/parser-worker-script.js"
|
|
3813
|
-
]) {
|
|
3861
|
+
for (const rel of ["./parser-worker-script.js", "./codebase-index/parser-worker-script.js"]) {
|
|
3814
3862
|
try {
|
|
3815
3863
|
const url = new URL(rel, import.meta.url);
|
|
3816
3864
|
if (url.protocol === "file:" && fs6.existsSync(fileURLToPath2(url))) return url;
|
|
@@ -3826,7 +3874,6 @@ function getParserPool() {
|
|
|
3826
3874
|
}
|
|
3827
3875
|
|
|
3828
3876
|
// src/codebase-index/writer.ts
|
|
3829
|
-
import { expectDefined as expectDefined4 } from "@wrongstack/core/utils";
|
|
3830
3877
|
import * as fs8 from "node:fs";
|
|
3831
3878
|
import * as path11 from "node:path";
|
|
3832
3879
|
|
|
@@ -3922,39 +3969,6 @@ var Bm25Index = class {
|
|
|
3922
3969
|
// src/codebase-index/writer.ts
|
|
3923
3970
|
init_languages();
|
|
3924
3971
|
|
|
3925
|
-
// src/codebase-index/lsp-kind.ts
|
|
3926
|
-
function lspKindToInternalKind(k) {
|
|
3927
|
-
switch (k) {
|
|
3928
|
-
case 5 /* Class */:
|
|
3929
|
-
return "class";
|
|
3930
|
-
case 6 /* Method */:
|
|
3931
|
-
return "method";
|
|
3932
|
-
case 7 /* Property */:
|
|
3933
|
-
case 8 /* Field */:
|
|
3934
|
-
return "property";
|
|
3935
|
-
case 9 /* Constructor */:
|
|
3936
|
-
return "class";
|
|
3937
|
-
case 10 /* Enum */:
|
|
3938
|
-
return "enum";
|
|
3939
|
-
case 11 /* Interface */:
|
|
3940
|
-
return "interface";
|
|
3941
|
-
case 12 /* Function */:
|
|
3942
|
-
return "function";
|
|
3943
|
-
case 13 /* Variable */:
|
|
3944
|
-
return "var";
|
|
3945
|
-
case 14 /* Constant */:
|
|
3946
|
-
return "const";
|
|
3947
|
-
case 22 /* EnumMember */:
|
|
3948
|
-
return "enum";
|
|
3949
|
-
case 26 /* TypeParameter */:
|
|
3950
|
-
return "type";
|
|
3951
|
-
case 3 /* Namespace */:
|
|
3952
|
-
return "namespace";
|
|
3953
|
-
default:
|
|
3954
|
-
return null;
|
|
3955
|
-
}
|
|
3956
|
-
}
|
|
3957
|
-
|
|
3958
3972
|
// src/codebase-index/schema.ts
|
|
3959
3973
|
var SCHEMA_VERSION = 4;
|
|
3960
3974
|
|
|
@@ -4101,91 +4115,14 @@ function runSqliteWithRetry(fn) {
|
|
|
4101
4115
|
throw lastError;
|
|
4102
4116
|
}
|
|
4103
4117
|
|
|
4104
|
-
// src/codebase-index/vector-search.ts
|
|
4105
|
-
var RRF_K = 60;
|
|
4106
|
-
var VECTOR_DIMENSIONS = 384;
|
|
4107
|
-
var NGRAM_SIZE = 3;
|
|
4108
|
-
function embedText(text) {
|
|
4109
|
-
const vec = new Float32Array(VECTOR_DIMENSIONS);
|
|
4110
|
-
const normalized = text.toLowerCase().trim();
|
|
4111
|
-
if (normalized.length < NGRAM_SIZE) {
|
|
4112
|
-
const padded = ` ${normalized} `.slice(0, Math.max(NGRAM_SIZE, normalized.length + 2));
|
|
4113
|
-
for (let i = 0; i <= padded.length - NGRAM_SIZE; i++) {
|
|
4114
|
-
const ngram = padded.slice(i, i + NGRAM_SIZE);
|
|
4115
|
-
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
4116
|
-
vec[bucket] += 1;
|
|
4117
|
-
}
|
|
4118
|
-
} else {
|
|
4119
|
-
for (let i = 0; i <= normalized.length - NGRAM_SIZE; i++) {
|
|
4120
|
-
const ngram = normalized.slice(i, i + NGRAM_SIZE);
|
|
4121
|
-
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
4122
|
-
vec[bucket] += 1;
|
|
4123
|
-
}
|
|
4124
|
-
}
|
|
4125
|
-
let norm = 0;
|
|
4126
|
-
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
4127
|
-
norm += vec[i] * vec[i];
|
|
4128
|
-
}
|
|
4129
|
-
norm = Math.sqrt(norm);
|
|
4130
|
-
if (norm > 0) {
|
|
4131
|
-
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
4132
|
-
vec[i] /= norm;
|
|
4133
|
-
}
|
|
4134
|
-
}
|
|
4135
|
-
return vec;
|
|
4136
|
-
}
|
|
4137
|
-
function hashNgram(str) {
|
|
4138
|
-
let hash = 2166136261;
|
|
4139
|
-
for (let i = 0; i < str.length; i++) {
|
|
4140
|
-
hash ^= str.charCodeAt(i);
|
|
4141
|
-
hash = Math.imul(hash, 16777619);
|
|
4142
|
-
}
|
|
4143
|
-
return hash >>> 0;
|
|
4144
|
-
}
|
|
4145
|
-
function cosineSimilarity(a, b) {
|
|
4146
|
-
let dot = 0;
|
|
4147
|
-
const len = Math.min(a.length, b.length);
|
|
4148
|
-
for (let i = 0; i < len; i++) {
|
|
4149
|
-
dot += a[i] * b[i];
|
|
4150
|
-
}
|
|
4151
|
-
return dot;
|
|
4152
|
-
}
|
|
4153
|
-
function encodeVector(vec) {
|
|
4154
|
-
return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
|
|
4155
|
-
}
|
|
4156
|
-
function decodeVector(buf) {
|
|
4157
|
-
const view = new DataView(
|
|
4158
|
-
buf.buffer,
|
|
4159
|
-
buf.byteOffset,
|
|
4160
|
-
buf.byteLength
|
|
4161
|
-
);
|
|
4162
|
-
const copy = new Float32Array(buf.byteLength / 4);
|
|
4163
|
-
for (let i = 0; i < copy.length; i++) {
|
|
4164
|
-
copy[i] = view.getFloat32(i * 4, true);
|
|
4165
|
-
}
|
|
4166
|
-
return copy;
|
|
4167
|
-
}
|
|
4168
|
-
function reciprocalRankFusion(bm25Ranks, vectorRanks, k = RRF_K) {
|
|
4169
|
-
const allIds = /* @__PURE__ */ new Set([...bm25Ranks.keys(), ...vectorRanks.keys()]);
|
|
4170
|
-
const scored = [];
|
|
4171
|
-
for (const id of allIds) {
|
|
4172
|
-
const bm25Rank = bm25Ranks.get(id);
|
|
4173
|
-
const vecRank = vectorRanks.get(id);
|
|
4174
|
-
let score = 0;
|
|
4175
|
-
if (bm25Rank !== void 0) score += 1 / (k + bm25Rank);
|
|
4176
|
-
if (vecRank !== void 0) score += 1 / (k + vecRank);
|
|
4177
|
-
scored.push([id, score]);
|
|
4178
|
-
}
|
|
4179
|
-
scored.sort((a, b) => b[1] - a[1]);
|
|
4180
|
-
return scored;
|
|
4181
|
-
}
|
|
4182
|
-
|
|
4183
4118
|
// src/codebase-index/writer-admin.ts
|
|
4184
4119
|
import * as fs7 from "node:fs";
|
|
4185
4120
|
import * as path10 from "node:path";
|
|
4186
4121
|
var DB_FILE = "index.db";
|
|
4187
4122
|
function getAllIndexableWithStatement(stmt) {
|
|
4188
|
-
return stmt("SELECT id, text FROM symbols").all().map(
|
|
4123
|
+
return stmt("SELECT id, text FROM symbols").all().map(
|
|
4124
|
+
({ id, text }) => ({ id, text })
|
|
4125
|
+
);
|
|
4189
4126
|
}
|
|
4190
4127
|
function getMaxSymbolIdWithStatement(stmt) {
|
|
4191
4128
|
const rows = stmt("SELECT MAX(id) AS m FROM symbols").all();
|
|
@@ -4410,10 +4347,16 @@ function buildFileGraphNodeState(pkgSyms, localFiles, packageOf) {
|
|
|
4410
4347
|
}
|
|
4411
4348
|
return { fileNodes, symToFile, fileStats, ensureFileNode };
|
|
4412
4349
|
}
|
|
4413
|
-
function buildSymbolGraphNodes(symById, relatedIds,
|
|
4350
|
+
function buildSymbolGraphNodes(symById, relatedIds, localFiles, packageOf) {
|
|
4351
|
+
const local = new Set(
|
|
4352
|
+
[...typeof localFiles === "string" ? [localFiles] : localFiles].map(
|
|
4353
|
+
(file) => file.replace(/\\/g, "/")
|
|
4354
|
+
)
|
|
4355
|
+
);
|
|
4356
|
+
const isLocal = (file) => local.has(file.replace(/\\/g, "/"));
|
|
4414
4357
|
return [...relatedIds].map((id) => symById.get(id)).filter((symbol) => symbol !== void 0).sort((a, b) => {
|
|
4415
|
-
const aExternal = a.file
|
|
4416
|
-
const bExternal = b.file
|
|
4358
|
+
const aExternal = isLocal(a.file) ? 0 : 1;
|
|
4359
|
+
const bExternal = isLocal(b.file) ? 0 : 1;
|
|
4417
4360
|
return aExternal - bExternal || a.file.localeCompare(b.file) || a.line - b.line || a.id - b.id;
|
|
4418
4361
|
}).map((s) => ({
|
|
4419
4362
|
id: `sym:${s.id}`,
|
|
@@ -4427,7 +4370,7 @@ function buildSymbolGraphNodes(symById, relatedIds, fileFilter, packageOf) {
|
|
|
4427
4370
|
line: s.line,
|
|
4428
4371
|
signature: s.signature,
|
|
4429
4372
|
scope: s.scope,
|
|
4430
|
-
external: s.file
|
|
4373
|
+
external: !isLocal(s.file)
|
|
4431
4374
|
}));
|
|
4432
4375
|
}
|
|
4433
4376
|
function addWeightedEdge(edgeMap, source, target, callType, weight) {
|
|
@@ -4462,6 +4405,52 @@ function materializeWeightedEdges(edgeMap, idPrefix) {
|
|
|
4462
4405
|
return edges;
|
|
4463
4406
|
}
|
|
4464
4407
|
|
|
4408
|
+
// src/codebase-index/writer-helpers.ts
|
|
4409
|
+
import { resolveWstackPaths } from "@wrongstack/core/utils";
|
|
4410
|
+
function escapeLike(value) {
|
|
4411
|
+
return value.replace(/[\\%_]/g, (char) => `\\${char}`);
|
|
4412
|
+
}
|
|
4413
|
+
function posixIndexPath(file) {
|
|
4414
|
+
return file.replace(/\\/g, "/").replace(/^\.\//, "");
|
|
4415
|
+
}
|
|
4416
|
+
function indexedFileMatchSql(column = "file") {
|
|
4417
|
+
return `(${column} = ? OR replace(${column}, '\\', '/') = ? OR replace(${column}, '\\', '/') LIKE ? ESCAPE '\\')`;
|
|
4418
|
+
}
|
|
4419
|
+
function indexedFileMatchArgs(file) {
|
|
4420
|
+
const posix4 = posixIndexPath(file.trim());
|
|
4421
|
+
return [file, posix4, `%/${escapeLike(posix4)}`];
|
|
4422
|
+
}
|
|
4423
|
+
function matchesIndexedPackageFilter(storedFile, packageLabel, filter) {
|
|
4424
|
+
if (packageLabel === filter) return true;
|
|
4425
|
+
const posixFile = posixIndexPath(storedFile);
|
|
4426
|
+
const posixFilter = posixIndexPath(filter.trim());
|
|
4427
|
+
if (!posixFilter) return false;
|
|
4428
|
+
return posixFile === posixFilter || posixFile.endsWith(`/${posixFilter}`) || posixFile.includes(`/${posixFilter}/`);
|
|
4429
|
+
}
|
|
4430
|
+
function assignRefsToSymbols(refs, symbols) {
|
|
4431
|
+
if (refs.length === 0 || symbols.length === 0) return [];
|
|
4432
|
+
const ordered = [...symbols].sort((a, b) => a.line - b.line || a.col - b.col || a.id - b.id);
|
|
4433
|
+
const seen = /* @__PURE__ */ new Set();
|
|
4434
|
+
const assigned = [];
|
|
4435
|
+
for (const ref of refs) {
|
|
4436
|
+
let owner;
|
|
4437
|
+
for (const symbol of ordered) {
|
|
4438
|
+
if (symbol.line > ref.line) break;
|
|
4439
|
+
owner = symbol;
|
|
4440
|
+
}
|
|
4441
|
+
if (!owner && ref.callType === "import") owner = ordered[0];
|
|
4442
|
+
if (!owner || owner.id <= 0) continue;
|
|
4443
|
+
const key = `${owner.id}:${ref.toName}:${ref.callType}:${ref.module ?? ""}`;
|
|
4444
|
+
if (seen.has(key)) continue;
|
|
4445
|
+
seen.add(key);
|
|
4446
|
+
assigned.push({ ...ref, fromId: owner.id });
|
|
4447
|
+
}
|
|
4448
|
+
return assigned;
|
|
4449
|
+
}
|
|
4450
|
+
function resolveIndexDir(projectRoot, override) {
|
|
4451
|
+
return override ?? resolveWstackPaths({ projectRoot }).projectCodebaseIndex;
|
|
4452
|
+
}
|
|
4453
|
+
|
|
4465
4454
|
// src/codebase-index/writer-ref-mapper.ts
|
|
4466
4455
|
function mapWriterRefRow(row) {
|
|
4467
4456
|
return {
|
|
@@ -4517,15 +4506,29 @@ function mapCallSiteRow(row) {
|
|
|
4517
4506
|
line: row.ref_line
|
|
4518
4507
|
};
|
|
4519
4508
|
}
|
|
4509
|
+
function resolveIndexedFiles(stmt, file) {
|
|
4510
|
+
const rows = stmt(
|
|
4511
|
+
`SELECT DISTINCT file FROM symbols WHERE ${indexedFileMatchSql("file")} ORDER BY length(file), file`
|
|
4512
|
+
).all(...indexedFileMatchArgs(file));
|
|
4513
|
+
return rows.map((row) => row.file);
|
|
4514
|
+
}
|
|
4520
4515
|
function resolveSymbolIds(stmt, symbolName, file) {
|
|
4521
|
-
|
|
4522
|
-
|
|
4523
|
-
|
|
4516
|
+
if (!file) {
|
|
4517
|
+
const rows2 = stmt("SELECT id FROM symbols WHERE name = ? ORDER BY id").all(symbolName);
|
|
4518
|
+
return rows2.map((r) => r.id);
|
|
4519
|
+
}
|
|
4520
|
+
const indexedFiles = resolveIndexedFiles(stmt, file);
|
|
4521
|
+
if (indexedFiles.length === 0) return [];
|
|
4522
|
+
const placeholders = indexedFiles.map(() => "?").join(",");
|
|
4523
|
+
const rows = stmt(
|
|
4524
|
+
`SELECT id FROM symbols WHERE name = ? AND file IN (${placeholders}) ORDER BY id`
|
|
4525
|
+
).all(symbolName, ...indexedFiles);
|
|
4524
4526
|
return rows.map((r) => r.id);
|
|
4525
4527
|
}
|
|
4526
4528
|
function findIncomingCallsByName(stmt, symbolName, file, limit) {
|
|
4527
4529
|
const targetIds = resolveSymbolIds(stmt, symbolName, file);
|
|
4528
|
-
if (targetIds.length === 0)
|
|
4530
|
+
if (targetIds.length === 0)
|
|
4531
|
+
return { calls: [], symbolFound: false, ambiguous: false, totalMatches: 0 };
|
|
4529
4532
|
let matchIds = targetIds;
|
|
4530
4533
|
let ambiguous = false;
|
|
4531
4534
|
if (file !== void 0) {
|
|
@@ -4576,11 +4579,17 @@ function findIncomingCallsByName(stmt, symbolName, file, limit) {
|
|
|
4576
4579
|
}
|
|
4577
4580
|
rows.sort((a, b) => a.ref_line - b.ref_line || a.sym_id - b.sym_id);
|
|
4578
4581
|
const allCalls = rows.map(mapCallSiteRow);
|
|
4579
|
-
return {
|
|
4582
|
+
return {
|
|
4583
|
+
calls: allCalls.slice(0, limit),
|
|
4584
|
+
symbolFound: true,
|
|
4585
|
+
ambiguous,
|
|
4586
|
+
totalMatches: allCalls.length
|
|
4587
|
+
};
|
|
4580
4588
|
}
|
|
4581
4589
|
function findOutgoingCallsByName(stmt, symbolName, file, limit) {
|
|
4582
4590
|
const sourceIds = resolveSymbolIds(stmt, symbolName, file);
|
|
4583
|
-
if (sourceIds.length === 0)
|
|
4591
|
+
if (sourceIds.length === 0)
|
|
4592
|
+
return { calls: [], symbolFound: false, unresolvedCount: 0, totalMatches: 0 };
|
|
4584
4593
|
const unresolvedCount = chunkedIdScalar(
|
|
4585
4594
|
stmt,
|
|
4586
4595
|
sourceIds,
|
|
@@ -4835,7 +4844,7 @@ function getFileGraphWithStatement(stmt, packageFilter) {
|
|
|
4835
4844
|
const allFiles = stmt("SELECT DISTINCT file FROM symbols").all();
|
|
4836
4845
|
const packageOf = readPackageLabeller(stmt);
|
|
4837
4846
|
const langOf = (file) => detectLang(file) ?? "other";
|
|
4838
|
-
const pkgFilePaths = allFiles.filter((f) => packageOf(f.file)
|
|
4847
|
+
const pkgFilePaths = allFiles.filter((f) => matchesIndexedPackageFilter(f.file, packageOf(f.file), packageFilter)).map((f) => f.file);
|
|
4839
4848
|
const localFiles = new Set(pkgFilePaths);
|
|
4840
4849
|
if (localFiles.size === 0) return { nodes: [], edges: [] };
|
|
4841
4850
|
const filePlaceholders = [...localFiles].map(() => "?").join(",");
|
|
@@ -4911,9 +4920,12 @@ function getFileGraphWithStatement(stmt, packageFilter) {
|
|
|
4911
4920
|
return { nodes: [...fileNodes.values()], edges };
|
|
4912
4921
|
}
|
|
4913
4922
|
function getSymbolGraphWithStatement(stmt, fileFilter) {
|
|
4923
|
+
const indexedFiles = resolveIndexedFiles(stmt, fileFilter);
|
|
4924
|
+
if (indexedFiles.length === 0) return { nodes: [], edges: [] };
|
|
4925
|
+
const filePlaceholders = indexedFiles.map(() => "?").join(",");
|
|
4914
4926
|
const syms = stmt(
|
|
4915
|
-
|
|
4916
|
-
).all(
|
|
4927
|
+
`SELECT id, name, kind, lang, file, line, signature, scope FROM symbols WHERE file IN (${filePlaceholders}) ORDER BY line, id`
|
|
4928
|
+
).all(...indexedFiles);
|
|
4917
4929
|
if (syms.length === 0) return { nodes: [], edges: [] };
|
|
4918
4930
|
const symById = new Map(syms.map((symbol) => [symbol.id, symbol]));
|
|
4919
4931
|
const relatedIds = new Set(syms.map((symbol) => symbol.id));
|
|
@@ -4923,16 +4935,16 @@ function getSymbolGraphWithStatement(stmt, fileFilter) {
|
|
|
4923
4935
|
SELECT r.from_id, r.to_id, r.to_name, r.call_type, r.line
|
|
4924
4936
|
FROM refs r
|
|
4925
4937
|
JOIN symbols s ON s.id = r.from_id
|
|
4926
|
-
WHERE s.file
|
|
4938
|
+
WHERE s.file IN (${filePlaceholders})
|
|
4927
4939
|
UNION
|
|
4928
4940
|
SELECT r.from_id, r.to_id, r.to_name, r.call_type, r.line
|
|
4929
4941
|
FROM refs r
|
|
4930
4942
|
JOIN symbols s ON s.id = r.to_id
|
|
4931
|
-
WHERE s.file
|
|
4943
|
+
WHERE s.file IN (${filePlaceholders})
|
|
4932
4944
|
)
|
|
4933
4945
|
WHERE to_id IS NOT NULL
|
|
4934
4946
|
GROUP BY from_id, to_id, call_type`
|
|
4935
|
-
).all(
|
|
4947
|
+
).all(...indexedFiles, ...indexedFiles);
|
|
4936
4948
|
const edgeMap = /* @__PURE__ */ new Map();
|
|
4937
4949
|
for (const r of refRows) {
|
|
4938
4950
|
if (r.to_id == null) continue;
|
|
@@ -4951,40 +4963,194 @@ function getSymbolGraphWithStatement(stmt, fileFilter) {
|
|
|
4951
4963
|
).all(...missingIds);
|
|
4952
4964
|
for (const s of extras) symById.set(s.id, s);
|
|
4953
4965
|
}
|
|
4954
|
-
const nodes = buildSymbolGraphNodes(
|
|
4966
|
+
const nodes = buildSymbolGraphNodes(
|
|
4967
|
+
symById,
|
|
4968
|
+
relatedIds,
|
|
4969
|
+
new Set(syms.map((symbol) => symbol.file)),
|
|
4970
|
+
readPackageLabeller(stmt)
|
|
4971
|
+
);
|
|
4955
4972
|
return { nodes, edges };
|
|
4956
4973
|
}
|
|
4957
4974
|
|
|
4958
|
-
// src/codebase-index/
|
|
4959
|
-
|
|
4960
|
-
|
|
4961
|
-
|
|
4962
|
-
|
|
4963
|
-
|
|
4964
|
-
|
|
4965
|
-
|
|
4966
|
-
|
|
4967
|
-
|
|
4968
|
-
|
|
4969
|
-
|
|
4970
|
-
|
|
4971
|
-
|
|
4972
|
-
|
|
4975
|
+
// src/codebase-index/vector-search.ts
|
|
4976
|
+
var RRF_K = 60;
|
|
4977
|
+
var VECTOR_DIMENSIONS = 384;
|
|
4978
|
+
var NGRAM_SIZE = 3;
|
|
4979
|
+
function embedText(text) {
|
|
4980
|
+
const vec = new Float32Array(VECTOR_DIMENSIONS);
|
|
4981
|
+
const normalized = text.toLowerCase().trim();
|
|
4982
|
+
if (normalized.length < NGRAM_SIZE) {
|
|
4983
|
+
const padded = ` ${normalized} `.slice(0, Math.max(NGRAM_SIZE, normalized.length + 2));
|
|
4984
|
+
for (let i = 0; i <= padded.length - NGRAM_SIZE; i++) {
|
|
4985
|
+
const ngram = padded.slice(i, i + NGRAM_SIZE);
|
|
4986
|
+
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
4987
|
+
vec[bucket] += 1;
|
|
4988
|
+
}
|
|
4989
|
+
} else {
|
|
4990
|
+
for (let i = 0; i <= normalized.length - NGRAM_SIZE; i++) {
|
|
4991
|
+
const ngram = normalized.slice(i, i + NGRAM_SIZE);
|
|
4992
|
+
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
4993
|
+
vec[bucket] += 1;
|
|
4973
4994
|
}
|
|
4974
|
-
if (!owner && ref.callType === "import") owner = ordered[0];
|
|
4975
|
-
if (!owner || owner.id <= 0) continue;
|
|
4976
|
-
const key = `${owner.id}:${ref.toName}:${ref.callType}:${ref.module ?? ""}`;
|
|
4977
|
-
if (seen.has(key)) continue;
|
|
4978
|
-
seen.add(key);
|
|
4979
|
-
assigned.push({ ...ref, fromId: owner.id });
|
|
4980
4995
|
}
|
|
4981
|
-
|
|
4996
|
+
let norm = 0;
|
|
4997
|
+
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
4998
|
+
norm += vec[i] * vec[i];
|
|
4999
|
+
}
|
|
5000
|
+
norm = Math.sqrt(norm);
|
|
5001
|
+
if (norm > 0) {
|
|
5002
|
+
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
5003
|
+
vec[i] /= norm;
|
|
5004
|
+
}
|
|
5005
|
+
}
|
|
5006
|
+
return vec;
|
|
4982
5007
|
}
|
|
4983
|
-
function
|
|
4984
|
-
|
|
5008
|
+
function hashNgram(str) {
|
|
5009
|
+
let hash = 2166136261;
|
|
5010
|
+
for (let i = 0; i < str.length; i++) {
|
|
5011
|
+
hash ^= str.charCodeAt(i);
|
|
5012
|
+
hash = Math.imul(hash, 16777619);
|
|
5013
|
+
}
|
|
5014
|
+
return hash >>> 0;
|
|
4985
5015
|
}
|
|
4986
|
-
|
|
4987
|
-
|
|
5016
|
+
function cosineSimilarity(a, b) {
|
|
5017
|
+
let dot = 0;
|
|
5018
|
+
const len = Math.min(a.length, b.length);
|
|
5019
|
+
for (let i = 0; i < len; i++) {
|
|
5020
|
+
dot += a[i] * b[i];
|
|
5021
|
+
}
|
|
5022
|
+
return dot;
|
|
5023
|
+
}
|
|
5024
|
+
function encodeVector(vec) {
|
|
5025
|
+
return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
|
|
5026
|
+
}
|
|
5027
|
+
function decodeVector(buf) {
|
|
5028
|
+
const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
5029
|
+
const copy = new Float32Array(buf.byteLength / 4);
|
|
5030
|
+
for (let i = 0; i < copy.length; i++) {
|
|
5031
|
+
copy[i] = view.getFloat32(i * 4, true);
|
|
5032
|
+
}
|
|
5033
|
+
return copy;
|
|
5034
|
+
}
|
|
5035
|
+
function reciprocalRankFusion(bm25Ranks, vectorRanks, k = RRF_K) {
|
|
5036
|
+
const allIds = /* @__PURE__ */ new Set([...bm25Ranks.keys(), ...vectorRanks.keys()]);
|
|
5037
|
+
const scored = [];
|
|
5038
|
+
for (const id of allIds) {
|
|
5039
|
+
const bm25Rank = bm25Ranks.get(id);
|
|
5040
|
+
const vecRank = vectorRanks.get(id);
|
|
5041
|
+
let score = 0;
|
|
5042
|
+
if (bm25Rank !== void 0) score += 1 / (k + bm25Rank);
|
|
5043
|
+
if (vecRank !== void 0) score += 1 / (k + vecRank);
|
|
5044
|
+
scored.push([id, score]);
|
|
5045
|
+
}
|
|
5046
|
+
scored.sort((a, b) => b[1] - a[1]);
|
|
5047
|
+
return scored;
|
|
5048
|
+
}
|
|
5049
|
+
|
|
5050
|
+
// src/codebase-index/writer-mutations.ts
|
|
5051
|
+
function commitBatchWithStatement(stmtFn, maxSqlVars, ftsAvailable, vectorsAvailable, allocateSymbolIds, invalidateIncomingRefsForFiles, resolveRefsForNamesUnsafe2, entries, options = {}) {
|
|
5052
|
+
if (entries.length === 0 && (options.deleteForFiles?.length ?? 0) === 0) {
|
|
5053
|
+
return [];
|
|
5054
|
+
}
|
|
5055
|
+
const affectedNames = /* @__PURE__ */ new Set();
|
|
5056
|
+
for (const entry of entries) {
|
|
5057
|
+
for (const symbol of entry.symbols) affectedNames.add(symbol.name);
|
|
5058
|
+
for (const ref of entry.refs) affectedNames.add(ref.toName);
|
|
5059
|
+
}
|
|
5060
|
+
if (options.deleteForFiles && options.deleteForFiles.length > 0) {
|
|
5061
|
+
const placeholders = options.deleteForFiles.map(() => "?").join(",");
|
|
5062
|
+
for (const name of invalidateIncomingRefsForFiles(options.deleteForFiles)) {
|
|
5063
|
+
affectedNames.add(name);
|
|
5064
|
+
}
|
|
5065
|
+
if (ftsAvailable) {
|
|
5066
|
+
stmtFn(
|
|
5067
|
+
`DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file IN (${placeholders}))`
|
|
5068
|
+
).run(...options.deleteForFiles);
|
|
5069
|
+
}
|
|
5070
|
+
if (vectorsAvailable) {
|
|
5071
|
+
stmtFn(
|
|
5072
|
+
`DELETE FROM symbol_vectors WHERE symbol_id IN (SELECT id FROM symbols WHERE file IN (${placeholders}))`
|
|
5073
|
+
).run(...options.deleteForFiles);
|
|
5074
|
+
}
|
|
5075
|
+
stmtFn(
|
|
5076
|
+
`DELETE FROM refs WHERE from_id IN (SELECT id FROM symbols WHERE file IN (${placeholders}))`
|
|
5077
|
+
).run(...options.deleteForFiles);
|
|
5078
|
+
stmtFn(`DELETE FROM symbols WHERE file IN (${placeholders})`).run(...options.deleteForFiles);
|
|
5079
|
+
}
|
|
5080
|
+
const totalSymbols = entries.reduce((n, e) => n + e.symbols.length, 0);
|
|
5081
|
+
let nextId = allocateSymbolIds(totalSymbols);
|
|
5082
|
+
const allInserted = [];
|
|
5083
|
+
const refsToInsert = [];
|
|
5084
|
+
const bulkSyms = [];
|
|
5085
|
+
const ftsRows = [];
|
|
5086
|
+
const vectorRows = [];
|
|
5087
|
+
for (const entry of entries) {
|
|
5088
|
+
const insertedForEntry = [];
|
|
5089
|
+
for (const s of entry.symbols) {
|
|
5090
|
+
const id = nextId++;
|
|
5091
|
+
bulkSyms.push({
|
|
5092
|
+
id,
|
|
5093
|
+
lang: s.lang,
|
|
5094
|
+
kind: s.kind,
|
|
5095
|
+
name: s.name,
|
|
5096
|
+
file: s.file,
|
|
5097
|
+
line: s.line,
|
|
5098
|
+
col: s.col,
|
|
5099
|
+
signature: s.signature,
|
|
5100
|
+
docComment: s.docComment,
|
|
5101
|
+
scope: s.scope,
|
|
5102
|
+
text: s.text
|
|
5103
|
+
});
|
|
5104
|
+
if (ftsAvailable) {
|
|
5105
|
+
ftsRows.push({
|
|
5106
|
+
id,
|
|
5107
|
+
text: buildIndexableText(s.name, s.signature, s.docComment)
|
|
5108
|
+
});
|
|
5109
|
+
}
|
|
5110
|
+
vectorRows.push({
|
|
5111
|
+
id,
|
|
5112
|
+
vector: encodeVector(
|
|
5113
|
+
embedText(s.text || buildIndexableText(s.name, s.signature, s.docComment))
|
|
5114
|
+
)
|
|
5115
|
+
});
|
|
5116
|
+
const inserted = { ...s, id };
|
|
5117
|
+
allInserted.push(inserted);
|
|
5118
|
+
insertedForEntry.push(inserted);
|
|
5119
|
+
}
|
|
5120
|
+
refsToInsert.push(...assignRefsToSymbols(entry.refs, insertedForEntry));
|
|
5121
|
+
}
|
|
5122
|
+
bulkInsertSymbolsWithStatement((sql) => stmtFn(sql), maxSqlVars, bulkSyms);
|
|
5123
|
+
bulkInsertFtsWithStatement((sql) => stmtFn(sql), maxSqlVars, ftsAvailable, ftsRows);
|
|
5124
|
+
if (vectorsAvailable) {
|
|
5125
|
+
bulkInsertVectorsWithStatement((sql) => stmtFn(sql), maxSqlVars, vectorRows);
|
|
5126
|
+
}
|
|
5127
|
+
bulkInsertRefsWithStatement((sql) => stmtFn(sql), maxSqlVars, refsToInsert);
|
|
5128
|
+
const upsertStmt = stmtFn(
|
|
5129
|
+
`INSERT INTO files(file, lang, mtime_ms, content_hash, symbol_count, last_indexed)
|
|
5130
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
5131
|
+
ON CONFLICT(file) DO UPDATE SET
|
|
5132
|
+
lang = excluded.lang,
|
|
5133
|
+
mtime_ms = excluded.mtime_ms,
|
|
5134
|
+
content_hash = excluded.content_hash,
|
|
5135
|
+
symbol_count = excluded.symbol_count,
|
|
5136
|
+
last_indexed = excluded.last_indexed`
|
|
5137
|
+
);
|
|
5138
|
+
const now = Date.now();
|
|
5139
|
+
for (const entry of entries) {
|
|
5140
|
+
upsertStmt.run(
|
|
5141
|
+
entry.file,
|
|
5142
|
+
entry.lang,
|
|
5143
|
+
entry.mtimeMs,
|
|
5144
|
+
entry.contentHash ?? "",
|
|
5145
|
+
entry.symbolCount,
|
|
5146
|
+
now
|
|
5147
|
+
);
|
|
5148
|
+
}
|
|
5149
|
+
resolveRefsForNamesUnsafe2(affectedNames);
|
|
5150
|
+
return allInserted;
|
|
5151
|
+
}
|
|
5152
|
+
|
|
5153
|
+
// src/codebase-index/writer-pragmas.ts
|
|
4988
5154
|
import { sqliteCachePragmas } from "@wrongstack/core/utils";
|
|
4989
5155
|
function applyIndexStorePragmas(db) {
|
|
4990
5156
|
try {
|
|
@@ -5096,6 +5262,237 @@ var SYMBOL_VECTORS_TABLE_SQL = `
|
|
|
5096
5262
|
);
|
|
5097
5263
|
`;
|
|
5098
5264
|
|
|
5265
|
+
// src/codebase-index/writer-refs.ts
|
|
5266
|
+
var FAMILY_MATCH_SQL = `(
|
|
5267
|
+
sym.lang = refs.lang
|
|
5268
|
+
OR EXISTS (
|
|
5269
|
+
SELECT 1 FROM lang_family lf1
|
|
5270
|
+
JOIN lang_family lf2 ON lf1.family = lf2.family
|
|
5271
|
+
WHERE lf1.lang = sym.lang AND lf2.lang = refs.lang
|
|
5272
|
+
)
|
|
5273
|
+
OR ? IN (
|
|
5274
|
+
SELECT family FROM lang_family WHERE lang = refs.lang
|
|
5275
|
+
)
|
|
5276
|
+
)`;
|
|
5277
|
+
function getNamespaceDeclarationsWithStatement(stmtFn) {
|
|
5278
|
+
return stmtFn(
|
|
5279
|
+
`SELECT name, file FROM symbols WHERE kind = 'namespace' ORDER BY file, id`
|
|
5280
|
+
).all();
|
|
5281
|
+
}
|
|
5282
|
+
function getFilePackagesWithStatement(stmtFn) {
|
|
5283
|
+
const rows = stmtFn("SELECT file, package FROM files WHERE package != ''").all();
|
|
5284
|
+
return new Map(rows.map((row) => [row.file, row.package]));
|
|
5285
|
+
}
|
|
5286
|
+
function getUnresolvedImportsWithStatement(stmtFn, maxSqlVars, onlyFiles) {
|
|
5287
|
+
const base = `SELECT DISTINCT s.file AS fromFile, r.lang AS lang, r.module AS module
|
|
5288
|
+
FROM refs r
|
|
5289
|
+
JOIN symbols s ON s.id = r.from_id
|
|
5290
|
+
WHERE r.call_type = 'import' AND r.module IS NOT NULL`;
|
|
5291
|
+
if (!onlyFiles?.length) {
|
|
5292
|
+
return stmtFn(base).all();
|
|
5293
|
+
}
|
|
5294
|
+
const out = [];
|
|
5295
|
+
for (let i = 0; i < onlyFiles.length; i += maxSqlVars) {
|
|
5296
|
+
const chunk = onlyFiles.slice(i, i + maxSqlVars);
|
|
5297
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
5298
|
+
out.push(
|
|
5299
|
+
...stmtFn(`${base} AND s.file IN (${placeholders})`).all(...chunk)
|
|
5300
|
+
);
|
|
5301
|
+
}
|
|
5302
|
+
return out;
|
|
5303
|
+
}
|
|
5304
|
+
function getAllResolvedRefsWithStatement(stmtFn) {
|
|
5305
|
+
return stmtFn(
|
|
5306
|
+
"SELECT from_id AS fromId, to_id AS toId, call_type AS callType FROM refs WHERE to_id IS NOT NULL"
|
|
5307
|
+
).all();
|
|
5308
|
+
}
|
|
5309
|
+
function getAllImportRefsWithStatement(stmtFn) {
|
|
5310
|
+
return stmtFn(
|
|
5311
|
+
`SELECT s.file AS sourceFile, r.to_name AS toName, r.to_id AS toId,
|
|
5312
|
+
r.call_type AS callType, r.line
|
|
5313
|
+
FROM refs r
|
|
5314
|
+
LEFT JOIN symbols s ON r.from_id = s.id
|
|
5315
|
+
WHERE r.call_type = 'import'
|
|
5316
|
+
ORDER BY r.line`
|
|
5317
|
+
).all();
|
|
5318
|
+
}
|
|
5319
|
+
function resolveRefsWithStatement(stmtFn) {
|
|
5320
|
+
try {
|
|
5321
|
+
const result = stmtFn(
|
|
5322
|
+
`UPDATE refs
|
|
5323
|
+
SET to_id = s.id
|
|
5324
|
+
FROM (
|
|
5325
|
+
SELECT sym.name AS name, lf.family AS family, MIN(sym.id) AS id
|
|
5326
|
+
FROM symbols sym
|
|
5327
|
+
JOIN lang_family lf ON lf.lang = sym.lang
|
|
5328
|
+
GROUP BY sym.name, lf.family
|
|
5329
|
+
UNION ALL
|
|
5330
|
+
SELECT sym.name AS name, '${LANG_FAMILY_WILDCARD}' AS family, MIN(sym.id) AS id
|
|
5331
|
+
FROM symbols sym
|
|
5332
|
+
GROUP BY sym.name
|
|
5333
|
+
) AS s,
|
|
5334
|
+
lang_family AS rf
|
|
5335
|
+
WHERE refs.to_id IS NULL
|
|
5336
|
+
AND refs.to_name IS NOT NULL
|
|
5337
|
+
AND rf.lang = refs.lang
|
|
5338
|
+
AND s.name = refs.to_name
|
|
5339
|
+
AND s.family = rf.family`
|
|
5340
|
+
).run();
|
|
5341
|
+
return result.changes ?? 0;
|
|
5342
|
+
} catch {
|
|
5343
|
+
const result = stmtFn(
|
|
5344
|
+
`UPDATE refs SET to_id = (
|
|
5345
|
+
SELECT sym.id FROM symbols sym
|
|
5346
|
+
WHERE sym.name = refs.to_name AND ${FAMILY_MATCH_SQL}
|
|
5347
|
+
ORDER BY sym.id LIMIT 1
|
|
5348
|
+
) WHERE to_id IS NULL AND to_name IS NOT NULL
|
|
5349
|
+
AND EXISTS (
|
|
5350
|
+
SELECT 1 FROM symbols sym
|
|
5351
|
+
WHERE sym.name = refs.to_name AND ${FAMILY_MATCH_SQL}
|
|
5352
|
+
)`
|
|
5353
|
+
).run(LANG_FAMILY_WILDCARD, LANG_FAMILY_WILDCARD);
|
|
5354
|
+
return result.changes ?? 0;
|
|
5355
|
+
}
|
|
5356
|
+
}
|
|
5357
|
+
function applyImportResolutionsWithStatement(db, stmtFn, runWithRetry, maxSqlVars, resolutions) {
|
|
5358
|
+
if (resolutions.length === 0) return 0;
|
|
5359
|
+
return runWithRetry(() => {
|
|
5360
|
+
db.exec("DROP TABLE IF EXISTS temp.import_resolution");
|
|
5361
|
+
db.exec(
|
|
5362
|
+
`CREATE TEMP TABLE import_resolution (
|
|
5363
|
+
from_file TEXT NOT NULL,
|
|
5364
|
+
lang TEXT NOT NULL,
|
|
5365
|
+
module TEXT NOT NULL,
|
|
5366
|
+
to_file TEXT NOT NULL
|
|
5367
|
+
)`
|
|
5368
|
+
);
|
|
5369
|
+
const chunkSize = Math.max(1, Math.floor(maxSqlVars / 4));
|
|
5370
|
+
for (let i = 0; i < resolutions.length; i += chunkSize) {
|
|
5371
|
+
const chunk = resolutions.slice(i, i + chunkSize);
|
|
5372
|
+
const placeholders = chunk.map(() => "(?, ?, ?, ?)").join(", ");
|
|
5373
|
+
const binds = [];
|
|
5374
|
+
for (const entry of chunk) {
|
|
5375
|
+
binds.push(entry.fromFile, entry.lang, entry.module, entry.toFile);
|
|
5376
|
+
}
|
|
5377
|
+
stmtFn(
|
|
5378
|
+
`INSERT INTO temp.import_resolution(from_file, lang, module, to_file)
|
|
5379
|
+
VALUES ${placeholders}`
|
|
5380
|
+
).run(...binds);
|
|
5381
|
+
}
|
|
5382
|
+
db.exec(
|
|
5383
|
+
`CREATE INDEX IF NOT EXISTS temp.idx_ir
|
|
5384
|
+
ON import_resolution(module, lang, from_file)`
|
|
5385
|
+
);
|
|
5386
|
+
const result = stmtFn(
|
|
5387
|
+
`UPDATE refs
|
|
5388
|
+
SET to_file = (
|
|
5389
|
+
SELECT ir.to_file
|
|
5390
|
+
FROM temp.import_resolution ir
|
|
5391
|
+
JOIN symbols s ON s.id = refs.from_id
|
|
5392
|
+
WHERE ir.module = refs.module
|
|
5393
|
+
AND ir.lang = refs.lang
|
|
5394
|
+
AND ir.from_file = s.file
|
|
5395
|
+
LIMIT 1
|
|
5396
|
+
)
|
|
5397
|
+
WHERE refs.call_type = 'import'
|
|
5398
|
+
AND refs.module IS NOT NULL
|
|
5399
|
+
AND EXISTS (
|
|
5400
|
+
SELECT 1
|
|
5401
|
+
FROM temp.import_resolution ir
|
|
5402
|
+
JOIN symbols s ON s.id = refs.from_id
|
|
5403
|
+
WHERE ir.module = refs.module
|
|
5404
|
+
AND ir.lang = refs.lang
|
|
5405
|
+
AND ir.from_file = s.file
|
|
5406
|
+
)`
|
|
5407
|
+
).run();
|
|
5408
|
+
db.exec("DROP TABLE IF EXISTS temp.import_resolution");
|
|
5409
|
+
return result.changes ?? 0;
|
|
5410
|
+
});
|
|
5411
|
+
}
|
|
5412
|
+
function resolveRefsForNamesUnsafe(stmtFn, maxSqlVars, names) {
|
|
5413
|
+
const list = [...names].filter((name) => name.length > 0);
|
|
5414
|
+
if (list.length === 0) return 0;
|
|
5415
|
+
let total = 0;
|
|
5416
|
+
for (let i = 0; i < list.length; i += maxSqlVars) {
|
|
5417
|
+
const chunk = list.slice(i, i + maxSqlVars);
|
|
5418
|
+
const placeholders = chunk.map(() => "?").join(",");
|
|
5419
|
+
try {
|
|
5420
|
+
const result = stmtFn(
|
|
5421
|
+
`UPDATE refs
|
|
5422
|
+
SET to_id = s.id
|
|
5423
|
+
FROM (
|
|
5424
|
+
SELECT sym.name AS name, lf.family AS family, MIN(sym.id) AS id
|
|
5425
|
+
FROM symbols sym
|
|
5426
|
+
JOIN lang_family lf ON lf.lang = sym.lang
|
|
5427
|
+
WHERE sym.name IN (${placeholders})
|
|
5428
|
+
GROUP BY sym.name, lf.family
|
|
5429
|
+
UNION ALL
|
|
5430
|
+
SELECT sym.name AS name, '${LANG_FAMILY_WILDCARD}' AS family, MIN(sym.id) AS id
|
|
5431
|
+
FROM symbols sym
|
|
5432
|
+
WHERE sym.name IN (${placeholders})
|
|
5433
|
+
GROUP BY sym.name
|
|
5434
|
+
) AS s,
|
|
5435
|
+
lang_family AS rf
|
|
5436
|
+
WHERE refs.to_name IN (${placeholders})
|
|
5437
|
+
AND rf.lang = refs.lang
|
|
5438
|
+
AND s.name = refs.to_name
|
|
5439
|
+
AND s.family = rf.family`
|
|
5440
|
+
).run(...chunk, ...chunk, ...chunk);
|
|
5441
|
+
total += result.changes ?? 0;
|
|
5442
|
+
} catch {
|
|
5443
|
+
const result = stmtFn(
|
|
5444
|
+
`UPDATE refs SET to_id = (
|
|
5445
|
+
SELECT sym.id FROM symbols sym
|
|
5446
|
+
WHERE sym.name = refs.to_name AND ${FAMILY_MATCH_SQL}
|
|
5447
|
+
ORDER BY sym.id LIMIT 1
|
|
5448
|
+
) WHERE refs.to_name IN (${placeholders})
|
|
5449
|
+
AND EXISTS (
|
|
5450
|
+
SELECT 1 FROM symbols sym
|
|
5451
|
+
WHERE sym.name = refs.to_name AND ${FAMILY_MATCH_SQL}
|
|
5452
|
+
)`
|
|
5453
|
+
).run(LANG_FAMILY_WILDCARD, ...chunk, LANG_FAMILY_WILDCARD);
|
|
5454
|
+
total += result.changes ?? 0;
|
|
5455
|
+
}
|
|
5456
|
+
}
|
|
5457
|
+
return total;
|
|
5458
|
+
}
|
|
5459
|
+
|
|
5460
|
+
// src/codebase-index/writer-search.ts
|
|
5461
|
+
import { expectDefined as expectDefined4 } from "@wrongstack/core/utils";
|
|
5462
|
+
|
|
5463
|
+
// src/codebase-index/lsp-kind.ts
|
|
5464
|
+
function lspKindToInternalKind(k) {
|
|
5465
|
+
switch (k) {
|
|
5466
|
+
case 5 /* Class */:
|
|
5467
|
+
return "class";
|
|
5468
|
+
case 6 /* Method */:
|
|
5469
|
+
return "method";
|
|
5470
|
+
case 7 /* Property */:
|
|
5471
|
+
case 8 /* Field */:
|
|
5472
|
+
return "property";
|
|
5473
|
+
case 9 /* Constructor */:
|
|
5474
|
+
return "class";
|
|
5475
|
+
case 10 /* Enum */:
|
|
5476
|
+
return "enum";
|
|
5477
|
+
case 11 /* Interface */:
|
|
5478
|
+
return "interface";
|
|
5479
|
+
case 12 /* Function */:
|
|
5480
|
+
return "function";
|
|
5481
|
+
case 13 /* Variable */:
|
|
5482
|
+
return "var";
|
|
5483
|
+
case 14 /* Constant */:
|
|
5484
|
+
return "const";
|
|
5485
|
+
case 22 /* EnumMember */:
|
|
5486
|
+
return "enum";
|
|
5487
|
+
case 26 /* TypeParameter */:
|
|
5488
|
+
return "type";
|
|
5489
|
+
case 3 /* Namespace */:
|
|
5490
|
+
return "namespace";
|
|
5491
|
+
default:
|
|
5492
|
+
return null;
|
|
5493
|
+
}
|
|
5494
|
+
}
|
|
5495
|
+
|
|
5099
5496
|
// src/codebase-index/writer-search-helpers.ts
|
|
5100
5497
|
var SEARCH_CANDIDATE_SCAN_CAP = 5e3;
|
|
5101
5498
|
function normalizeSearchLimit(limit) {
|
|
@@ -5149,6 +5546,173 @@ function mapWriterSearchRow(row, lspKind, score = 0, snippet = "") {
|
|
|
5149
5546
|
};
|
|
5150
5547
|
}
|
|
5151
5548
|
|
|
5549
|
+
// src/codebase-index/writer-search.ts
|
|
5550
|
+
function searchWithStatement(stmtFn, query, filter, opts) {
|
|
5551
|
+
const built = buildWriterSearchWhere(query, filter);
|
|
5552
|
+
if (built === null) return [];
|
|
5553
|
+
const { where, values } = built;
|
|
5554
|
+
const limit = normalizeSearchLimit(opts?.limit);
|
|
5555
|
+
const limitSql = limit !== void 0 ? " LIMIT ?" : "";
|
|
5556
|
+
const sql = `SELECT id, lang, kind, name, file, line, col, signature, doc_comment FROM symbols ${where}${limitSql}`;
|
|
5557
|
+
const binds = limit !== void 0 ? [...values, limit] : values;
|
|
5558
|
+
const rows = stmtFn(sql).all(...binds);
|
|
5559
|
+
return rows.map((row) => mapWriterSearchRow(row, filter?.lspKind));
|
|
5560
|
+
}
|
|
5561
|
+
function countSearchWithStatement(stmtFn, query, filter) {
|
|
5562
|
+
const built = buildWriterSearchWhere(query, filter);
|
|
5563
|
+
if (built === null) return 0;
|
|
5564
|
+
const row = stmtFn(`SELECT COUNT(*) AS n FROM symbols ${built.where}`).get(
|
|
5565
|
+
...built.values
|
|
5566
|
+
);
|
|
5567
|
+
return Number(row?.n ?? 0);
|
|
5568
|
+
}
|
|
5569
|
+
function searchRankedWithStatement(stmtFn, searchFn, ftsAvailable, vectorsAvailable, getOrBuildBm25, query, filter, limit) {
|
|
5570
|
+
const rawLimit = Number.isFinite(limit) ? Math.trunc(limit) : 20;
|
|
5571
|
+
const safeLimit = Math.max(1, Math.min(rawLimit, 100));
|
|
5572
|
+
const tokens = tokenise(query);
|
|
5573
|
+
if (tokens.length === 0 || !ftsAvailable) {
|
|
5574
|
+
return searchRankedFallbackWithStatement(
|
|
5575
|
+
stmtFn,
|
|
5576
|
+
searchFn,
|
|
5577
|
+
getOrBuildBm25,
|
|
5578
|
+
query,
|
|
5579
|
+
filter,
|
|
5580
|
+
safeLimit
|
|
5581
|
+
);
|
|
5582
|
+
}
|
|
5583
|
+
let effectiveKind = filter?.kind;
|
|
5584
|
+
if (filter?.lspKind !== void 0) {
|
|
5585
|
+
const mapped = lspKindToInternalKind(filter.lspKind);
|
|
5586
|
+
if (mapped === null) return { results: [], total: 0 };
|
|
5587
|
+
effectiveKind = mapped;
|
|
5588
|
+
}
|
|
5589
|
+
const longTokens = tokens.filter((t) => t.length >= 3);
|
|
5590
|
+
const shortTokens = tokens.filter((t) => t.length < 3);
|
|
5591
|
+
if (longTokens.length === 0) {
|
|
5592
|
+
return searchRankedFallbackWithStatement(
|
|
5593
|
+
stmtFn,
|
|
5594
|
+
searchFn,
|
|
5595
|
+
getOrBuildBm25,
|
|
5596
|
+
query,
|
|
5597
|
+
filter,
|
|
5598
|
+
safeLimit
|
|
5599
|
+
);
|
|
5600
|
+
}
|
|
5601
|
+
const match = longTokens.map((t) => `"${t.replaceAll('"', "")}"`).join(" OR ");
|
|
5602
|
+
const conditions = ["symbols_fts MATCH ?"];
|
|
5603
|
+
const values = [match];
|
|
5604
|
+
for (const shortTok of shortTokens) {
|
|
5605
|
+
conditions.push("s.text LIKE ? ESCAPE '\\'");
|
|
5606
|
+
values.push(`%${escapeLike(shortTok)}%`);
|
|
5607
|
+
}
|
|
5608
|
+
if (effectiveKind) {
|
|
5609
|
+
conditions.push("s.kind = ?");
|
|
5610
|
+
values.push(effectiveKind);
|
|
5611
|
+
}
|
|
5612
|
+
if (filter?.lang) {
|
|
5613
|
+
conditions.push("s.lang = ?");
|
|
5614
|
+
values.push(filter.lang);
|
|
5615
|
+
}
|
|
5616
|
+
if (filter?.file) {
|
|
5617
|
+
conditions.push("replace(s.file, '\\', '/') LIKE ? ESCAPE '\\'");
|
|
5618
|
+
values.push(`%${escapeLike(filter.file.replace(/\\/g, "/"))}%`);
|
|
5619
|
+
}
|
|
5620
|
+
const where = conditions.join(" AND ");
|
|
5621
|
+
const countRows = stmtFn(
|
|
5622
|
+
`SELECT COUNT(*) AS n FROM symbols_fts JOIN symbols s ON s.id = symbols_fts.rowid WHERE ${where}`
|
|
5623
|
+
).all(...values);
|
|
5624
|
+
const total = countRows[0] ? Number(countRows[0].n) : 0;
|
|
5625
|
+
if (total === 0) return { results: [], total: 0 };
|
|
5626
|
+
const bm25Rows = stmtFn(
|
|
5627
|
+
`SELECT s.id, s.lang, s.kind, s.name, s.file, s.line, s.col, s.signature, s.doc_comment,
|
|
5628
|
+
-bm25(symbols_fts) AS score,
|
|
5629
|
+
snippet(symbols_fts, 0, '', '', '\u2026', 12) AS snippet
|
|
5630
|
+
FROM symbols_fts JOIN symbols s ON s.id = symbols_fts.rowid
|
|
5631
|
+
WHERE ${where}
|
|
5632
|
+
ORDER BY
|
|
5633
|
+
CASE WHEN lower(s.name) = lower(?) THEN 0
|
|
5634
|
+
WHEN lower(s.name) LIKE lower(?) ESCAPE '\\' THEN 1
|
|
5635
|
+
ELSE 2 END,
|
|
5636
|
+
bm25(symbols_fts), lower(s.name), s.file, s.line, s.col, s.id
|
|
5637
|
+
LIMIT ?`
|
|
5638
|
+
).all(...values, query.trim(), `${escapeLike(query.trim())}%`, safeLimit);
|
|
5639
|
+
if (vectorsAvailable && bm25Rows.length > 0) {
|
|
5640
|
+
const queryVec = embedText(query);
|
|
5641
|
+
const candidateIds = bm25Rows.map((r) => r.id);
|
|
5642
|
+
const placeholders = candidateIds.map(() => "?").join(",");
|
|
5643
|
+
const vecRows = stmtFn(
|
|
5644
|
+
`SELECT sv.symbol_id, sv.vector FROM symbol_vectors sv WHERE sv.symbol_id IN (${placeholders})`
|
|
5645
|
+
).all(...candidateIds);
|
|
5646
|
+
const vecScores = vecRows.map((r) => ({
|
|
5647
|
+
id: r.symbol_id,
|
|
5648
|
+
sim: cosineSimilarity(queryVec, decodeVector(r.vector))
|
|
5649
|
+
})).sort((a, b) => b.sim - a.sim);
|
|
5650
|
+
const bm25Rank = /* @__PURE__ */ new Map();
|
|
5651
|
+
bm25Rows.forEach((r, i) => {
|
|
5652
|
+
bm25Rank.set(r.id, i);
|
|
5653
|
+
});
|
|
5654
|
+
const vecRank = /* @__PURE__ */ new Map();
|
|
5655
|
+
vecScores.forEach((r, i) => {
|
|
5656
|
+
vecRank.set(r.id, i);
|
|
5657
|
+
});
|
|
5658
|
+
const fused = reciprocalRankFusion(bm25Rank, vecRank, 60);
|
|
5659
|
+
const fusedScore = new Map(fused);
|
|
5660
|
+
const sorted = [...bm25Rows].sort(
|
|
5661
|
+
(a, b) => (fusedScore.get(b.id) ?? 0) - (fusedScore.get(a.id) ?? 0)
|
|
5662
|
+
);
|
|
5663
|
+
return {
|
|
5664
|
+
results: sorted.map(
|
|
5665
|
+
(row) => mapWriterSearchRow(row, filter?.lspKind, Math.max(1e-4, row.score), row.snippet)
|
|
5666
|
+
),
|
|
5667
|
+
total
|
|
5668
|
+
};
|
|
5669
|
+
}
|
|
5670
|
+
return {
|
|
5671
|
+
results: bm25Rows.map(
|
|
5672
|
+
(row) => mapWriterSearchRow(row, filter?.lspKind, Math.max(1e-4, row.score), row.snippet)
|
|
5673
|
+
),
|
|
5674
|
+
total
|
|
5675
|
+
};
|
|
5676
|
+
}
|
|
5677
|
+
function searchRankedFallbackWithStatement(stmtFn, searchFn, getOrBuildBm25, query, filter, limit) {
|
|
5678
|
+
if (!query.trim()) {
|
|
5679
|
+
const total2 = countSearchWithStatement(stmtFn, query, filter);
|
|
5680
|
+
if (total2 === 0) return { results: [], total: 0 };
|
|
5681
|
+
return { results: searchFn(query, filter, { limit }), total: total2 };
|
|
5682
|
+
}
|
|
5683
|
+
const total = countSearchWithStatement(stmtFn, query, filter);
|
|
5684
|
+
if (total === 0) return { results: [], total: 0 };
|
|
5685
|
+
const candidates = searchFn(query, filter, {
|
|
5686
|
+
limit: SEARCH_CANDIDATE_SCAN_CAP
|
|
5687
|
+
});
|
|
5688
|
+
if (candidates.length === 0) return { results: [], total: 0 };
|
|
5689
|
+
const candidateById = new Map(candidates.map((c) => [c.id, c]));
|
|
5690
|
+
const bm25 = getOrBuildBm25();
|
|
5691
|
+
const scored = bm25.score(query, (id) => candidateById.has(id));
|
|
5692
|
+
const q = query.trim().toLowerCase();
|
|
5693
|
+
const rank = (id) => {
|
|
5694
|
+
const name = candidateById.get(id)?.name.toLowerCase() ?? "";
|
|
5695
|
+
if (name === q) return 0;
|
|
5696
|
+
if (name.startsWith(q)) return 1;
|
|
5697
|
+
return 2;
|
|
5698
|
+
};
|
|
5699
|
+
scored.sort((a, b) => {
|
|
5700
|
+
const rankDiff = rank(a.id) - rank(b.id);
|
|
5701
|
+
if (rankDiff !== 0) return rankDiff;
|
|
5702
|
+
const scoreDiff = b.score - a.score;
|
|
5703
|
+
if (scoreDiff !== 0) return scoreDiff;
|
|
5704
|
+
const left = expectDefined4(candidateById.get(a.id));
|
|
5705
|
+
const right = expectDefined4(candidateById.get(b.id));
|
|
5706
|
+
return left.name.localeCompare(right.name) || left.file.localeCompare(right.file) || left.line - right.line || left.col - right.col || left.id - right.id;
|
|
5707
|
+
});
|
|
5708
|
+
const qTokens = tokenise(query);
|
|
5709
|
+
const results = scored.slice(0, limit).map(({ id, score }) => {
|
|
5710
|
+
const c = expectDefined4(candidateById.get(id));
|
|
5711
|
+
return { ...c, score, snippet: bm25.extractSnippet(id, qTokens) };
|
|
5712
|
+
});
|
|
5713
|
+
return { results, total };
|
|
5714
|
+
}
|
|
5715
|
+
|
|
5152
5716
|
// src/codebase-index/writer-store-pool.ts
|
|
5153
5717
|
var DEFAULT_MAX_WARM_STORES = 2;
|
|
5154
5718
|
var StorePool = class {
|
|
@@ -5240,68 +5804,14 @@ var DB_FILE2 = "index.db";
|
|
|
5240
5804
|
var MAX_STATEMENT_CACHE = 128;
|
|
5241
5805
|
var IndexStore = class _IndexStore {
|
|
5242
5806
|
db;
|
|
5243
|
-
/**
|
|
5244
|
-
* True while an index run owns one outer SQLite transaction. Individual
|
|
5245
|
-
* writer methods normally protect themselves with BEGIN/COMMIT, but during
|
|
5246
|
-
* a refresh they join this transaction so readers observe either the last
|
|
5247
|
-
* completed index or the next completed index, never an in-between batch.
|
|
5248
|
-
*/
|
|
5249
5807
|
atomicIndexUpdateActive = false;
|
|
5250
5808
|
writeSavepointSequence = 0;
|
|
5251
|
-
/** Absolute path to this project's index directory. */
|
|
5252
5809
|
indexDir;
|
|
5253
|
-
/**
|
|
5254
|
-
* True when the SQLite build provides FTS5 (Node's bundled SQLite does).
|
|
5255
|
-
* When false, ranked search falls back to the LIKE + in-process BM25 path.
|
|
5256
|
-
*/
|
|
5257
5810
|
ftsAvailable = false;
|
|
5258
|
-
/**
|
|
5259
|
-
* Phase 3: true when the `symbol_vectors` table was created successfully.
|
|
5260
|
-
* When false, hybrid search skips the vector pass and falls back to FTS5
|
|
5261
|
-
* (or LIKE) only.
|
|
5262
|
-
*/
|
|
5263
5811
|
vectorsAvailable = false;
|
|
5264
|
-
/**
|
|
5265
|
-
* Cache of prepared statements keyed by their SQL text. `DatabaseSync`
|
|
5266
|
-
* compiles SQL on every `.prepare()` call; for the fixed-SQL methods
|
|
5267
|
-
* (upsertFile, getFileMeta, deleteFile, insertRefs, …) that runs thousands
|
|
5268
|
-
* of times during a full reindex. `StatementSync` objects are reusable
|
|
5269
|
-
* across calls on the same connection, so we compile each distinct SQL once
|
|
5270
|
-
* and reuse it. Cleared in {@link close} when the connection is torn down.
|
|
5271
|
-
*/
|
|
5272
5812
|
stmtCache = /* @__PURE__ */ new Map();
|
|
5273
|
-
/**
|
|
5274
|
-
* Cached full-corpus BM25 index for the FTS5-unavailable fallback path.
|
|
5275
|
-
* Built lazily on the first `searchRankedFallback` call and invalidated
|
|
5276
|
-
* (via `bm25Dirty`) whenever the `symbols` table is mutated. Computing
|
|
5277
|
-
* IDF over the full corpus is also more correct than the old per-query
|
|
5278
|
-
* candidate-subset IDF.
|
|
5279
|
-
*
|
|
5280
|
-
* Cache-lifecycle invariants (single source of truth lives at the
|
|
5281
|
-
* `invalidateBm25()` helper — see its docblock for the "every mutation
|
|
5282
|
-
* MUST call this" contract):
|
|
5283
|
-
* - declaration: this field + `bm25Dirty` (here)
|
|
5284
|
-
* - invalidation: `invalidateBm25()` flips the flag and nulls the cache
|
|
5285
|
-
* - build: `getOrBuildBm25()` rebuilds against current `symbols` rows
|
|
5286
|
-
* - teardown: `close()` resets the flag and nulls the cache
|
|
5287
|
-
*/
|
|
5288
5813
|
bm25Cache = null;
|
|
5289
|
-
// Dirty on open so the first getOrBuildBm25() rebuilds against current rows;
|
|
5290
|
-
// an empty or pre-existing corpus makes a stale IDF table meaningless.
|
|
5291
5814
|
bm25Dirty = true;
|
|
5292
|
-
/**
|
|
5293
|
-
* Prepare-once helper: compile `sql` on first use, reuse thereafter.
|
|
5294
|
-
*
|
|
5295
|
-
* Bounded LRU rather than an open Map. The cache is keyed by SQL TEXT, and
|
|
5296
|
-
* the fallback search builder emits one `text LIKE ?` clause per query token
|
|
5297
|
-
* — so the SQL varies with the token count and a stream of differently-sized
|
|
5298
|
-
* queries grew the cache without limit. Sage's store already bounds its
|
|
5299
|
-
* equivalent at 128 (WS-096).
|
|
5300
|
-
*
|
|
5301
|
-
* Re-inserting on a hit keeps the hot fixed-SQL statements (upsertFile,
|
|
5302
|
-
* insertRefs, …) at the young end, so a burst of one-off search SQL evicts
|
|
5303
|
-
* itself rather than the reindex hot path.
|
|
5304
|
-
*/
|
|
5305
5815
|
stmt(sql) {
|
|
5306
5816
|
const cached = this.stmtCache.get(sql);
|
|
5307
5817
|
if (cached !== void 0) {
|
|
@@ -5328,7 +5838,6 @@ var IndexStore = class _IndexStore {
|
|
|
5328
5838
|
runWithRetry(fn) {
|
|
5329
5839
|
return runSqliteWithRetry(fn);
|
|
5330
5840
|
}
|
|
5331
|
-
/** Run a complete index mutation as one WAL-visible publication. */
|
|
5332
5841
|
async runAtomicIndexUpdate(job) {
|
|
5333
5842
|
if (this.atomicIndexUpdateActive) return job();
|
|
5334
5843
|
this.runWithRetry(() => this.db.exec("BEGIN IMMEDIATE"));
|
|
@@ -5347,11 +5856,6 @@ var IndexStore = class _IndexStore {
|
|
|
5347
5856
|
this.atomicIndexUpdateActive = false;
|
|
5348
5857
|
}
|
|
5349
5858
|
}
|
|
5350
|
-
/**
|
|
5351
|
-
* Begin a method-local transaction. Inside an atomic index publication a
|
|
5352
|
-
* SAVEPOINT preserves the old per-batch rollback boundary, which is needed
|
|
5353
|
-
* when commitBatch falls back to per-file writes after one batch fails.
|
|
5354
|
-
*/
|
|
5355
5859
|
beginWriteTransaction() {
|
|
5356
5860
|
if (this.atomicIndexUpdateActive) {
|
|
5357
5861
|
const savepoint = `index_write_${++this.writeSavepointSequence}`;
|
|
@@ -5373,35 +5877,11 @@ var IndexStore = class _IndexStore {
|
|
|
5373
5877
|
this.db.exec("ROLLBACK");
|
|
5374
5878
|
}
|
|
5375
5879
|
}
|
|
5376
|
-
/**
|
|
5377
|
-
* Mirror the in-process language→family map into SQLite.
|
|
5378
|
-
*
|
|
5379
|
-
* Rewritten on every open rather than only on schema bumps: the mapping is
|
|
5380
|
-
* static lookup data, so a code-side change (a new language, a language
|
|
5381
|
-
* moving families) must take effect without forcing a full reindex.
|
|
5382
|
-
*/
|
|
5383
5880
|
seedLangFamilies() {
|
|
5384
5881
|
const insert = this.stmt("INSERT OR REPLACE INTO lang_family(lang, family) VALUES (?, ?)");
|
|
5385
5882
|
for (const [lang, family] of LANG_FAMILY_ENTRIES) insert.run(lang, family);
|
|
5386
5883
|
insert.run("", LANG_FAMILY_WILDCARD);
|
|
5387
5884
|
}
|
|
5388
|
-
/**
|
|
5389
|
-
* Add any column the current schema expects but the on-disk table lacks.
|
|
5390
|
-
*
|
|
5391
|
-
* `CREATE TABLE IF NOT EXISTS` silently keeps an existing table's old shape,
|
|
5392
|
-
* and the version check above only rebuilds on a version *mismatch*. That
|
|
5393
|
-
* leaves a real gap: several wstack processes share this database, and while
|
|
5394
|
-
* a version upgrade is rolling out one of them may still be running the
|
|
5395
|
-
* previous build. That older process sees the newer version number, drops the
|
|
5396
|
-
* tables, and recreates them from *its* DDL — without the newer columns —
|
|
5397
|
-
* while the metadata row still reads the new version. Every later query for
|
|
5398
|
-
* one of those columns then fails with `no such column`, and no amount of
|
|
5399
|
-
* reindexing fixes it, because the version numbers already agree.
|
|
5400
|
-
*
|
|
5401
|
-
* Repairing column-by-column makes the schema self-healing from any of those
|
|
5402
|
-
* states. Table and column names are compile-time literals from this module,
|
|
5403
|
-
* never user input.
|
|
5404
|
-
*/
|
|
5405
5885
|
repairMissingColumns() {
|
|
5406
5886
|
const expected = [
|
|
5407
5887
|
{
|
|
@@ -5506,27 +5986,8 @@ var IndexStore = class _IndexStore {
|
|
|
5506
5986
|
}
|
|
5507
5987
|
this.ensureNextSymbolIdSeeded();
|
|
5508
5988
|
}
|
|
5509
|
-
// ─── ID allocation & bulk helpers ────────────────────────────────────────────
|
|
5510
5989
|
static NEXT_SYMBOL_ID_KEY = "next_symbol_id";
|
|
5511
|
-
/** Stay under typical SQLite SQLITE_MAX_VARIABLE_NUMBER (often 999). */
|
|
5512
5990
|
static MAX_SQL_VARS = 900;
|
|
5513
|
-
/**
|
|
5514
|
-
* Correlated predicate: the ref in `refs` and the candidate symbol aliased
|
|
5515
|
-
* `sym` belong to the same language family — or the ref carries no language,
|
|
5516
|
-
* in which case the wildcard bind matches everything.
|
|
5517
|
-
*
|
|
5518
|
-
* Each textual occurrence consumes one `?` bind of {@link LANG_FAMILY_WILDCARD}.
|
|
5519
|
-
*/
|
|
5520
|
-
static FAMILY_MATCH_SQL = `(
|
|
5521
|
-
(SELECT family FROM lang_family WHERE lang = refs.lang) = ?
|
|
5522
|
-
OR (SELECT family FROM lang_family WHERE lang = sym.lang)
|
|
5523
|
-
= (SELECT family FROM lang_family WHERE lang = refs.lang)
|
|
5524
|
-
)`;
|
|
5525
|
-
/**
|
|
5526
|
-
* Ensure `metadata.next_symbol_id` exists. Safe to call outside a write
|
|
5527
|
-
* transaction on open; the first concurrent writer under BEGIN IMMEDIATE
|
|
5528
|
-
* re-reads and advances the counter atomically.
|
|
5529
|
-
*/
|
|
5530
5991
|
ensureNextSymbolIdSeeded() {
|
|
5531
5992
|
const existing = this.stmt("SELECT value FROM metadata WHERE key = ?").get(
|
|
5532
5993
|
_IndexStore.NEXT_SYMBOL_ID_KEY
|
|
@@ -5539,10 +6000,6 @@ var IndexStore = class _IndexStore {
|
|
|
5539
6000
|
String(next)
|
|
5540
6001
|
);
|
|
5541
6002
|
}
|
|
5542
|
-
/**
|
|
5543
|
-
* Reserve `count` consecutive symbol ids. MUST run inside BEGIN IMMEDIATE
|
|
5544
|
-
* so concurrent indexers cannot hand out overlapping ranges.
|
|
5545
|
-
*/
|
|
5546
6003
|
allocateSymbolIds(count) {
|
|
5547
6004
|
if (count <= 0) return this.getMaxSymbolId() + 1;
|
|
5548
6005
|
this.ensureNextSymbolIdSeeded();
|
|
@@ -5556,16 +6013,8 @@ var IndexStore = class _IndexStore {
|
|
|
5556
6013
|
);
|
|
5557
6014
|
return start;
|
|
5558
6015
|
}
|
|
5559
|
-
/**
|
|
5560
|
-
* Disconnect inbound refs before their target symbols are replaced and
|
|
5561
|
-
* return the affected names for scoped re-resolution.
|
|
5562
|
-
*
|
|
5563
|
-
* This also repairs a long-standing dangling-id edge case: `refs.to_id` has
|
|
5564
|
-
* no physical FK, so deleting a symbol previously left callers pointing at a
|
|
5565
|
-
* non-existent row.
|
|
5566
|
-
*/
|
|
5567
6016
|
invalidateIncomingRefsForFiles(files) {
|
|
5568
|
-
if (files.length === 0) return
|
|
6017
|
+
if (files.length === 0) return /* @__PURE__ */ new Set();
|
|
5569
6018
|
const placeholders = files.map(() => "?").join(",");
|
|
5570
6019
|
const names = this.stmt(`SELECT DISTINCT name FROM symbols WHERE file IN (${placeholders})`).all(
|
|
5571
6020
|
...files
|
|
@@ -5574,36 +6023,11 @@ var IndexStore = class _IndexStore {
|
|
|
5574
6023
|
`UPDATE refs SET to_id = NULL
|
|
5575
6024
|
WHERE to_id IN (SELECT id FROM symbols WHERE file IN (${placeholders}))`
|
|
5576
6025
|
).run(...files);
|
|
5577
|
-
return names;
|
|
6026
|
+
return new Set(names);
|
|
5578
6027
|
}
|
|
5579
|
-
/** Resolve only refs whose target names may have changed. */
|
|
5580
6028
|
resolveRefsForNamesUnsafe(names) {
|
|
5581
|
-
|
|
5582
|
-
let changes = 0;
|
|
5583
|
-
for (let start = 0; start < unique.length; start += _IndexStore.MAX_SQL_VARS) {
|
|
5584
|
-
const chunk = unique.slice(start, start + _IndexStore.MAX_SQL_VARS);
|
|
5585
|
-
const placeholders = chunk.map(() => "?").join(",");
|
|
5586
|
-
const result = this.stmt(
|
|
5587
|
-
`UPDATE refs
|
|
5588
|
-
SET to_id = (
|
|
5589
|
-
SELECT MIN(sym.id) FROM symbols sym
|
|
5590
|
-
WHERE sym.name = refs.to_name AND ${_IndexStore.FAMILY_MATCH_SQL}
|
|
5591
|
-
)
|
|
5592
|
-
WHERE to_name IN (${placeholders})`
|
|
5593
|
-
).run(LANG_FAMILY_WILDCARD, ...chunk);
|
|
5594
|
-
changes += result.changes ?? 0;
|
|
5595
|
-
}
|
|
5596
|
-
return changes;
|
|
6029
|
+
return resolveRefsForNamesUnsafe((sql) => this.stmt(sql), _IndexStore.MAX_SQL_VARS, names);
|
|
5597
6030
|
}
|
|
5598
|
-
// ─── Symbol CRUD ─────────────────────────────────────────────────────────────
|
|
5599
|
-
/**
|
|
5600
|
-
* Insert symbols, assigning IDs atomically inside `BEGIN IMMEDIATE` /
|
|
5601
|
-
* `COMMIT`. Id ranges come from the `next_symbol_id` metadata counter
|
|
5602
|
-
* (O(1)); multi-row INSERT amortizes bind overhead for large files.
|
|
5603
|
-
*
|
|
5604
|
-
* @returns The symbols array with `id` fields populated so the caller can
|
|
5605
|
-
* use them for refs without re-reading from the DB.
|
|
5606
|
-
*/
|
|
5607
6031
|
insertSymbols(symbols) {
|
|
5608
6032
|
this.invalidateBm25();
|
|
5609
6033
|
return this.runWithRetry(() => {
|
|
@@ -5632,12 +6056,6 @@ var IndexStore = class _IndexStore {
|
|
|
5632
6056
|
if (this.ftsAvailable) {
|
|
5633
6057
|
ftsRows.push({ id, text: buildIndexableText(s.name, s.signature, s.docComment) });
|
|
5634
6058
|
}
|
|
5635
|
-
vectorRows.push({
|
|
5636
|
-
id,
|
|
5637
|
-
vector: encodeVector(
|
|
5638
|
-
embedText(s.text || buildIndexableText(s.name, s.signature, s.docComment))
|
|
5639
|
-
)
|
|
5640
|
-
});
|
|
5641
6059
|
result.push({ ...s, id });
|
|
5642
6060
|
}
|
|
5643
6061
|
bulkInsertSymbolsWithStatement((sql) => this.stmt(sql), _IndexStore.MAX_SQL_VARS, bulk);
|
|
@@ -5687,11 +6105,6 @@ var IndexStore = class _IndexStore {
|
|
|
5687
6105
|
}
|
|
5688
6106
|
});
|
|
5689
6107
|
}
|
|
5690
|
-
/**
|
|
5691
|
-
* Remove every trace of a file (refs, symbols, FTS rows, file meta). Used
|
|
5692
|
-
* when a source file disappears between index runs — previously this only
|
|
5693
|
-
* dropped the `files` row, leaving its symbols orphaned but still searchable.
|
|
5694
|
-
*/
|
|
5695
6108
|
deleteFile(file) {
|
|
5696
6109
|
this.invalidateBm25();
|
|
5697
6110
|
this.runWithRetry(() => {
|
|
@@ -5721,7 +6134,6 @@ var IndexStore = class _IndexStore {
|
|
|
5721
6134
|
}
|
|
5722
6135
|
});
|
|
5723
6136
|
}
|
|
5724
|
-
// ─── File metadata ──────────────────────────────────────────────────────────
|
|
5725
6137
|
upsertFile(meta) {
|
|
5726
6138
|
this.runWithRetry(() => {
|
|
5727
6139
|
this.stmt(
|
|
@@ -5749,8 +6161,6 @@ var IndexStore = class _IndexStore {
|
|
|
5749
6161
|
getAllFileMetas() {
|
|
5750
6162
|
return getAllFileMetasWithStatement((sql) => this.stmt(sql));
|
|
5751
6163
|
}
|
|
5752
|
-
// ─── Project structure & module resolution ──────────────────────────────────
|
|
5753
|
-
/** Store the Code Atlas grouping label for each indexed file. */
|
|
5754
6164
|
setFilePackages(entries) {
|
|
5755
6165
|
if (entries.size === 0) return;
|
|
5756
6166
|
this.runWithRetry(() => {
|
|
@@ -5758,272 +6168,50 @@ var IndexStore = class _IndexStore {
|
|
|
5758
6168
|
for (const [file, label] of entries) update.run(label, file);
|
|
5759
6169
|
});
|
|
5760
6170
|
}
|
|
5761
|
-
/**
|
|
5762
|
-
* Every indexed `namespace`/`module` declaration, for ecosystems whose import
|
|
5763
|
-
* specifiers name a namespace rather than a path (C#, PHP, Elixir, Haskell).
|
|
5764
|
-
* Ordered so the resolver's choice among duplicate declarations is stable.
|
|
5765
|
-
*/
|
|
5766
6171
|
getNamespaceDeclarations() {
|
|
5767
|
-
return this.stmt(
|
|
5768
|
-
`SELECT name, file FROM symbols WHERE kind = 'namespace' ORDER BY file, id`
|
|
5769
|
-
).all();
|
|
6172
|
+
return getNamespaceDeclarationsWithStatement((sql) => this.stmt(sql));
|
|
5770
6173
|
}
|
|
5771
|
-
/** `file → package` for every indexed file that has a label. */
|
|
5772
6174
|
getFilePackages() {
|
|
5773
|
-
|
|
5774
|
-
return new Map(rows.map((row) => [row.file, row.package]));
|
|
6175
|
+
return getFilePackagesWithStatement((sql) => this.stmt(sql));
|
|
5775
6176
|
}
|
|
5776
|
-
/**
|
|
5777
|
-
* Distinct `(fromFile, lang, module)` triples needing module resolution.
|
|
5778
|
-
*
|
|
5779
|
-
* Distinct rather than per-ref because resolution depends only on these three
|
|
5780
|
-
* values: a file importing the same module twenty times resolves it once.
|
|
5781
|
-
*/
|
|
5782
6177
|
getUnresolvedImports(onlyFiles) {
|
|
5783
|
-
|
|
5784
|
-
|
|
5785
|
-
|
|
5786
|
-
|
|
5787
|
-
|
|
5788
|
-
return this.stmt(base).all();
|
|
5789
|
-
}
|
|
5790
|
-
const out = [];
|
|
5791
|
-
for (let i = 0; i < onlyFiles.length; i += _IndexStore.MAX_SQL_VARS) {
|
|
5792
|
-
const chunk = onlyFiles.slice(i, i + _IndexStore.MAX_SQL_VARS);
|
|
5793
|
-
const placeholders = chunk.map(() => "?").join(",");
|
|
5794
|
-
out.push(
|
|
5795
|
-
...this.stmt(`${base} AND s.file IN (${placeholders})`).all(...chunk)
|
|
5796
|
-
);
|
|
5797
|
-
}
|
|
5798
|
-
return out;
|
|
6178
|
+
return getUnresolvedImportsWithStatement(
|
|
6179
|
+
(sql) => this.stmt(sql),
|
|
6180
|
+
_IndexStore.MAX_SQL_VARS,
|
|
6181
|
+
onlyFiles
|
|
6182
|
+
);
|
|
5799
6183
|
}
|
|
5800
|
-
/**
|
|
5801
|
-
* Write resolved import targets back onto `refs.to_file`.
|
|
5802
|
-
*
|
|
5803
|
-
* Applied through a temp table and a single UPDATE: one statement per
|
|
5804
|
-
* resolution would mean thousands of round-trips on a first index.
|
|
5805
|
-
*/
|
|
5806
6184
|
applyImportResolutions(resolutions) {
|
|
5807
|
-
|
|
5808
|
-
|
|
5809
|
-
|
|
5810
|
-
this.
|
|
5811
|
-
|
|
5812
|
-
|
|
5813
|
-
lang TEXT NOT NULL,
|
|
5814
|
-
module TEXT NOT NULL,
|
|
5815
|
-
to_file TEXT NOT NULL
|
|
5816
|
-
)`
|
|
5817
|
-
);
|
|
5818
|
-
const chunkSize = Math.max(1, Math.floor(_IndexStore.MAX_SQL_VARS / 4));
|
|
5819
|
-
for (let i = 0; i < resolutions.length; i += chunkSize) {
|
|
5820
|
-
const chunk = resolutions.slice(i, i + chunkSize);
|
|
5821
|
-
const placeholders = chunk.map(() => "(?, ?, ?, ?)").join(", ");
|
|
5822
|
-
const binds = [];
|
|
5823
|
-
for (const entry of chunk) {
|
|
5824
|
-
binds.push(entry.fromFile, entry.lang, entry.module, entry.toFile);
|
|
5825
|
-
}
|
|
5826
|
-
this.stmt(
|
|
5827
|
-
`INSERT INTO temp.import_resolution(from_file, lang, module, to_file)
|
|
5828
|
-
VALUES ${placeholders}`
|
|
5829
|
-
).run(...binds);
|
|
5830
|
-
}
|
|
5831
|
-
this.db.exec(
|
|
5832
|
-
`CREATE INDEX IF NOT EXISTS temp.idx_ir
|
|
5833
|
-
ON import_resolution(module, lang, from_file)`
|
|
5834
|
-
);
|
|
5835
|
-
const result = this.stmt(
|
|
5836
|
-
`UPDATE refs
|
|
5837
|
-
SET to_file = (
|
|
5838
|
-
SELECT ir.to_file
|
|
5839
|
-
FROM temp.import_resolution ir
|
|
5840
|
-
JOIN symbols s ON s.id = refs.from_id
|
|
5841
|
-
WHERE ir.module = refs.module
|
|
5842
|
-
AND ir.lang = refs.lang
|
|
5843
|
-
AND ir.from_file = s.file
|
|
5844
|
-
LIMIT 1
|
|
5845
|
-
)
|
|
5846
|
-
WHERE refs.call_type = 'import'
|
|
5847
|
-
AND refs.module IS NOT NULL
|
|
5848
|
-
AND EXISTS (
|
|
5849
|
-
SELECT 1
|
|
5850
|
-
FROM temp.import_resolution ir
|
|
5851
|
-
JOIN symbols s ON s.id = refs.from_id
|
|
5852
|
-
WHERE ir.module = refs.module
|
|
5853
|
-
AND ir.lang = refs.lang
|
|
5854
|
-
AND ir.from_file = s.file
|
|
5855
|
-
)`
|
|
5856
|
-
).run();
|
|
5857
|
-
this.db.exec("DROP TABLE IF EXISTS temp.import_resolution");
|
|
5858
|
-
return result.changes ?? 0;
|
|
5859
|
-
});
|
|
5860
|
-
}
|
|
5861
|
-
// ─── Search ──────────────────────────────────────────────────────────────────
|
|
5862
|
-
search(query, filter, opts) {
|
|
5863
|
-
const built = this.buildSearchWhere(query, filter);
|
|
5864
|
-
if (built === null) return [];
|
|
5865
|
-
const { where, values } = built;
|
|
5866
|
-
const limit = normalizeSearchLimit(opts?.limit);
|
|
5867
|
-
const limitSql = limit !== void 0 ? " LIMIT ?" : "";
|
|
5868
|
-
const sql = `SELECT id, lang, kind, name, file, line, col, signature, doc_comment FROM symbols ${where}${limitSql}`;
|
|
5869
|
-
const binds = limit !== void 0 ? [...values, limit] : values;
|
|
5870
|
-
const rows = this.stmt(sql).all(
|
|
5871
|
-
...binds
|
|
6185
|
+
return applyImportResolutionsWithStatement(
|
|
6186
|
+
this.db,
|
|
6187
|
+
(sql) => this.stmt(sql),
|
|
6188
|
+
this.runWithRetry.bind(this),
|
|
6189
|
+
_IndexStore.MAX_SQL_VARS,
|
|
6190
|
+
resolutions
|
|
5872
6191
|
);
|
|
5873
|
-
return rows.map((row) => mapWriterSearchRow(row, filter?.lspKind));
|
|
5874
6192
|
}
|
|
5875
|
-
|
|
5876
|
-
|
|
5877
|
-
return buildWriterSearchWhere(query, filter);
|
|
6193
|
+
search(query, filter, opts) {
|
|
6194
|
+
return searchWithStatement((sql) => this.stmt(sql), query, filter, opts);
|
|
5878
6195
|
}
|
|
5879
6196
|
countSearch(query, filter) {
|
|
5880
|
-
|
|
5881
|
-
if (built === null) return 0;
|
|
5882
|
-
const row = this.stmt(`SELECT COUNT(*) AS n FROM symbols ${built.where}`).get(
|
|
5883
|
-
...built.values
|
|
5884
|
-
);
|
|
5885
|
-
return Number(row?.n ?? 0);
|
|
6197
|
+
return countSearchWithStatement((sql) => this.stmt(sql), query, filter);
|
|
5886
6198
|
}
|
|
5887
|
-
/**
|
|
5888
|
-
* Ranked search — the one-stop query the codebase-search tool and plug-lsp
|
|
5889
|
-
* use. With FTS5 this is a single indexed `MATCH` ranked by SQLite's native
|
|
5890
|
-
* `bm25()` with a built-in `snippet()`; without FTS5 it falls back to the
|
|
5891
|
-
* legacy LIKE scan + in-process BM25 (identical semantics, slower).
|
|
5892
|
-
*
|
|
5893
|
-
* Tokens are matched as prefixes (`"tok"*`), mirroring the old
|
|
5894
|
-
* `LIKE '%tok%'` recall for the common symbol-search shapes ("user" finds
|
|
5895
|
-
* "users", camelCase-split text makes "complex" find "complexOperation").
|
|
5896
|
-
*/
|
|
5897
6199
|
searchRanked(query, filter, limit) {
|
|
5898
|
-
|
|
5899
|
-
|
|
5900
|
-
|
|
5901
|
-
|
|
5902
|
-
|
|
5903
|
-
|
|
5904
|
-
|
|
5905
|
-
|
|
5906
|
-
|
|
5907
|
-
|
|
5908
|
-
effectiveKind = mapped;
|
|
5909
|
-
}
|
|
5910
|
-
const longTokens = tokens.filter((t) => t.length >= 3);
|
|
5911
|
-
const shortTokens = tokens.filter((t) => t.length < 3);
|
|
5912
|
-
if (longTokens.length === 0) {
|
|
5913
|
-
return this.searchRankedFallback(query, filter, safeLimit);
|
|
5914
|
-
}
|
|
5915
|
-
const match = longTokens.map((t) => `"${t.replaceAll('"', "")}"`).join(" OR ");
|
|
5916
|
-
const conditions = ["symbols_fts MATCH ?"];
|
|
5917
|
-
const values = [match];
|
|
5918
|
-
for (const shortTok of shortTokens) {
|
|
5919
|
-
conditions.push("s.text LIKE ? ESCAPE '\\'");
|
|
5920
|
-
values.push(`%${escapeLike(shortTok)}%`);
|
|
5921
|
-
}
|
|
5922
|
-
if (effectiveKind) {
|
|
5923
|
-
conditions.push("s.kind = ?");
|
|
5924
|
-
values.push(effectiveKind);
|
|
5925
|
-
}
|
|
5926
|
-
if (filter?.lang) {
|
|
5927
|
-
conditions.push("s.lang = ?");
|
|
5928
|
-
values.push(filter.lang);
|
|
5929
|
-
}
|
|
5930
|
-
if (filter?.file) {
|
|
5931
|
-
conditions.push("replace(s.file, '\\', '/') LIKE ? ESCAPE '\\'");
|
|
5932
|
-
values.push(`%${escapeLike(filter.file.replace(/\\/g, "/"))}%`);
|
|
5933
|
-
}
|
|
5934
|
-
const where = conditions.join(" AND ");
|
|
5935
|
-
const countRows = this.stmt(
|
|
5936
|
-
`SELECT COUNT(*) AS n FROM symbols_fts JOIN symbols s ON s.id = symbols_fts.rowid WHERE ${where}`
|
|
5937
|
-
).all(...values);
|
|
5938
|
-
const total = countRows[0] ? Number(countRows[0].n) : 0;
|
|
5939
|
-
if (total === 0) return { results: [], total: 0 };
|
|
5940
|
-
const bm25Rows = this.stmt(
|
|
5941
|
-
`SELECT s.id, s.lang, s.kind, s.name, s.file, s.line, s.col, s.signature, s.doc_comment,
|
|
5942
|
-
-bm25(symbols_fts) AS score,
|
|
5943
|
-
snippet(symbols_fts, 0, '', '', '\u2026', 12) AS snippet
|
|
5944
|
-
FROM symbols_fts JOIN symbols s ON s.id = symbols_fts.rowid
|
|
5945
|
-
WHERE ${where}
|
|
5946
|
-
ORDER BY
|
|
5947
|
-
CASE WHEN lower(s.name) = lower(?) THEN 0
|
|
5948
|
-
WHEN lower(s.name) LIKE lower(?) ESCAPE '\\' THEN 1
|
|
5949
|
-
ELSE 2 END,
|
|
5950
|
-
bm25(symbols_fts), lower(s.name), s.file, s.line, s.col, s.id
|
|
5951
|
-
LIMIT ?`
|
|
5952
|
-
).all(...values, query.trim(), `${escapeLike(query.trim())}%`, safeLimit);
|
|
5953
|
-
if (this.vectorsAvailable && bm25Rows.length > 0) {
|
|
5954
|
-
const queryVec = embedText(query);
|
|
5955
|
-
const candidateIds = bm25Rows.map((r) => r.id);
|
|
5956
|
-
const placeholders = candidateIds.map(() => "?").join(",");
|
|
5957
|
-
const vecRows = this.stmt(
|
|
5958
|
-
`SELECT sv.symbol_id, sv.vector FROM symbol_vectors sv WHERE sv.symbol_id IN (${placeholders})`
|
|
5959
|
-
).all(...candidateIds);
|
|
5960
|
-
const vecScores = vecRows.map((r) => ({
|
|
5961
|
-
id: r.symbol_id,
|
|
5962
|
-
sim: cosineSimilarity(queryVec, decodeVector(r.vector))
|
|
5963
|
-
})).sort((a, b) => b.sim - a.sim);
|
|
5964
|
-
const bm25Rank = /* @__PURE__ */ new Map();
|
|
5965
|
-
bm25Rows.forEach((r, i) => {
|
|
5966
|
-
bm25Rank.set(r.id, i);
|
|
5967
|
-
});
|
|
5968
|
-
const vecRank = /* @__PURE__ */ new Map();
|
|
5969
|
-
vecScores.forEach((r, i) => {
|
|
5970
|
-
vecRank.set(r.id, i);
|
|
5971
|
-
});
|
|
5972
|
-
const fused = reciprocalRankFusion(bm25Rank, vecRank, 60);
|
|
5973
|
-
const fusedScore = new Map(fused);
|
|
5974
|
-
const sorted = [...bm25Rows].sort(
|
|
5975
|
-
(a, b) => (fusedScore.get(b.id) ?? 0) - (fusedScore.get(a.id) ?? 0)
|
|
5976
|
-
);
|
|
5977
|
-
return {
|
|
5978
|
-
results: sorted.map(
|
|
5979
|
-
(row) => mapWriterSearchRow(row, filter?.lspKind, Math.max(1e-4, row.score), row.snippet)
|
|
5980
|
-
),
|
|
5981
|
-
total
|
|
5982
|
-
};
|
|
5983
|
-
}
|
|
5984
|
-
return {
|
|
5985
|
-
results: bm25Rows.map(
|
|
5986
|
-
(row) => mapWriterSearchRow(row, filter?.lspKind, Math.max(1e-4, row.score), row.snippet)
|
|
5987
|
-
),
|
|
5988
|
-
total
|
|
5989
|
-
};
|
|
6200
|
+
return searchRankedWithStatement(
|
|
6201
|
+
(sql) => this.stmt(sql),
|
|
6202
|
+
this.search.bind(this),
|
|
6203
|
+
this.ftsAvailable,
|
|
6204
|
+
this.vectorsAvailable,
|
|
6205
|
+
this.getOrBuildBm25.bind(this),
|
|
6206
|
+
query,
|
|
6207
|
+
filter,
|
|
6208
|
+
limit
|
|
6209
|
+
);
|
|
5990
6210
|
}
|
|
5991
|
-
/**
|
|
5992
|
-
* Invalidate the cached BM25 index.
|
|
5993
|
-
*
|
|
5994
|
-
* **Contract: every method that mutates `symbols` MUST call this before
|
|
5995
|
-
* returning.** (`refs` mutations do not affect the BM25 fallback because
|
|
5996
|
-
* the corpus is built from `symbols.text` via `getAllIndexable()` and the
|
|
5997
|
-
* BM25 score is filtered by the LIKE-selected candidate set in
|
|
5998
|
-
* `searchRankedFallback`.) Today the call sites are `repairDrift`,
|
|
5999
|
-
* `insertSymbols`, `deleteSymbolsForFile`, `deleteFile`, `clearAll`, and
|
|
6000
|
-
* `commitBatch`. A future mutation that adds a new write path (e.g.
|
|
6001
|
-
* `renameFile`, `updateSignature`) MUST also call this — otherwise the
|
|
6002
|
-
* FTS5-unavailable fallback will serve stale search results. The
|
|
6003
|
-
* `close()` reset at L1820-1821 tears the cache down on store shutdown,
|
|
6004
|
-
* which is the only legitimate place that flips the flag outside this
|
|
6005
|
-
* helper.
|
|
6006
|
-
*
|
|
6007
|
-
* Called *before* `runWithRetry` on purpose: if the write fails all
|
|
6008
|
-
* retries the flag stays set, forcing a rebuild on the next search rather
|
|
6009
|
-
* than trusting a cache that may not reflect the intended mutation.
|
|
6010
|
-
* Do not move this inside the retry closure.
|
|
6011
|
-
*/
|
|
6012
6211
|
invalidateBm25() {
|
|
6013
6212
|
this.bm25Dirty = true;
|
|
6014
6213
|
this.bm25Cache = null;
|
|
6015
6214
|
}
|
|
6016
|
-
/**
|
|
6017
|
-
* Return the cached full-corpus BM25 index, rebuilding it only when the
|
|
6018
|
-
* symbols table has been mutated since the last build. The full-corpus IDF
|
|
6019
|
-
* is more correct than the old per-query candidate-subset IDF, and the
|
|
6020
|
-
* amortized build cost drops from O(symbols × tokens) per search to once
|
|
6021
|
-
* per write batch.
|
|
6022
|
-
*
|
|
6023
|
-
* Note: the first call after a long idle (or on a freshly opened store)
|
|
6024
|
-
* pays the full corpus rebuild synchronously on the search path. For a
|
|
6025
|
-
* 5 500+ symbol corpus this is a visible one-time latency spike.
|
|
6026
|
-
*/
|
|
6027
6215
|
getOrBuildBm25() {
|
|
6028
6216
|
if (this.bm25Cache && !this.bm25Dirty) return this.bm25Cache;
|
|
6029
6217
|
const docs = this.getAllIndexable();
|
|
@@ -6031,57 +6219,12 @@ var IndexStore = class _IndexStore {
|
|
|
6031
6219
|
this.bm25Dirty = false;
|
|
6032
6220
|
return this.bm25Cache;
|
|
6033
6221
|
}
|
|
6034
|
-
/** Legacy ranked path: LIKE candidates + in-process BM25 + JS snippets. */
|
|
6035
|
-
searchRankedFallback(query, filter, limit) {
|
|
6036
|
-
if (!query.trim()) {
|
|
6037
|
-
const total2 = this.countSearch(query, filter);
|
|
6038
|
-
if (total2 === 0) return { results: [], total: 0 };
|
|
6039
|
-
return { results: this.search(query, filter, { limit }), total: total2 };
|
|
6040
|
-
}
|
|
6041
|
-
const total = this.countSearch(query, filter);
|
|
6042
|
-
if (total === 0) return { results: [], total: 0 };
|
|
6043
|
-
const candidates = this.search(query, filter, { limit: SEARCH_CANDIDATE_SCAN_CAP });
|
|
6044
|
-
if (candidates.length === 0) return { results: [], total: 0 };
|
|
6045
|
-
const candidateById = new Map(candidates.map((c) => [c.id, c]));
|
|
6046
|
-
const bm25 = this.getOrBuildBm25();
|
|
6047
|
-
const scored = bm25.score(query, (id) => candidateById.has(id));
|
|
6048
|
-
const q = query.trim().toLowerCase();
|
|
6049
|
-
const rank = (id) => {
|
|
6050
|
-
const name = candidateById.get(id)?.name.toLowerCase() ?? "";
|
|
6051
|
-
if (name === q) return 0;
|
|
6052
|
-
if (name.startsWith(q)) return 1;
|
|
6053
|
-
return 2;
|
|
6054
|
-
};
|
|
6055
|
-
scored.sort((a, b) => {
|
|
6056
|
-
const rankDiff = rank(a.id) - rank(b.id);
|
|
6057
|
-
if (rankDiff !== 0) return rankDiff;
|
|
6058
|
-
const scoreDiff = b.score - a.score;
|
|
6059
|
-
if (scoreDiff !== 0) return scoreDiff;
|
|
6060
|
-
const left = expectDefined4(candidateById.get(a.id));
|
|
6061
|
-
const right = expectDefined4(candidateById.get(b.id));
|
|
6062
|
-
return left.name.localeCompare(right.name) || left.file.localeCompare(right.file) || left.line - right.line || left.col - right.col || left.id - right.id;
|
|
6063
|
-
});
|
|
6064
|
-
const qTokens = tokenise(query);
|
|
6065
|
-
const results = scored.slice(0, limit).map(({ id, score }) => {
|
|
6066
|
-
const c = expectDefined4(candidateById.get(id));
|
|
6067
|
-
return { ...c, score, snippet: bm25.extractSnippet(id, qTokens) };
|
|
6068
|
-
});
|
|
6069
|
-
return { results, total };
|
|
6070
|
-
}
|
|
6071
6222
|
getAllIndexable() {
|
|
6072
6223
|
return getAllIndexableWithStatement((sql) => this.stmt(sql));
|
|
6073
6224
|
}
|
|
6074
|
-
/**
|
|
6075
|
-
* Largest symbol id currently in the table (0 when empty). New ids must be
|
|
6076
|
-
* allocated from this, NOT from `COUNT(*)`: incremental reindexes delete a
|
|
6077
|
-
* changed file's rows, so the row count drops below the max id and a
|
|
6078
|
-
* count-based id would collide with a surviving row (UNIQUE constraint on
|
|
6079
|
-
* `symbols.id`). Ids may have gaps — that is fine.
|
|
6080
|
-
*/
|
|
6081
6225
|
getMaxSymbolId() {
|
|
6082
6226
|
return getMaxSymbolIdWithStatement((sql) => this.stmt(sql));
|
|
6083
6227
|
}
|
|
6084
|
-
// ─── Stats ───────────────────────────────────────────────────────────────────
|
|
6085
6228
|
getStats() {
|
|
6086
6229
|
return getStatsWithStatement((sql) => this.stmt(sql), this.indexDir);
|
|
6087
6230
|
}
|
|
@@ -6124,11 +6267,6 @@ var IndexStore = class _IndexStore {
|
|
|
6124
6267
|
}
|
|
6125
6268
|
});
|
|
6126
6269
|
}
|
|
6127
|
-
// ─── Ref CRUD ────────────────────────────────────────────────────────────────
|
|
6128
|
-
/**
|
|
6129
|
-
* Insert cross-references for a given source symbol id.
|
|
6130
|
-
* Replaces any existing refs from the same source (idempotent on re-index).
|
|
6131
|
-
*/
|
|
6132
6270
|
insertRefs(fromId, refs) {
|
|
6133
6271
|
this.runWithRetry(() => {
|
|
6134
6272
|
this.stmt("DELETE FROM refs WHERE from_id = ?").run(fromId);
|
|
@@ -6140,167 +6278,36 @@ var IndexStore = class _IndexStore {
|
|
|
6140
6278
|
);
|
|
6141
6279
|
});
|
|
6142
6280
|
}
|
|
6143
|
-
/**
|
|
6144
|
-
* Bulk-insert refs for many source symbols in a single transaction.
|
|
6145
|
-
*
|
|
6146
|
-
* Unlike {@link insertRefs} this does NOT delete per source id — the caller
|
|
6147
|
-
* (the indexer) has already cleared stale refs for the file via
|
|
6148
|
-
* {@link deleteRefsForFile}, so the per-source DELETE would be redundant work
|
|
6149
|
-
* repeated once per symbol. One transaction for the whole file instead of one
|
|
6150
|
-
* per symbol turns an O(symbols) transaction count into O(1).
|
|
6151
|
-
*
|
|
6152
|
-
* Each ref's own {@link Ref.fromId} is used; pass an empty array to no-op.
|
|
6153
|
-
*/
|
|
6154
6281
|
insertRefsBatch(refs) {
|
|
6155
6282
|
if (refs.length === 0) return;
|
|
6156
6283
|
this.runWithRetry(() => {
|
|
6157
6284
|
bulkInsertRefsWithStatement((sql) => this.stmt(sql), _IndexStore.MAX_SQL_VARS, refs);
|
|
6158
6285
|
});
|
|
6159
6286
|
}
|
|
6160
|
-
/**
|
|
6161
|
-
* Commit a batch of file-level symbol/refs/upserts in a single transaction.
|
|
6162
|
-
*
|
|
6163
|
-
* Used by the indexer to amortize SQLite commit overhead across many files.
|
|
6164
|
-
* Before this, the indexer issued one transaction per file (BEGIN IMMEDIATE
|
|
6165
|
-
* for symbols, plus per-file deletes and an upsertFile call), so a 20-file
|
|
6166
|
-
* parallel batch cost ~5+ transactions × 20 files = 100+ commits. With
|
|
6167
|
-
* this entry point we do exactly one BEGIN/COMMIT per parallel batch.
|
|
6168
|
-
*
|
|
6169
|
-
* Each entry must already be a fully-parsed FileSymbols (symbols + refs).
|
|
6170
|
-
* The caller is responsible for the per-file prefix accounting
|
|
6171
|
-
* (refsByLine → flat list with `fromId` populated). `deleteForFiles` lets
|
|
6172
|
-
* the caller clear stale symbols/refs for any files being re-indexed before
|
|
6173
|
-
* the inserts run (required to keep refs → symbols FK invariants).
|
|
6174
|
-
*
|
|
6175
|
-
* Returns the symbols back with their assigned `id` (same shape as
|
|
6176
|
-
* {@link insertSymbols}) so callers can build final per-file results.
|
|
6177
|
-
*/
|
|
6178
6287
|
commitBatch(entries, options = {}) {
|
|
6179
|
-
if (entries.length === 0 && (options.deleteForFiles?.length ?? 0) === 0) {
|
|
6180
|
-
return [];
|
|
6181
|
-
}
|
|
6182
6288
|
this.invalidateBm25();
|
|
6183
6289
|
return this.runWithRetry(() => {
|
|
6184
6290
|
const ownsTransaction = this.beginWriteTransaction();
|
|
6185
6291
|
try {
|
|
6186
|
-
const
|
|
6187
|
-
for (const entry of entries) {
|
|
6188
|
-
for (const symbol of entry.symbols) affectedNames.add(symbol.name);
|
|
6189
|
-
for (const ref of entry.refs) affectedNames.add(ref.toName);
|
|
6190
|
-
}
|
|
6191
|
-
if (options.deleteForFiles && options.deleteForFiles.length > 0) {
|
|
6192
|
-
const placeholders = options.deleteForFiles.map(() => "?").join(",");
|
|
6193
|
-
for (const name of this.invalidateIncomingRefsForFiles(options.deleteForFiles)) {
|
|
6194
|
-
affectedNames.add(name);
|
|
6195
|
-
}
|
|
6196
|
-
if (this.ftsAvailable) {
|
|
6197
|
-
this.stmt(
|
|
6198
|
-
`DELETE FROM symbols_fts WHERE rowid IN (SELECT id FROM symbols WHERE file IN (${placeholders}))`
|
|
6199
|
-
).run(...options.deleteForFiles);
|
|
6200
|
-
}
|
|
6201
|
-
if (this.vectorsAvailable) {
|
|
6202
|
-
this.stmt(
|
|
6203
|
-
`DELETE FROM symbol_vectors WHERE symbol_id IN (SELECT id FROM symbols WHERE file IN (${placeholders}))`
|
|
6204
|
-
).run(...options.deleteForFiles);
|
|
6205
|
-
}
|
|
6206
|
-
this.stmt(
|
|
6207
|
-
`DELETE FROM refs WHERE from_id IN (SELECT id FROM symbols WHERE file IN (${placeholders}))`
|
|
6208
|
-
).run(...options.deleteForFiles);
|
|
6209
|
-
this.stmt(`DELETE FROM symbols WHERE file IN (${placeholders})`).run(
|
|
6210
|
-
...options.deleteForFiles
|
|
6211
|
-
);
|
|
6212
|
-
}
|
|
6213
|
-
const totalSymbols = entries.reduce((n, e) => n + e.symbols.length, 0);
|
|
6214
|
-
let nextId = this.allocateSymbolIds(totalSymbols);
|
|
6215
|
-
const allInserted = [];
|
|
6216
|
-
const refsToInsert = [];
|
|
6217
|
-
const bulkSyms = [];
|
|
6218
|
-
const ftsRows = [];
|
|
6219
|
-
const vectorRows = [];
|
|
6220
|
-
for (const entry of entries) {
|
|
6221
|
-
const insertedForEntry = [];
|
|
6222
|
-
for (const s of entry.symbols) {
|
|
6223
|
-
const id = nextId++;
|
|
6224
|
-
bulkSyms.push({
|
|
6225
|
-
id,
|
|
6226
|
-
lang: s.lang,
|
|
6227
|
-
kind: s.kind,
|
|
6228
|
-
name: s.name,
|
|
6229
|
-
file: s.file,
|
|
6230
|
-
line: s.line,
|
|
6231
|
-
col: s.col,
|
|
6232
|
-
signature: s.signature,
|
|
6233
|
-
docComment: s.docComment,
|
|
6234
|
-
scope: s.scope,
|
|
6235
|
-
text: s.text
|
|
6236
|
-
});
|
|
6237
|
-
if (this.ftsAvailable) {
|
|
6238
|
-
ftsRows.push({
|
|
6239
|
-
id,
|
|
6240
|
-
text: buildIndexableText(s.name, s.signature, s.docComment)
|
|
6241
|
-
});
|
|
6242
|
-
}
|
|
6243
|
-
vectorRows.push({
|
|
6244
|
-
id,
|
|
6245
|
-
vector: encodeVector(
|
|
6246
|
-
embedText(s.text || buildIndexableText(s.name, s.signature, s.docComment))
|
|
6247
|
-
)
|
|
6248
|
-
});
|
|
6249
|
-
const inserted = { ...s, id };
|
|
6250
|
-
allInserted.push(inserted);
|
|
6251
|
-
insertedForEntry.push(inserted);
|
|
6252
|
-
}
|
|
6253
|
-
refsToInsert.push(...assignRefsToSymbols(entry.refs, insertedForEntry));
|
|
6254
|
-
}
|
|
6255
|
-
bulkInsertSymbolsWithStatement((sql) => this.stmt(sql), _IndexStore.MAX_SQL_VARS, bulkSyms);
|
|
6256
|
-
bulkInsertFtsWithStatement(
|
|
6292
|
+
const result = commitBatchWithStatement(
|
|
6257
6293
|
(sql) => this.stmt(sql),
|
|
6258
6294
|
_IndexStore.MAX_SQL_VARS,
|
|
6259
6295
|
this.ftsAvailable,
|
|
6260
|
-
|
|
6296
|
+
this.vectorsAvailable,
|
|
6297
|
+
this.allocateSymbolIds.bind(this),
|
|
6298
|
+
this.invalidateIncomingRefsForFiles.bind(this),
|
|
6299
|
+
this.resolveRefsForNamesUnsafe.bind(this),
|
|
6300
|
+
entries,
|
|
6301
|
+
options
|
|
6261
6302
|
);
|
|
6262
|
-
if (this.vectorsAvailable) {
|
|
6263
|
-
bulkInsertVectorsWithStatement(
|
|
6264
|
-
(sql) => this.stmt(sql),
|
|
6265
|
-
_IndexStore.MAX_SQL_VARS,
|
|
6266
|
-
vectorRows
|
|
6267
|
-
);
|
|
6268
|
-
}
|
|
6269
|
-
bulkInsertRefsWithStatement((sql) => this.stmt(sql), _IndexStore.MAX_SQL_VARS, refsToInsert);
|
|
6270
|
-
const upsertStmt = this.stmt(
|
|
6271
|
-
`INSERT INTO files(file, lang, mtime_ms, content_hash, symbol_count, last_indexed)
|
|
6272
|
-
VALUES (?, ?, ?, ?, ?, ?)
|
|
6273
|
-
ON CONFLICT(file) DO UPDATE SET
|
|
6274
|
-
lang = excluded.lang,
|
|
6275
|
-
mtime_ms = excluded.mtime_ms,
|
|
6276
|
-
content_hash = excluded.content_hash,
|
|
6277
|
-
symbol_count = excluded.symbol_count,
|
|
6278
|
-
last_indexed = excluded.last_indexed`
|
|
6279
|
-
);
|
|
6280
|
-
const now = Date.now();
|
|
6281
|
-
for (const entry of entries) {
|
|
6282
|
-
upsertStmt.run(
|
|
6283
|
-
entry.file,
|
|
6284
|
-
entry.lang,
|
|
6285
|
-
entry.mtimeMs,
|
|
6286
|
-
entry.contentHash ?? "",
|
|
6287
|
-
entry.symbolCount,
|
|
6288
|
-
now
|
|
6289
|
-
);
|
|
6290
|
-
}
|
|
6291
|
-
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
6292
6303
|
this.commitWriteTransaction(ownsTransaction);
|
|
6293
|
-
return
|
|
6304
|
+
return result;
|
|
6294
6305
|
} catch (err) {
|
|
6295
6306
|
this.rollbackWriteTransaction(ownsTransaction);
|
|
6296
6307
|
throw err;
|
|
6297
6308
|
}
|
|
6298
6309
|
});
|
|
6299
6310
|
}
|
|
6300
|
-
/**
|
|
6301
|
-
* Delete all refs whose source symbols are in a given file.
|
|
6302
|
-
* Used when re-indexing a file to clear stale refs.
|
|
6303
|
-
*/
|
|
6304
6311
|
deleteRefsForFile(file) {
|
|
6305
6312
|
this.runWithRetry(() => {
|
|
6306
6313
|
this.stmt("DELETE FROM refs WHERE from_id IN (SELECT id FROM symbols WHERE file = ?)").run(
|
|
@@ -6308,64 +6315,12 @@ var IndexStore = class _IndexStore {
|
|
|
6308
6315
|
);
|
|
6309
6316
|
});
|
|
6310
6317
|
}
|
|
6311
|
-
/**
|
|
6312
|
-
* Resolve `to_name` → `to_id` for all refs that have a name but no id.
|
|
6313
|
-
* Call this after all symbols have been inserted to fill in cross-references.
|
|
6314
|
-
*
|
|
6315
|
-
* A match additionally requires the referencing ref and the target symbol to
|
|
6316
|
-
* be in the same {@link LangFamily}. Without that guard a name match is a
|
|
6317
|
-
* cross-language accident waiting to happen — `main`, `New`, `Parse` and
|
|
6318
|
-
* `Config` are declared in most languages at once, and each collision draws a
|
|
6319
|
-
* Code Atlas edge between files that never reference each other. Refs stored
|
|
6320
|
-
* without a language keep the old global behaviour via the `'*'` wildcard row.
|
|
6321
|
-
*/
|
|
6322
6318
|
resolveRefs() {
|
|
6323
|
-
return this.runWithRetry(() =>
|
|
6324
|
-
try {
|
|
6325
|
-
const result = this.stmt(
|
|
6326
|
-
`UPDATE refs
|
|
6327
|
-
SET to_id = s.id
|
|
6328
|
-
FROM (
|
|
6329
|
-
SELECT sym.name AS name, lf.family AS family, MIN(sym.id) AS id
|
|
6330
|
-
FROM symbols sym
|
|
6331
|
-
JOIN lang_family lf ON lf.lang = sym.lang
|
|
6332
|
-
GROUP BY sym.name, lf.family
|
|
6333
|
-
UNION ALL
|
|
6334
|
-
SELECT sym.name AS name, '${LANG_FAMILY_WILDCARD}' AS family, MIN(sym.id) AS id
|
|
6335
|
-
FROM symbols sym
|
|
6336
|
-
GROUP BY sym.name
|
|
6337
|
-
) AS s,
|
|
6338
|
-
lang_family AS rf
|
|
6339
|
-
WHERE refs.to_id IS NULL
|
|
6340
|
-
AND refs.to_name IS NOT NULL
|
|
6341
|
-
AND rf.lang = refs.lang
|
|
6342
|
-
AND s.name = refs.to_name
|
|
6343
|
-
AND s.family = rf.family`
|
|
6344
|
-
).run();
|
|
6345
|
-
return result.changes ?? 0;
|
|
6346
|
-
} catch {
|
|
6347
|
-
const result = this.stmt(
|
|
6348
|
-
`UPDATE refs SET to_id = (
|
|
6349
|
-
SELECT sym.id FROM symbols sym
|
|
6350
|
-
WHERE sym.name = refs.to_name AND ${_IndexStore.FAMILY_MATCH_SQL}
|
|
6351
|
-
ORDER BY sym.id LIMIT 1
|
|
6352
|
-
) WHERE to_id IS NULL AND to_name IS NOT NULL
|
|
6353
|
-
AND EXISTS (
|
|
6354
|
-
SELECT 1 FROM symbols sym
|
|
6355
|
-
WHERE sym.name = refs.to_name AND ${_IndexStore.FAMILY_MATCH_SQL}
|
|
6356
|
-
)`
|
|
6357
|
-
).run(LANG_FAMILY_WILDCARD, LANG_FAMILY_WILDCARD);
|
|
6358
|
-
return result.changes ?? 0;
|
|
6359
|
-
}
|
|
6360
|
-
});
|
|
6319
|
+
return this.runWithRetry(() => resolveRefsWithStatement((sql) => this.stmt(sql)));
|
|
6361
6320
|
}
|
|
6362
6321
|
resolveRefsForNames(names) {
|
|
6363
6322
|
return this.runWithRetry(() => this.resolveRefsForNamesUnsafe(names));
|
|
6364
6323
|
}
|
|
6365
|
-
/**
|
|
6366
|
-
* Clear symbols/refs for a file and mark it as indexed with zero symbols.
|
|
6367
|
-
* Used by the indexer for empty-parse results so three writes share one txn.
|
|
6368
|
-
*/
|
|
6369
6324
|
replaceEmptyFile(meta) {
|
|
6370
6325
|
this.invalidateBm25();
|
|
6371
6326
|
this.runWithRetry(() => {
|
|
@@ -6411,20 +6366,12 @@ var IndexStore = class _IndexStore {
|
|
|
6411
6366
|
}
|
|
6412
6367
|
});
|
|
6413
6368
|
}
|
|
6414
|
-
/** Best-effort query planner refresh after a large reindex. */
|
|
6415
6369
|
optimize() {
|
|
6416
6370
|
try {
|
|
6417
6371
|
this.db.exec("PRAGMA optimize");
|
|
6418
6372
|
} catch {
|
|
6419
6373
|
}
|
|
6420
6374
|
}
|
|
6421
|
-
/**
|
|
6422
|
-
* Reclaim page churn left by repeated force rebuilds.
|
|
6423
|
-
*
|
|
6424
|
-
* SQLite's DROP/CREATE path makes rebuilds fast but leaves pages on the
|
|
6425
|
-
* freelist. Compact only large, materially sparse databases and only when the
|
|
6426
|
-
* caller is already on a full-index maintenance path.
|
|
6427
|
-
*/
|
|
6428
6375
|
compactIfNeeded(options = {}) {
|
|
6429
6376
|
const minBytes = options.minBytes ?? 256 * 1024 * 1024;
|
|
6430
6377
|
const minFreeRatio = options.minFreeRatio ?? 0.35;
|
|
@@ -6451,115 +6398,44 @@ var IndexStore = class _IndexStore {
|
|
|
6451
6398
|
return false;
|
|
6452
6399
|
}
|
|
6453
6400
|
}
|
|
6454
|
-
/**
|
|
6455
|
-
* Find all symbols that reference the named target symbol (incoming callers).
|
|
6456
|
-
* Accepts a name instead of an id so the agent doesn't need a prior lookup.
|
|
6457
|
-
*/
|
|
6458
6401
|
findIncomingCallsByName(symbolName, file, limit = 100) {
|
|
6459
6402
|
return findIncomingCallsByName((sql) => this.stmt(sql), symbolName, file, limit);
|
|
6460
6403
|
}
|
|
6461
|
-
/**
|
|
6462
|
-
* Find all symbols that the named source symbol references (outgoing callees).
|
|
6463
|
-
* Accepts a name instead of an id so the agent doesn't need a prior lookup.
|
|
6464
|
-
*/
|
|
6465
6404
|
findOutgoingCallsByName(symbolName, file, limit = 100) {
|
|
6466
6405
|
return findOutgoingCallsByName((sql) => this.stmt(sql), symbolName, file, limit);
|
|
6467
6406
|
}
|
|
6468
|
-
/**
|
|
6469
|
-
* Transitive incoming-call tree: all symbols that transitively call the
|
|
6470
|
-
* target, to an unbounded depth (cycle-safe via SQL UNION deduplication).
|
|
6471
|
-
* Used by `codebase-incoming-calls` when the caller wants the full call
|
|
6472
|
-
* chain rather than just direct callers.
|
|
6473
|
-
*/
|
|
6474
6407
|
findTransitiveIncomingCallsByName(symbolName, file, limit = 200) {
|
|
6475
6408
|
return findTransitiveIncomingCallsByName((sql) => this.stmt(sql), symbolName, file, limit);
|
|
6476
6409
|
}
|
|
6477
|
-
/**
|
|
6478
|
-
* Transitive outgoing-call tree: all symbols the target transitively calls.
|
|
6479
|
-
* Used by `codebase-outgoing-calls` when the caller wants the full
|
|
6480
|
-
* dependency chain rather than just direct callees.
|
|
6481
|
-
*/
|
|
6482
6410
|
findTransitiveOutgoingCallsByName(symbolName, file, limit = 200) {
|
|
6483
6411
|
return findTransitiveOutgoingCallsByName((sql) => this.stmt(sql), symbolName, file, limit);
|
|
6484
6412
|
}
|
|
6485
|
-
/**
|
|
6486
|
-
* Compute the set of symbol IDs reachable from the given seed IDs using a
|
|
6487
|
-
* native SQLite recursive CTE. Used by dead-code detection to replace the
|
|
6488
|
-
* in-memory BFS.
|
|
6489
|
-
*/
|
|
6490
6413
|
findReachableSymbolIds(seedIds) {
|
|
6491
6414
|
return findReachableSymbolIds((sql) => this.stmt(sql), seedIds);
|
|
6492
6415
|
}
|
|
6493
|
-
/**
|
|
6494
|
-
* Find all references TO a given symbol (who calls / uses this symbol?).
|
|
6495
|
-
*/
|
|
6496
6416
|
findRefsTo(symbolId) {
|
|
6497
6417
|
return findRefsToWithStatement((sql) => this.stmt(sql), symbolId);
|
|
6498
6418
|
}
|
|
6499
|
-
/**
|
|
6500
|
-
* Find all references FROM a given symbol (what does this symbol call/use?).
|
|
6501
|
-
*/
|
|
6502
6419
|
findRefsFrom(symbolId) {
|
|
6503
6420
|
return findRefsFromWithStatement((sql) => this.stmt(sql), symbolId);
|
|
6504
6421
|
}
|
|
6505
|
-
// ─── CodeMap graph aggregation ──────────────────────────────────────────────
|
|
6506
|
-
/**
|
|
6507
|
-
* Package-level graph: each workspace package is a node; edges are derived
|
|
6508
|
-
* from cross-package symbol references (a symbol in package A references a
|
|
6509
|
-
* symbol resolved in package B). Node metadata includes symbol/file counts.
|
|
6510
|
-
*/
|
|
6511
6422
|
getPackageGraph() {
|
|
6512
6423
|
return getPackageGraphWithStatement((sql) => this.stmt(sql));
|
|
6513
6424
|
}
|
|
6514
|
-
/**
|
|
6515
|
-
* File-level graph for a single package: each file is a node; edges are
|
|
6516
|
-
* derived from cross-file symbol references within the package.
|
|
6517
|
-
*/
|
|
6518
6425
|
getFileGraph(packageFilter) {
|
|
6519
6426
|
return getFileGraphWithStatement((sql) => this.stmt(sql), packageFilter);
|
|
6520
6427
|
}
|
|
6521
|
-
/**
|
|
6522
|
-
* Symbol-level graph for a single file: each symbol is a node; edges are
|
|
6523
|
-
* derived from intra-file and cross-file symbol references (who calls whom).
|
|
6524
|
-
*/
|
|
6525
6428
|
getSymbolGraph(fileFilter) {
|
|
6526
6429
|
return getSymbolGraphWithStatement((sql) => this.stmt(sql), fileFilter);
|
|
6527
6430
|
}
|
|
6528
|
-
/**
|
|
6529
|
-
* Returns every symbol in the index. Used by dead-code analysis to
|
|
6530
|
-
* build the full symbol universe for the reachability scan.
|
|
6531
|
-
*/
|
|
6532
6431
|
getAllSymbols() {
|
|
6533
6432
|
return this.stmt("SELECT id, name, file, kind, line FROM symbols ORDER BY id").all().map((r) => ({ ...r, kind: r.kind }));
|
|
6534
6433
|
}
|
|
6535
|
-
/**
|
|
6536
|
-
* Returns every resolved reference (to_id IS NOT NULL). Used by
|
|
6537
|
-
* dead-code analysis to build the consumer-ship graph. Refs whose
|
|
6538
|
-
* target symbol id is null (unresolved imports) are excluded.
|
|
6539
|
-
*/
|
|
6540
6434
|
getAllResolvedRefs() {
|
|
6541
|
-
return this.stmt(
|
|
6542
|
-
"SELECT from_id AS fromId, to_id AS toId, call_type AS callType FROM refs WHERE to_id IS NOT NULL"
|
|
6543
|
-
).all();
|
|
6435
|
+
return getAllResolvedRefsWithStatement((sql) => this.stmt(sql));
|
|
6544
6436
|
}
|
|
6545
|
-
/**
|
|
6546
|
-
* Returns ALL import refs (including unresolved) with their source-file
|
|
6547
|
-
* path and resolved target id. Used by the dead-code scan's file-level
|
|
6548
|
-
* graph traversal to handle barrel-only entry points where no symbol
|
|
6549
|
-
* carries the ref.
|
|
6550
|
-
*
|
|
6551
|
-
* Refs whose `from_id` doesn't match a known symbol (e.g. pure-barrel
|
|
6552
|
-
* files with no declarations) will have `sourceFile === null`.
|
|
6553
|
-
*/
|
|
6554
6437
|
getAllImportRefs() {
|
|
6555
|
-
return this.stmt(
|
|
6556
|
-
`SELECT s.file AS sourceFile, r.to_name AS toName, r.to_id AS toId,
|
|
6557
|
-
r.call_type AS callType, r.line
|
|
6558
|
-
FROM refs r
|
|
6559
|
-
LEFT JOIN symbols s ON r.from_id = s.id
|
|
6560
|
-
WHERE r.call_type = 'import'
|
|
6561
|
-
ORDER BY r.line`
|
|
6562
|
-
).all();
|
|
6438
|
+
return getAllImportRefsWithStatement((sql) => this.stmt(sql));
|
|
6563
6439
|
}
|
|
6564
6440
|
close() {
|
|
6565
6441
|
this.stmtCache.clear();
|