@wrongstack/tools 0.303.0 → 0.305.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/builtin.d.ts +6 -0
- package/dist/builtin.js +715 -378
- package/dist/codebase-index/codebase-index-tool.d.ts +6 -0
- package/dist/codebase-index/index.js +112 -89
- package/dist/codebase-index/indexer.d.ts +6 -0
- package/dist/codebase-index/project-server.js +112 -89
- package/dist/codebase-index/schema.d.ts +11 -0
- package/dist/codebase-index/worker.js +112 -89
- package/dist/index.d.ts +3 -3
- package/dist/index.js +777 -397
- package/dist/kanban-contract-actions.d.ts +7 -0
- package/dist/kanban-task-inputs.d.ts +15 -2
- package/dist/kanban-tool-schema.d.ts +2 -2
- package/dist/kanban-tool-types.d.ts +32 -11
- package/dist/kanban.js +366 -245
- package/dist/pack.js +714 -378
- package/dist/plan.js +601 -289
- package/dist/read.js +112 -89
- package/dist/session-kanban.d.ts +86 -1
- package/dist/session-kanban.js +216 -46
- package/dist/task.js +601 -289
- package/dist/todo.js +601 -289
- package/dist/tool-tier.js +714 -378
- package/package.json +3 -3
|
@@ -6,6 +6,12 @@ interface CodebaseIndexInput {
|
|
|
6
6
|
}
|
|
7
7
|
interface CodebaseIndexOutput {
|
|
8
8
|
filesIndexed: number;
|
|
9
|
+
fileOutcomes?: {
|
|
10
|
+
parsed: number;
|
|
11
|
+
skipped: number;
|
|
12
|
+
empty: number;
|
|
13
|
+
failed: number;
|
|
14
|
+
} | undefined;
|
|
9
15
|
symbolsIndexed: number;
|
|
10
16
|
langStats: Record<string, number>;
|
|
11
17
|
durationMs: number;
|
|
@@ -4665,7 +4665,7 @@ var IndexStore = class _IndexStore {
|
|
|
4665
4665
|
const ftsSchema = this.stmt(
|
|
4666
4666
|
"SELECT sql FROM sqlite_master WHERE type='table' AND name='symbols_fts'"
|
|
4667
4667
|
).get();
|
|
4668
|
-
if (ftsSchema?.sql
|
|
4668
|
+
if (ftsSchema?.sql?.includes("unicode61")) {
|
|
4669
4669
|
this.db.exec("DROP TABLE IF EXISTS symbols_fts");
|
|
4670
4670
|
}
|
|
4671
4671
|
this.db.exec(SYMBOLS_FTS_SQL);
|
|
@@ -5158,9 +5158,13 @@ var IndexStore = class _IndexStore {
|
|
|
5158
5158
|
sim: cosineSimilarity(queryVec, decodeVector(r.vector))
|
|
5159
5159
|
})).sort((a, b) => b.sim - a.sim);
|
|
5160
5160
|
const bm25Rank = /* @__PURE__ */ new Map();
|
|
5161
|
-
bm25Rows.forEach((r, i) =>
|
|
5161
|
+
bm25Rows.forEach((r, i) => {
|
|
5162
|
+
bm25Rank.set(r.id, i);
|
|
5163
|
+
});
|
|
5162
5164
|
const vecRank = /* @__PURE__ */ new Map();
|
|
5163
|
-
vecScores.forEach((r, i) =>
|
|
5165
|
+
vecScores.forEach((r, i) => {
|
|
5166
|
+
vecRank.set(r.id, i);
|
|
5167
|
+
});
|
|
5164
5168
|
const fused = reciprocalRankFusion(bm25Rank, vecRank, 60);
|
|
5165
5169
|
const fusedScore = new Map(fused);
|
|
5166
5170
|
const sorted = [...bm25Rows].sort(
|
|
@@ -6563,6 +6567,86 @@ import {
|
|
|
6563
6567
|
isFrugalPerf
|
|
6564
6568
|
} from "@wrongstack/core/utils";
|
|
6565
6569
|
|
|
6570
|
+
// src/codebase-index/content-hash.ts
|
|
6571
|
+
var PRIME64_1 = 0x9e3779b185ebca87n;
|
|
6572
|
+
var PRIME64_2 = 0xc2b2ae3d27d4eb4fn;
|
|
6573
|
+
var PRIME64_3 = 0x165667b19e3779f9n;
|
|
6574
|
+
var PRIME64_4 = 0x85ebca77c2b2ae63n;
|
|
6575
|
+
var PRIME64_5 = 0x27d4eb2f165667c5n;
|
|
6576
|
+
var MASK64 = 0xffffffffffffffffn;
|
|
6577
|
+
function mul64(a, b) {
|
|
6578
|
+
return (a & MASK64) * (b & MASK64) & MASK64;
|
|
6579
|
+
}
|
|
6580
|
+
function rotl64(x, n) {
|
|
6581
|
+
const v = x & MASK64;
|
|
6582
|
+
return (v << BigInt(n) | v >> BigInt(64 - n)) & MASK64;
|
|
6583
|
+
}
|
|
6584
|
+
function readU64LE(buf, off) {
|
|
6585
|
+
let v = 0n;
|
|
6586
|
+
for (let i = 7; i >= 0; i--) {
|
|
6587
|
+
v = v << 8n | BigInt(buf[off + i] ?? 0);
|
|
6588
|
+
}
|
|
6589
|
+
return v & MASK64;
|
|
6590
|
+
}
|
|
6591
|
+
function readU32LE(buf, off) {
|
|
6592
|
+
return (BigInt(buf[off] ?? 0) | BigInt(buf[off + 1] ?? 0) << 8n | BigInt(buf[off + 2] ?? 0) << 16n | BigInt(buf[off + 3] ?? 0) << 24n) & MASK64;
|
|
6593
|
+
}
|
|
6594
|
+
function xxh64Round(acc, lane) {
|
|
6595
|
+
return mul64(rotl64(acc + mul64(lane, PRIME64_2) & MASK64, 31), PRIME64_1);
|
|
6596
|
+
}
|
|
6597
|
+
function xxh64MergeRound(acc, val) {
|
|
6598
|
+
return mul64(acc ^ xxh64Round(0n, val), PRIME64_1) + PRIME64_4 & MASK64;
|
|
6599
|
+
}
|
|
6600
|
+
function xxhash64Hex(buf, explicitLen) {
|
|
6601
|
+
const length = explicitLen ?? buf.length;
|
|
6602
|
+
let h;
|
|
6603
|
+
let off = 0;
|
|
6604
|
+
if (length >= 32) {
|
|
6605
|
+
let v1 = PRIME64_1 + PRIME64_2 & MASK64;
|
|
6606
|
+
let v2 = PRIME64_2;
|
|
6607
|
+
let v3 = 0n;
|
|
6608
|
+
let v4 = 0n - PRIME64_1 & MASK64;
|
|
6609
|
+
const end32 = length - 32;
|
|
6610
|
+
while (off <= end32) {
|
|
6611
|
+
v1 = xxh64Round(v1, readU64LE(buf, off));
|
|
6612
|
+
v2 = xxh64Round(v2, readU64LE(buf, off + 8));
|
|
6613
|
+
v3 = xxh64Round(v3, readU64LE(buf, off + 16));
|
|
6614
|
+
v4 = xxh64Round(v4, readU64LE(buf, off + 24));
|
|
6615
|
+
off += 32;
|
|
6616
|
+
}
|
|
6617
|
+
h = rotl64(v1, 1) + rotl64(v2, 7) + rotl64(v3, 12) + rotl64(v4, 18) & MASK64;
|
|
6618
|
+
h = xxh64MergeRound(h, v1);
|
|
6619
|
+
h = xxh64MergeRound(h, v2);
|
|
6620
|
+
h = xxh64MergeRound(h, v3);
|
|
6621
|
+
h = xxh64MergeRound(h, v4);
|
|
6622
|
+
} else {
|
|
6623
|
+
h = PRIME64_5;
|
|
6624
|
+
}
|
|
6625
|
+
h = h + BigInt(length) & MASK64;
|
|
6626
|
+
while (off + 8 <= length) {
|
|
6627
|
+
const k1 = xxh64Round(0n, readU64LE(buf, off));
|
|
6628
|
+
h = mul64(rotl64(h ^ k1, 27), PRIME64_1) + PRIME64_4 & MASK64;
|
|
6629
|
+
off += 8;
|
|
6630
|
+
}
|
|
6631
|
+
if (off + 4 <= length) {
|
|
6632
|
+
h = mul64(rotl64(h ^ mul64(readU32LE(buf, off), PRIME64_1), 23), PRIME64_2) + PRIME64_3 & MASK64;
|
|
6633
|
+
off += 4;
|
|
6634
|
+
}
|
|
6635
|
+
while (off < length) {
|
|
6636
|
+
h = mul64(rotl64(h ^ mul64(BigInt(buf[off] ?? 0), PRIME64_5), 11), PRIME64_1) & MASK64;
|
|
6637
|
+
off += 1;
|
|
6638
|
+
}
|
|
6639
|
+
h = (h ^ h >> 33n) & MASK64;
|
|
6640
|
+
h = mul64(h, PRIME64_2);
|
|
6641
|
+
h = (h ^ h >> 29n) & MASK64;
|
|
6642
|
+
h = mul64(h, PRIME64_3);
|
|
6643
|
+
h = (h ^ h >> 32n) & MASK64;
|
|
6644
|
+
return h.toString(16).padStart(16, "0");
|
|
6645
|
+
}
|
|
6646
|
+
function xxhash64String(content) {
|
|
6647
|
+
return xxhash64Hex(new TextEncoder().encode(content));
|
|
6648
|
+
}
|
|
6649
|
+
|
|
6566
6650
|
// src/codebase-index/gitignore.ts
|
|
6567
6651
|
import * as fs6 from "node:fs/promises";
|
|
6568
6652
|
import * as path6 from "node:path";
|
|
@@ -7290,91 +7374,14 @@ function getParserPool() {
|
|
|
7290
7374
|
return _pool;
|
|
7291
7375
|
}
|
|
7292
7376
|
|
|
7293
|
-
// src/codebase-index/content-hash.ts
|
|
7294
|
-
var PRIME64_1 = 0x9e3779b185ebca87n;
|
|
7295
|
-
var PRIME64_2 = 0xc2b2ae3d27d4eb4fn;
|
|
7296
|
-
var PRIME64_3 = 0x165667b19e3779f9n;
|
|
7297
|
-
var PRIME64_4 = 0x85ebca77c2b2ae63n;
|
|
7298
|
-
var PRIME64_5 = 0x27d4eb2f165667c5n;
|
|
7299
|
-
var MASK64 = 0xffffffffffffffffn;
|
|
7300
|
-
function mul64(a, b) {
|
|
7301
|
-
return (a & MASK64) * (b & MASK64) & MASK64;
|
|
7302
|
-
}
|
|
7303
|
-
function rotl64(x, n) {
|
|
7304
|
-
const v = x & MASK64;
|
|
7305
|
-
return (v << BigInt(n) | v >> BigInt(64 - n)) & MASK64;
|
|
7306
|
-
}
|
|
7307
|
-
function readU64LE(buf, off) {
|
|
7308
|
-
let v = 0n;
|
|
7309
|
-
for (let i = 7; i >= 0; i--) {
|
|
7310
|
-
v = v << 8n | BigInt(buf[off + i] ?? 0);
|
|
7311
|
-
}
|
|
7312
|
-
return v & MASK64;
|
|
7313
|
-
}
|
|
7314
|
-
function readU32LE(buf, off) {
|
|
7315
|
-
return (BigInt(buf[off] ?? 0) | BigInt(buf[off + 1] ?? 0) << 8n | BigInt(buf[off + 2] ?? 0) << 16n | BigInt(buf[off + 3] ?? 0) << 24n) & MASK64;
|
|
7316
|
-
}
|
|
7317
|
-
function xxh64Round(acc, lane) {
|
|
7318
|
-
return mul64(rotl64(acc + mul64(lane, PRIME64_2) & MASK64, 31), PRIME64_1);
|
|
7319
|
-
}
|
|
7320
|
-
function xxh64MergeRound(acc, val) {
|
|
7321
|
-
return mul64(acc ^ xxh64Round(0n, val), PRIME64_1) + PRIME64_4 & MASK64;
|
|
7322
|
-
}
|
|
7323
|
-
function xxhash64Hex(buf, explicitLen) {
|
|
7324
|
-
const length = explicitLen ?? buf.length;
|
|
7325
|
-
let h;
|
|
7326
|
-
let off = 0;
|
|
7327
|
-
if (length >= 32) {
|
|
7328
|
-
let v1 = PRIME64_1 + PRIME64_2 & MASK64;
|
|
7329
|
-
let v2 = PRIME64_2;
|
|
7330
|
-
let v3 = 0n;
|
|
7331
|
-
let v4 = 0n - PRIME64_1 & MASK64;
|
|
7332
|
-
const end32 = length - 32;
|
|
7333
|
-
while (off <= end32) {
|
|
7334
|
-
v1 = xxh64Round(v1, readU64LE(buf, off));
|
|
7335
|
-
v2 = xxh64Round(v2, readU64LE(buf, off + 8));
|
|
7336
|
-
v3 = xxh64Round(v3, readU64LE(buf, off + 16));
|
|
7337
|
-
v4 = xxh64Round(v4, readU64LE(buf, off + 24));
|
|
7338
|
-
off += 32;
|
|
7339
|
-
}
|
|
7340
|
-
h = rotl64(v1, 1) + rotl64(v2, 7) + rotl64(v3, 12) + rotl64(v4, 18) & MASK64;
|
|
7341
|
-
h = xxh64MergeRound(h, v1);
|
|
7342
|
-
h = xxh64MergeRound(h, v2);
|
|
7343
|
-
h = xxh64MergeRound(h, v3);
|
|
7344
|
-
h = xxh64MergeRound(h, v4);
|
|
7345
|
-
} else {
|
|
7346
|
-
h = PRIME64_5;
|
|
7347
|
-
}
|
|
7348
|
-
h = h + BigInt(length) & MASK64;
|
|
7349
|
-
while (off + 8 <= length) {
|
|
7350
|
-
const k1 = xxh64Round(0n, readU64LE(buf, off));
|
|
7351
|
-
h = mul64(rotl64(h ^ k1, 27), PRIME64_1) + PRIME64_4 & MASK64;
|
|
7352
|
-
off += 8;
|
|
7353
|
-
}
|
|
7354
|
-
if (off + 4 <= length) {
|
|
7355
|
-
h = mul64(rotl64(h ^ mul64(readU32LE(buf, off), PRIME64_1), 23), PRIME64_2) + PRIME64_3 & MASK64;
|
|
7356
|
-
off += 4;
|
|
7357
|
-
}
|
|
7358
|
-
while (off < length) {
|
|
7359
|
-
h = mul64(rotl64(h ^ mul64(BigInt(buf[off] ?? 0), PRIME64_5), 11), PRIME64_1) & MASK64;
|
|
7360
|
-
off += 1;
|
|
7361
|
-
}
|
|
7362
|
-
h = (h ^ h >> 33n) & MASK64;
|
|
7363
|
-
h = mul64(h, PRIME64_2);
|
|
7364
|
-
h = (h ^ h >> 29n) & MASK64;
|
|
7365
|
-
h = mul64(h, PRIME64_3);
|
|
7366
|
-
h = (h ^ h >> 32n) & MASK64;
|
|
7367
|
-
return h.toString(16).padStart(16, "0");
|
|
7368
|
-
}
|
|
7369
|
-
function xxhash64String(content) {
|
|
7370
|
-
return xxhash64Hex(new TextEncoder().encode(content));
|
|
7371
|
-
}
|
|
7372
|
-
|
|
7373
7377
|
// src/codebase-index/indexer.ts
|
|
7374
7378
|
var YIELD_EVERY_N = 50;
|
|
7375
7379
|
function resolveParallelBatch() {
|
|
7376
7380
|
return indexParallelBatchSize(availableParallelism());
|
|
7377
7381
|
}
|
|
7382
|
+
function shouldUseParserWorkerPool(candidateFileCount, parseBatchCount) {
|
|
7383
|
+
return !isFrugalPerf() && candidateFileCount >= WORKER_POOL_THRESHOLD && parseBatchCount > 1;
|
|
7384
|
+
}
|
|
7378
7385
|
function yieldEventLoop() {
|
|
7379
7386
|
return new Promise((resolve4) => setImmediate(resolve4));
|
|
7380
7387
|
}
|
|
@@ -7551,11 +7558,7 @@ async function resolveProjectRelations(store, projectRoot, opts) {
|
|
|
7551
7558
|
const structure = await detectModuleRoots(projectRoot, indexedFiles);
|
|
7552
7559
|
if (opts.signal?.aborted) return;
|
|
7553
7560
|
store.setFilePackages(assignPackageLabels(structure, indexedFiles));
|
|
7554
|
-
const resolver = new ModuleResolver(
|
|
7555
|
-
structure,
|
|
7556
|
-
indexedFiles,
|
|
7557
|
-
store.getNamespaceDeclarations()
|
|
7558
|
-
);
|
|
7561
|
+
const resolver = new ModuleResolver(structure, indexedFiles, store.getNamespaceDeclarations());
|
|
7559
7562
|
const pending2 = store.getUnresolvedImports(opts.onlyFiles);
|
|
7560
7563
|
const resolutions = [];
|
|
7561
7564
|
for (const entry of pending2) {
|
|
@@ -7591,6 +7594,10 @@ async function runIndexerWithStore(store, opts) {
|
|
|
7591
7594
|
const errors = [];
|
|
7592
7595
|
const langStats = {};
|
|
7593
7596
|
let filesIndexed = 0;
|
|
7597
|
+
let filesParsed = 0;
|
|
7598
|
+
let filesSkipped = 0;
|
|
7599
|
+
let filesEmpty = 0;
|
|
7600
|
+
let filesFailed = 0;
|
|
7594
7601
|
let symbolsIndexed = 0;
|
|
7595
7602
|
const isGitIgnored = await loadGitignoreMatcher(projectRoot);
|
|
7596
7603
|
let files;
|
|
@@ -7632,12 +7639,14 @@ async function runIndexerWithStore(store, opts) {
|
|
|
7632
7639
|
langStats[meta.lang] = (langStats[meta.lang] ?? 0) + meta.symbolCount;
|
|
7633
7640
|
symbolsIndexed += meta.symbolCount;
|
|
7634
7641
|
filesIndexed++;
|
|
7642
|
+
filesSkipped++;
|
|
7635
7643
|
filesPreSkipped++;
|
|
7636
7644
|
return false;
|
|
7637
7645
|
});
|
|
7638
7646
|
if (filesPreSkipped > 0) opts.onProgress?.(filesPreSkipped, totalFilesForProgress);
|
|
7639
7647
|
}
|
|
7640
7648
|
const parallelBatch = resolveParallelBatch();
|
|
7649
|
+
const parserPoolCandidateCount = files.length;
|
|
7641
7650
|
let filesSinceLastYield = 0;
|
|
7642
7651
|
for (let batchStart = 0; batchStart < files.length; batchStart += parallelBatch) {
|
|
7643
7652
|
const batchEnd = Math.min(batchStart + parallelBatch, files.length);
|
|
@@ -7730,7 +7739,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
7730
7739
|
});
|
|
7731
7740
|
}
|
|
7732
7741
|
if (toParse.length > 0) {
|
|
7733
|
-
let pool = toParse.length
|
|
7742
|
+
let pool = shouldUseParserWorkerPool(parserPoolCandidateCount, toParse.length) ? getParserPool() : null;
|
|
7734
7743
|
if (pool) {
|
|
7735
7744
|
try {
|
|
7736
7745
|
await pool.ensureReady();
|
|
@@ -7780,12 +7789,14 @@ async function runIndexerWithStore(store, opts) {
|
|
|
7780
7789
|
const err = settled.reason;
|
|
7781
7790
|
if (err instanceof Error && isAbortError(err)) throw err;
|
|
7782
7791
|
errors.push(`batch error: ${file}: ${err instanceof Error ? err.message : String(err)}`);
|
|
7792
|
+
filesFailed++;
|
|
7783
7793
|
continue;
|
|
7784
7794
|
}
|
|
7785
7795
|
const result = settled.value;
|
|
7786
7796
|
if (result.error) {
|
|
7787
7797
|
if (result.missing) store.deleteFile(file);
|
|
7788
7798
|
errors.push(`${file}: ${result.error}`);
|
|
7799
|
+
filesFailed++;
|
|
7789
7800
|
continue;
|
|
7790
7801
|
}
|
|
7791
7802
|
const { stat: stat2, lang, parsed } = result;
|
|
@@ -7793,6 +7804,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
7793
7804
|
langStats[lang] = (langStats[lang] ?? 0) + result.skippedMeta.symbolCount;
|
|
7794
7805
|
symbolsIndexed += result.skippedMeta.symbolCount;
|
|
7795
7806
|
filesIndexed++;
|
|
7807
|
+
filesSkipped++;
|
|
7796
7808
|
const stored = existingMeta.get(file);
|
|
7797
7809
|
if (stored && stored.mtimeMs !== result.skippedMeta.mtimeMs) {
|
|
7798
7810
|
store.upsertFile({
|
|
@@ -7817,6 +7829,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
7817
7829
|
contentHash: result.contentHash ?? ""
|
|
7818
7830
|
});
|
|
7819
7831
|
filesIndexed++;
|
|
7832
|
+
filesEmpty++;
|
|
7820
7833
|
}
|
|
7821
7834
|
continue;
|
|
7822
7835
|
}
|
|
@@ -7830,6 +7843,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
7830
7843
|
contentHash: result.contentHash ?? ""
|
|
7831
7844
|
});
|
|
7832
7845
|
filesIndexed++;
|
|
7846
|
+
filesEmpty++;
|
|
7833
7847
|
continue;
|
|
7834
7848
|
}
|
|
7835
7849
|
batchEntries.push({
|
|
@@ -7851,6 +7865,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
7851
7865
|
symbolsIndexed += count;
|
|
7852
7866
|
langStats[entry.lang] = (langStats[entry.lang] ?? 0) + count;
|
|
7853
7867
|
filesIndexed++;
|
|
7868
|
+
filesParsed++;
|
|
7854
7869
|
}
|
|
7855
7870
|
} catch (err) {
|
|
7856
7871
|
const message = err instanceof Error ? err.message : String(err);
|
|
@@ -7863,6 +7878,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
7863
7878
|
symbolsIndexed += symbolsWithIds.length;
|
|
7864
7879
|
langStats[entry.lang] = (langStats[entry.lang] ?? 0) + symbolsWithIds.length;
|
|
7865
7880
|
filesIndexed++;
|
|
7881
|
+
filesParsed++;
|
|
7866
7882
|
if (entry.refs.length > 0 && symbolsWithIds.length > 0) {
|
|
7867
7883
|
const fallbackBatch = assignRefsToSymbols2(entry.refs, symbolsWithIds);
|
|
7868
7884
|
if (fallbackBatch.length > 0) store.insertRefsBatch(fallbackBatch);
|
|
@@ -7880,6 +7896,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
7880
7896
|
contentHash: entry.contentHash
|
|
7881
7897
|
});
|
|
7882
7898
|
} catch (innerErr) {
|
|
7899
|
+
filesFailed++;
|
|
7883
7900
|
errors.push(
|
|
7884
7901
|
`fallback write failed: ${entry.file}: ${innerErr instanceof Error ? innerErr.message : String(innerErr)}`
|
|
7885
7902
|
);
|
|
@@ -7912,6 +7929,12 @@ async function runIndexerWithStore(store, opts) {
|
|
|
7912
7929
|
const durationMs = Date.now() - startMs;
|
|
7913
7930
|
return {
|
|
7914
7931
|
filesIndexed,
|
|
7932
|
+
fileOutcomes: {
|
|
7933
|
+
parsed: filesParsed,
|
|
7934
|
+
skipped: filesSkipped,
|
|
7935
|
+
empty: filesEmpty,
|
|
7936
|
+
failed: filesFailed
|
|
7937
|
+
},
|
|
7915
7938
|
symbolsIndexed,
|
|
7916
7939
|
langStats,
|
|
7917
7940
|
durationMs,
|
|
@@ -6,6 +6,12 @@ import { IndexStore } from './writer.js';
|
|
|
6
6
|
* Re-resolved at the start of each index run so env profile changes apply.
|
|
7
7
|
*/
|
|
8
8
|
export declare function resolveParallelBatch(): number;
|
|
9
|
+
/**
|
|
10
|
+
* Pool startup is amortized across the complete index run, not one outer
|
|
11
|
+
* batch. Balanced batches are capped at 40 files, so comparing the per-batch
|
|
12
|
+
* parse count with the 500-file threshold made the worker path unreachable.
|
|
13
|
+
*/
|
|
14
|
+
export declare function shouldUseParserWorkerPool(candidateFileCount: number, parseBatchCount: number): boolean;
|
|
9
15
|
interface IndexerOptions {
|
|
10
16
|
projectRoot: string;
|
|
11
17
|
files?: string[] | undefined;
|
|
@@ -2726,6 +2726,86 @@ import {
|
|
|
2726
2726
|
isFrugalPerf
|
|
2727
2727
|
} from "@wrongstack/core/utils";
|
|
2728
2728
|
|
|
2729
|
+
// src/codebase-index/content-hash.ts
|
|
2730
|
+
var PRIME64_1 = 0x9e3779b185ebca87n;
|
|
2731
|
+
var PRIME64_2 = 0xc2b2ae3d27d4eb4fn;
|
|
2732
|
+
var PRIME64_3 = 0x165667b19e3779f9n;
|
|
2733
|
+
var PRIME64_4 = 0x85ebca77c2b2ae63n;
|
|
2734
|
+
var PRIME64_5 = 0x27d4eb2f165667c5n;
|
|
2735
|
+
var MASK64 = 0xffffffffffffffffn;
|
|
2736
|
+
function mul64(a, b) {
|
|
2737
|
+
return (a & MASK64) * (b & MASK64) & MASK64;
|
|
2738
|
+
}
|
|
2739
|
+
function rotl64(x, n) {
|
|
2740
|
+
const v = x & MASK64;
|
|
2741
|
+
return (v << BigInt(n) | v >> BigInt(64 - n)) & MASK64;
|
|
2742
|
+
}
|
|
2743
|
+
function readU64LE(buf, off) {
|
|
2744
|
+
let v = 0n;
|
|
2745
|
+
for (let i = 7; i >= 0; i--) {
|
|
2746
|
+
v = v << 8n | BigInt(buf[off + i] ?? 0);
|
|
2747
|
+
}
|
|
2748
|
+
return v & MASK64;
|
|
2749
|
+
}
|
|
2750
|
+
function readU32LE(buf, off) {
|
|
2751
|
+
return (BigInt(buf[off] ?? 0) | BigInt(buf[off + 1] ?? 0) << 8n | BigInt(buf[off + 2] ?? 0) << 16n | BigInt(buf[off + 3] ?? 0) << 24n) & MASK64;
|
|
2752
|
+
}
|
|
2753
|
+
function xxh64Round(acc, lane) {
|
|
2754
|
+
return mul64(rotl64(acc + mul64(lane, PRIME64_2) & MASK64, 31), PRIME64_1);
|
|
2755
|
+
}
|
|
2756
|
+
function xxh64MergeRound(acc, val) {
|
|
2757
|
+
return mul64(acc ^ xxh64Round(0n, val), PRIME64_1) + PRIME64_4 & MASK64;
|
|
2758
|
+
}
|
|
2759
|
+
function xxhash64Hex(buf, explicitLen) {
|
|
2760
|
+
const length = explicitLen ?? buf.length;
|
|
2761
|
+
let h;
|
|
2762
|
+
let off = 0;
|
|
2763
|
+
if (length >= 32) {
|
|
2764
|
+
let v1 = PRIME64_1 + PRIME64_2 & MASK64;
|
|
2765
|
+
let v2 = PRIME64_2;
|
|
2766
|
+
let v3 = 0n;
|
|
2767
|
+
let v4 = 0n - PRIME64_1 & MASK64;
|
|
2768
|
+
const end32 = length - 32;
|
|
2769
|
+
while (off <= end32) {
|
|
2770
|
+
v1 = xxh64Round(v1, readU64LE(buf, off));
|
|
2771
|
+
v2 = xxh64Round(v2, readU64LE(buf, off + 8));
|
|
2772
|
+
v3 = xxh64Round(v3, readU64LE(buf, off + 16));
|
|
2773
|
+
v4 = xxh64Round(v4, readU64LE(buf, off + 24));
|
|
2774
|
+
off += 32;
|
|
2775
|
+
}
|
|
2776
|
+
h = rotl64(v1, 1) + rotl64(v2, 7) + rotl64(v3, 12) + rotl64(v4, 18) & MASK64;
|
|
2777
|
+
h = xxh64MergeRound(h, v1);
|
|
2778
|
+
h = xxh64MergeRound(h, v2);
|
|
2779
|
+
h = xxh64MergeRound(h, v3);
|
|
2780
|
+
h = xxh64MergeRound(h, v4);
|
|
2781
|
+
} else {
|
|
2782
|
+
h = PRIME64_5;
|
|
2783
|
+
}
|
|
2784
|
+
h = h + BigInt(length) & MASK64;
|
|
2785
|
+
while (off + 8 <= length) {
|
|
2786
|
+
const k1 = xxh64Round(0n, readU64LE(buf, off));
|
|
2787
|
+
h = mul64(rotl64(h ^ k1, 27), PRIME64_1) + PRIME64_4 & MASK64;
|
|
2788
|
+
off += 8;
|
|
2789
|
+
}
|
|
2790
|
+
if (off + 4 <= length) {
|
|
2791
|
+
h = mul64(rotl64(h ^ mul64(readU32LE(buf, off), PRIME64_1), 23), PRIME64_2) + PRIME64_3 & MASK64;
|
|
2792
|
+
off += 4;
|
|
2793
|
+
}
|
|
2794
|
+
while (off < length) {
|
|
2795
|
+
h = mul64(rotl64(h ^ mul64(BigInt(buf[off] ?? 0), PRIME64_5), 11), PRIME64_1) & MASK64;
|
|
2796
|
+
off += 1;
|
|
2797
|
+
}
|
|
2798
|
+
h = (h ^ h >> 33n) & MASK64;
|
|
2799
|
+
h = mul64(h, PRIME64_2);
|
|
2800
|
+
h = (h ^ h >> 29n) & MASK64;
|
|
2801
|
+
h = mul64(h, PRIME64_3);
|
|
2802
|
+
h = (h ^ h >> 32n) & MASK64;
|
|
2803
|
+
return h.toString(16).padStart(16, "0");
|
|
2804
|
+
}
|
|
2805
|
+
function xxhash64String(content) {
|
|
2806
|
+
return xxhash64Hex(new TextEncoder().encode(content));
|
|
2807
|
+
}
|
|
2808
|
+
|
|
2729
2809
|
// src/codebase-index/gitignore.ts
|
|
2730
2810
|
import * as fs from "node:fs/promises";
|
|
2731
2811
|
import * as path from "node:path";
|
|
@@ -3757,86 +3837,6 @@ function getParserPool() {
|
|
|
3757
3837
|
return _pool;
|
|
3758
3838
|
}
|
|
3759
3839
|
|
|
3760
|
-
// src/codebase-index/content-hash.ts
|
|
3761
|
-
var PRIME64_1 = 0x9e3779b185ebca87n;
|
|
3762
|
-
var PRIME64_2 = 0xc2b2ae3d27d4eb4fn;
|
|
3763
|
-
var PRIME64_3 = 0x165667b19e3779f9n;
|
|
3764
|
-
var PRIME64_4 = 0x85ebca77c2b2ae63n;
|
|
3765
|
-
var PRIME64_5 = 0x27d4eb2f165667c5n;
|
|
3766
|
-
var MASK64 = 0xffffffffffffffffn;
|
|
3767
|
-
function mul64(a, b) {
|
|
3768
|
-
return (a & MASK64) * (b & MASK64) & MASK64;
|
|
3769
|
-
}
|
|
3770
|
-
function rotl64(x, n) {
|
|
3771
|
-
const v = x & MASK64;
|
|
3772
|
-
return (v << BigInt(n) | v >> BigInt(64 - n)) & MASK64;
|
|
3773
|
-
}
|
|
3774
|
-
function readU64LE(buf, off) {
|
|
3775
|
-
let v = 0n;
|
|
3776
|
-
for (let i = 7; i >= 0; i--) {
|
|
3777
|
-
v = v << 8n | BigInt(buf[off + i] ?? 0);
|
|
3778
|
-
}
|
|
3779
|
-
return v & MASK64;
|
|
3780
|
-
}
|
|
3781
|
-
function readU32LE(buf, off) {
|
|
3782
|
-
return (BigInt(buf[off] ?? 0) | BigInt(buf[off + 1] ?? 0) << 8n | BigInt(buf[off + 2] ?? 0) << 16n | BigInt(buf[off + 3] ?? 0) << 24n) & MASK64;
|
|
3783
|
-
}
|
|
3784
|
-
function xxh64Round(acc, lane) {
|
|
3785
|
-
return mul64(rotl64(acc + mul64(lane, PRIME64_2) & MASK64, 31), PRIME64_1);
|
|
3786
|
-
}
|
|
3787
|
-
function xxh64MergeRound(acc, val) {
|
|
3788
|
-
return mul64(acc ^ xxh64Round(0n, val), PRIME64_1) + PRIME64_4 & MASK64;
|
|
3789
|
-
}
|
|
3790
|
-
function xxhash64Hex(buf, explicitLen) {
|
|
3791
|
-
const length = explicitLen ?? buf.length;
|
|
3792
|
-
let h;
|
|
3793
|
-
let off = 0;
|
|
3794
|
-
if (length >= 32) {
|
|
3795
|
-
let v1 = PRIME64_1 + PRIME64_2 & MASK64;
|
|
3796
|
-
let v2 = PRIME64_2;
|
|
3797
|
-
let v3 = 0n;
|
|
3798
|
-
let v4 = 0n - PRIME64_1 & MASK64;
|
|
3799
|
-
const end32 = length - 32;
|
|
3800
|
-
while (off <= end32) {
|
|
3801
|
-
v1 = xxh64Round(v1, readU64LE(buf, off));
|
|
3802
|
-
v2 = xxh64Round(v2, readU64LE(buf, off + 8));
|
|
3803
|
-
v3 = xxh64Round(v3, readU64LE(buf, off + 16));
|
|
3804
|
-
v4 = xxh64Round(v4, readU64LE(buf, off + 24));
|
|
3805
|
-
off += 32;
|
|
3806
|
-
}
|
|
3807
|
-
h = rotl64(v1, 1) + rotl64(v2, 7) + rotl64(v3, 12) + rotl64(v4, 18) & MASK64;
|
|
3808
|
-
h = xxh64MergeRound(h, v1);
|
|
3809
|
-
h = xxh64MergeRound(h, v2);
|
|
3810
|
-
h = xxh64MergeRound(h, v3);
|
|
3811
|
-
h = xxh64MergeRound(h, v4);
|
|
3812
|
-
} else {
|
|
3813
|
-
h = PRIME64_5;
|
|
3814
|
-
}
|
|
3815
|
-
h = h + BigInt(length) & MASK64;
|
|
3816
|
-
while (off + 8 <= length) {
|
|
3817
|
-
const k1 = xxh64Round(0n, readU64LE(buf, off));
|
|
3818
|
-
h = mul64(rotl64(h ^ k1, 27), PRIME64_1) + PRIME64_4 & MASK64;
|
|
3819
|
-
off += 8;
|
|
3820
|
-
}
|
|
3821
|
-
if (off + 4 <= length) {
|
|
3822
|
-
h = mul64(rotl64(h ^ mul64(readU32LE(buf, off), PRIME64_1), 23), PRIME64_2) + PRIME64_3 & MASK64;
|
|
3823
|
-
off += 4;
|
|
3824
|
-
}
|
|
3825
|
-
while (off < length) {
|
|
3826
|
-
h = mul64(rotl64(h ^ mul64(BigInt(buf[off] ?? 0), PRIME64_5), 11), PRIME64_1) & MASK64;
|
|
3827
|
-
off += 1;
|
|
3828
|
-
}
|
|
3829
|
-
h = (h ^ h >> 33n) & MASK64;
|
|
3830
|
-
h = mul64(h, PRIME64_2);
|
|
3831
|
-
h = (h ^ h >> 29n) & MASK64;
|
|
3832
|
-
h = mul64(h, PRIME64_3);
|
|
3833
|
-
h = (h ^ h >> 32n) & MASK64;
|
|
3834
|
-
return h.toString(16).padStart(16, "0");
|
|
3835
|
-
}
|
|
3836
|
-
function xxhash64String(content) {
|
|
3837
|
-
return xxhash64Hex(new TextEncoder().encode(content));
|
|
3838
|
-
}
|
|
3839
|
-
|
|
3840
3840
|
// src/codebase-index/writer.ts
|
|
3841
3841
|
import { expectDefined as expectDefined4 } from "@wrongstack/core/utils";
|
|
3842
3842
|
import * as fs8 from "node:fs";
|
|
@@ -5426,7 +5426,7 @@ var IndexStore = class _IndexStore {
|
|
|
5426
5426
|
const ftsSchema = this.stmt(
|
|
5427
5427
|
"SELECT sql FROM sqlite_master WHERE type='table' AND name='symbols_fts'"
|
|
5428
5428
|
).get();
|
|
5429
|
-
if (ftsSchema?.sql
|
|
5429
|
+
if (ftsSchema?.sql?.includes("unicode61")) {
|
|
5430
5430
|
this.db.exec("DROP TABLE IF EXISTS symbols_fts");
|
|
5431
5431
|
}
|
|
5432
5432
|
this.db.exec(SYMBOLS_FTS_SQL);
|
|
@@ -5919,9 +5919,13 @@ var IndexStore = class _IndexStore {
|
|
|
5919
5919
|
sim: cosineSimilarity(queryVec, decodeVector(r.vector))
|
|
5920
5920
|
})).sort((a, b) => b.sim - a.sim);
|
|
5921
5921
|
const bm25Rank = /* @__PURE__ */ new Map();
|
|
5922
|
-
bm25Rows.forEach((r, i) =>
|
|
5922
|
+
bm25Rows.forEach((r, i) => {
|
|
5923
|
+
bm25Rank.set(r.id, i);
|
|
5924
|
+
});
|
|
5923
5925
|
const vecRank = /* @__PURE__ */ new Map();
|
|
5924
|
-
vecScores.forEach((r, i) =>
|
|
5926
|
+
vecScores.forEach((r, i) => {
|
|
5927
|
+
vecRank.set(r.id, i);
|
|
5928
|
+
});
|
|
5925
5929
|
const fused = reciprocalRankFusion(bm25Rank, vecRank, 60);
|
|
5926
5930
|
const fusedScore = new Map(fused);
|
|
5927
5931
|
const sorted = [...bm25Rows].sort(
|
|
@@ -6531,6 +6535,9 @@ var YIELD_EVERY_N = 50;
|
|
|
6531
6535
|
function resolveParallelBatch() {
|
|
6532
6536
|
return indexParallelBatchSize(availableParallelism());
|
|
6533
6537
|
}
|
|
6538
|
+
function shouldUseParserWorkerPool(candidateFileCount, parseBatchCount) {
|
|
6539
|
+
return !isFrugalPerf() && candidateFileCount >= WORKER_POOL_THRESHOLD && parseBatchCount > 1;
|
|
6540
|
+
}
|
|
6534
6541
|
function yieldEventLoop() {
|
|
6535
6542
|
return new Promise((resolve4) => setImmediate(resolve4));
|
|
6536
6543
|
}
|
|
@@ -6707,11 +6714,7 @@ async function resolveProjectRelations(store, projectRoot2, opts) {
|
|
|
6707
6714
|
const structure = await detectModuleRoots(projectRoot2, indexedFiles);
|
|
6708
6715
|
if (opts.signal?.aborted) return;
|
|
6709
6716
|
store.setFilePackages(assignPackageLabels(structure, indexedFiles));
|
|
6710
|
-
const resolver = new ModuleResolver(
|
|
6711
|
-
structure,
|
|
6712
|
-
indexedFiles,
|
|
6713
|
-
store.getNamespaceDeclarations()
|
|
6714
|
-
);
|
|
6717
|
+
const resolver = new ModuleResolver(structure, indexedFiles, store.getNamespaceDeclarations());
|
|
6715
6718
|
const pending = store.getUnresolvedImports(opts.onlyFiles);
|
|
6716
6719
|
const resolutions = [];
|
|
6717
6720
|
for (const entry of pending) {
|
|
@@ -6736,6 +6739,10 @@ async function runIndexerWithStore(store, opts) {
|
|
|
6736
6739
|
const errors = [];
|
|
6737
6740
|
const langStats = {};
|
|
6738
6741
|
let filesIndexed = 0;
|
|
6742
|
+
let filesParsed = 0;
|
|
6743
|
+
let filesSkipped = 0;
|
|
6744
|
+
let filesEmpty = 0;
|
|
6745
|
+
let filesFailed = 0;
|
|
6739
6746
|
let symbolsIndexed = 0;
|
|
6740
6747
|
const isGitIgnored = await loadGitignoreMatcher(projectRoot2);
|
|
6741
6748
|
let files;
|
|
@@ -6777,12 +6784,14 @@ async function runIndexerWithStore(store, opts) {
|
|
|
6777
6784
|
langStats[meta.lang] = (langStats[meta.lang] ?? 0) + meta.symbolCount;
|
|
6778
6785
|
symbolsIndexed += meta.symbolCount;
|
|
6779
6786
|
filesIndexed++;
|
|
6787
|
+
filesSkipped++;
|
|
6780
6788
|
filesPreSkipped++;
|
|
6781
6789
|
return false;
|
|
6782
6790
|
});
|
|
6783
6791
|
if (filesPreSkipped > 0) opts.onProgress?.(filesPreSkipped, totalFilesForProgress);
|
|
6784
6792
|
}
|
|
6785
6793
|
const parallelBatch = resolveParallelBatch();
|
|
6794
|
+
const parserPoolCandidateCount = files.length;
|
|
6786
6795
|
let filesSinceLastYield = 0;
|
|
6787
6796
|
for (let batchStart = 0; batchStart < files.length; batchStart += parallelBatch) {
|
|
6788
6797
|
const batchEnd = Math.min(batchStart + parallelBatch, files.length);
|
|
@@ -6875,7 +6884,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
6875
6884
|
});
|
|
6876
6885
|
}
|
|
6877
6886
|
if (toParse.length > 0) {
|
|
6878
|
-
let pool = toParse.length
|
|
6887
|
+
let pool = shouldUseParserWorkerPool(parserPoolCandidateCount, toParse.length) ? getParserPool() : null;
|
|
6879
6888
|
if (pool) {
|
|
6880
6889
|
try {
|
|
6881
6890
|
await pool.ensureReady();
|
|
@@ -6925,12 +6934,14 @@ async function runIndexerWithStore(store, opts) {
|
|
|
6925
6934
|
const err = settled.reason;
|
|
6926
6935
|
if (err instanceof Error && isAbortError(err)) throw err;
|
|
6927
6936
|
errors.push(`batch error: ${file}: ${err instanceof Error ? err.message : String(err)}`);
|
|
6937
|
+
filesFailed++;
|
|
6928
6938
|
continue;
|
|
6929
6939
|
}
|
|
6930
6940
|
const result = settled.value;
|
|
6931
6941
|
if (result.error) {
|
|
6932
6942
|
if (result.missing) store.deleteFile(file);
|
|
6933
6943
|
errors.push(`${file}: ${result.error}`);
|
|
6944
|
+
filesFailed++;
|
|
6934
6945
|
continue;
|
|
6935
6946
|
}
|
|
6936
6947
|
const { stat: stat2, lang, parsed: parsed2 } = result;
|
|
@@ -6938,6 +6949,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
6938
6949
|
langStats[lang] = (langStats[lang] ?? 0) + result.skippedMeta.symbolCount;
|
|
6939
6950
|
symbolsIndexed += result.skippedMeta.symbolCount;
|
|
6940
6951
|
filesIndexed++;
|
|
6952
|
+
filesSkipped++;
|
|
6941
6953
|
const stored = existingMeta.get(file);
|
|
6942
6954
|
if (stored && stored.mtimeMs !== result.skippedMeta.mtimeMs) {
|
|
6943
6955
|
store.upsertFile({
|
|
@@ -6962,6 +6974,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
6962
6974
|
contentHash: result.contentHash ?? ""
|
|
6963
6975
|
});
|
|
6964
6976
|
filesIndexed++;
|
|
6977
|
+
filesEmpty++;
|
|
6965
6978
|
}
|
|
6966
6979
|
continue;
|
|
6967
6980
|
}
|
|
@@ -6975,6 +6988,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
6975
6988
|
contentHash: result.contentHash ?? ""
|
|
6976
6989
|
});
|
|
6977
6990
|
filesIndexed++;
|
|
6991
|
+
filesEmpty++;
|
|
6978
6992
|
continue;
|
|
6979
6993
|
}
|
|
6980
6994
|
batchEntries.push({
|
|
@@ -6996,6 +7010,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
6996
7010
|
symbolsIndexed += count;
|
|
6997
7011
|
langStats[entry.lang] = (langStats[entry.lang] ?? 0) + count;
|
|
6998
7012
|
filesIndexed++;
|
|
7013
|
+
filesParsed++;
|
|
6999
7014
|
}
|
|
7000
7015
|
} catch (err) {
|
|
7001
7016
|
const message = err instanceof Error ? err.message : String(err);
|
|
@@ -7008,6 +7023,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
7008
7023
|
symbolsIndexed += symbolsWithIds.length;
|
|
7009
7024
|
langStats[entry.lang] = (langStats[entry.lang] ?? 0) + symbolsWithIds.length;
|
|
7010
7025
|
filesIndexed++;
|
|
7026
|
+
filesParsed++;
|
|
7011
7027
|
if (entry.refs.length > 0 && symbolsWithIds.length > 0) {
|
|
7012
7028
|
const fallbackBatch = assignRefsToSymbols2(entry.refs, symbolsWithIds);
|
|
7013
7029
|
if (fallbackBatch.length > 0) store.insertRefsBatch(fallbackBatch);
|
|
@@ -7025,6 +7041,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
7025
7041
|
contentHash: entry.contentHash
|
|
7026
7042
|
});
|
|
7027
7043
|
} catch (innerErr) {
|
|
7044
|
+
filesFailed++;
|
|
7028
7045
|
errors.push(
|
|
7029
7046
|
`fallback write failed: ${entry.file}: ${innerErr instanceof Error ? innerErr.message : String(innerErr)}`
|
|
7030
7047
|
);
|
|
@@ -7057,6 +7074,12 @@ async function runIndexerWithStore(store, opts) {
|
|
|
7057
7074
|
const durationMs = Date.now() - startMs;
|
|
7058
7075
|
return {
|
|
7059
7076
|
filesIndexed,
|
|
7077
|
+
fileOutcomes: {
|
|
7078
|
+
parsed: filesParsed,
|
|
7079
|
+
skipped: filesSkipped,
|
|
7080
|
+
empty: filesEmpty,
|
|
7081
|
+
failed: filesFailed
|
|
7082
|
+
},
|
|
7060
7083
|
symbolsIndexed,
|
|
7061
7084
|
langStats,
|
|
7062
7085
|
durationMs,
|
|
@@ -76,6 +76,17 @@ export interface SearchResult {
|
|
|
76
76
|
/** Result of a full reindex. */
|
|
77
77
|
export interface IndexResult {
|
|
78
78
|
filesIndexed: number;
|
|
79
|
+
/** Outcome detail for this run. Optional for compatibility with older project daemons. */
|
|
80
|
+
fileOutcomes?: {
|
|
81
|
+
/** Files parsed and committed with one or more symbols. */
|
|
82
|
+
parsed: number;
|
|
83
|
+
/** Files reused from trusted metadata or an unchanged content hash. */
|
|
84
|
+
skipped: number;
|
|
85
|
+
/** Files successfully represented in the index with zero symbols. */
|
|
86
|
+
empty: number;
|
|
87
|
+
/** Files that could not be read, parsed, or committed. */
|
|
88
|
+
failed: number;
|
|
89
|
+
} | undefined;
|
|
79
90
|
symbolsIndexed: number;
|
|
80
91
|
langStats: Record<SymbolLang, number>;
|
|
81
92
|
durationMs: number;
|