@wrongstack/tools 0.306.2 → 0.306.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/builtin.js +241 -137
- package/dist/codebase-index/index.js +241 -137
- package/dist/codebase-index/project-server.js +224 -114
- package/dist/codebase-index/worker.js +210 -107
- package/dist/codebase-index/writer.d.ts +18 -0
- package/dist/index.js +241 -137
- package/dist/pack.js +241 -137
- package/dist/read.js +210 -110
- package/dist/tool-tier.js +241 -137
- package/package.json +4 -4
|
@@ -3045,6 +3045,85 @@ function runSqliteWithRetry(fn) {
|
|
|
3045
3045
|
throw lastError;
|
|
3046
3046
|
}
|
|
3047
3047
|
|
|
3048
|
+
// src/codebase-index/vector-search.ts
|
|
3049
|
+
var RRF_K = 60;
|
|
3050
|
+
var VECTOR_DIMENSIONS = 384;
|
|
3051
|
+
var NGRAM_SIZE = 3;
|
|
3052
|
+
function embedText(text) {
|
|
3053
|
+
const vec = new Float32Array(VECTOR_DIMENSIONS);
|
|
3054
|
+
const normalized = text.toLowerCase().trim();
|
|
3055
|
+
if (normalized.length < NGRAM_SIZE) {
|
|
3056
|
+
const padded = ` ${normalized} `.slice(0, Math.max(NGRAM_SIZE, normalized.length + 2));
|
|
3057
|
+
for (let i = 0; i <= padded.length - NGRAM_SIZE; i++) {
|
|
3058
|
+
const ngram = padded.slice(i, i + NGRAM_SIZE);
|
|
3059
|
+
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
3060
|
+
vec[bucket] += 1;
|
|
3061
|
+
}
|
|
3062
|
+
} else {
|
|
3063
|
+
for (let i = 0; i <= normalized.length - NGRAM_SIZE; i++) {
|
|
3064
|
+
const ngram = normalized.slice(i, i + NGRAM_SIZE);
|
|
3065
|
+
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
3066
|
+
vec[bucket] += 1;
|
|
3067
|
+
}
|
|
3068
|
+
}
|
|
3069
|
+
let norm = 0;
|
|
3070
|
+
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
3071
|
+
norm += vec[i] * vec[i];
|
|
3072
|
+
}
|
|
3073
|
+
norm = Math.sqrt(norm);
|
|
3074
|
+
if (norm > 0) {
|
|
3075
|
+
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
3076
|
+
vec[i] /= norm;
|
|
3077
|
+
}
|
|
3078
|
+
}
|
|
3079
|
+
return vec;
|
|
3080
|
+
}
|
|
3081
|
+
function hashNgram(str) {
|
|
3082
|
+
let hash = 2166136261;
|
|
3083
|
+
for (let i = 0; i < str.length; i++) {
|
|
3084
|
+
hash ^= str.charCodeAt(i);
|
|
3085
|
+
hash = Math.imul(hash, 16777619);
|
|
3086
|
+
}
|
|
3087
|
+
return hash >>> 0;
|
|
3088
|
+
}
|
|
3089
|
+
function cosineSimilarity(a, b) {
|
|
3090
|
+
let dot = 0;
|
|
3091
|
+
const len = Math.min(a.length, b.length);
|
|
3092
|
+
for (let i = 0; i < len; i++) {
|
|
3093
|
+
dot += a[i] * b[i];
|
|
3094
|
+
}
|
|
3095
|
+
return dot;
|
|
3096
|
+
}
|
|
3097
|
+
function encodeVector(vec) {
|
|
3098
|
+
return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
|
|
3099
|
+
}
|
|
3100
|
+
function decodeVector(buf) {
|
|
3101
|
+
const view = new DataView(
|
|
3102
|
+
buf.buffer,
|
|
3103
|
+
buf.byteOffset,
|
|
3104
|
+
buf.byteLength
|
|
3105
|
+
);
|
|
3106
|
+
const copy = new Float32Array(buf.byteLength / 4);
|
|
3107
|
+
for (let i = 0; i < copy.length; i++) {
|
|
3108
|
+
copy[i] = view.getFloat32(i * 4, true);
|
|
3109
|
+
}
|
|
3110
|
+
return copy;
|
|
3111
|
+
}
|
|
3112
|
+
function reciprocalRankFusion(bm25Ranks, vectorRanks, k = RRF_K) {
|
|
3113
|
+
const allIds = /* @__PURE__ */ new Set([...bm25Ranks.keys(), ...vectorRanks.keys()]);
|
|
3114
|
+
const scored = [];
|
|
3115
|
+
for (const id of allIds) {
|
|
3116
|
+
const bm25Rank = bm25Ranks.get(id);
|
|
3117
|
+
const vecRank = vectorRanks.get(id);
|
|
3118
|
+
let score = 0;
|
|
3119
|
+
if (bm25Rank !== void 0) score += 1 / (k + bm25Rank);
|
|
3120
|
+
if (vecRank !== void 0) score += 1 / (k + vecRank);
|
|
3121
|
+
scored.push([id, score]);
|
|
3122
|
+
}
|
|
3123
|
+
scored.sort((a, b) => b[1] - a[1]);
|
|
3124
|
+
return scored;
|
|
3125
|
+
}
|
|
3126
|
+
|
|
3048
3127
|
// src/codebase-index/writer-admin.ts
|
|
3049
3128
|
import * as fs from "node:fs";
|
|
3050
3129
|
import * as path2 from "node:path";
|
|
@@ -4406,90 +4485,19 @@ var StorePool = class {
|
|
|
4406
4485
|
}
|
|
4407
4486
|
};
|
|
4408
4487
|
|
|
4409
|
-
// src/codebase-index/vector-search.ts
|
|
4410
|
-
var RRF_K = 60;
|
|
4411
|
-
var VECTOR_DIMENSIONS = 384;
|
|
4412
|
-
var NGRAM_SIZE = 3;
|
|
4413
|
-
function embedText(text) {
|
|
4414
|
-
const vec = new Float32Array(VECTOR_DIMENSIONS);
|
|
4415
|
-
const normalized = text.toLowerCase().trim();
|
|
4416
|
-
if (normalized.length < NGRAM_SIZE) {
|
|
4417
|
-
const padded = ` ${normalized} `.slice(0, Math.max(NGRAM_SIZE, normalized.length + 2));
|
|
4418
|
-
for (let i = 0; i <= padded.length - NGRAM_SIZE; i++) {
|
|
4419
|
-
const ngram = padded.slice(i, i + NGRAM_SIZE);
|
|
4420
|
-
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
4421
|
-
vec[bucket] += 1;
|
|
4422
|
-
}
|
|
4423
|
-
} else {
|
|
4424
|
-
for (let i = 0; i <= normalized.length - NGRAM_SIZE; i++) {
|
|
4425
|
-
const ngram = normalized.slice(i, i + NGRAM_SIZE);
|
|
4426
|
-
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
4427
|
-
vec[bucket] += 1;
|
|
4428
|
-
}
|
|
4429
|
-
}
|
|
4430
|
-
let norm = 0;
|
|
4431
|
-
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
4432
|
-
norm += vec[i] * vec[i];
|
|
4433
|
-
}
|
|
4434
|
-
norm = Math.sqrt(norm);
|
|
4435
|
-
if (norm > 0) {
|
|
4436
|
-
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
4437
|
-
vec[i] /= norm;
|
|
4438
|
-
}
|
|
4439
|
-
}
|
|
4440
|
-
return vec;
|
|
4441
|
-
}
|
|
4442
|
-
function hashNgram(str) {
|
|
4443
|
-
let hash = 2166136261;
|
|
4444
|
-
for (let i = 0; i < str.length; i++) {
|
|
4445
|
-
hash ^= str.charCodeAt(i);
|
|
4446
|
-
hash = Math.imul(hash, 16777619);
|
|
4447
|
-
}
|
|
4448
|
-
return hash >>> 0;
|
|
4449
|
-
}
|
|
4450
|
-
function cosineSimilarity(a, b) {
|
|
4451
|
-
let dot = 0;
|
|
4452
|
-
const len = Math.min(a.length, b.length);
|
|
4453
|
-
for (let i = 0; i < len; i++) {
|
|
4454
|
-
dot += a[i] * b[i];
|
|
4455
|
-
}
|
|
4456
|
-
return dot;
|
|
4457
|
-
}
|
|
4458
|
-
function encodeVector(vec) {
|
|
4459
|
-
return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
|
|
4460
|
-
}
|
|
4461
|
-
function decodeVector(buf) {
|
|
4462
|
-
const view = new DataView(
|
|
4463
|
-
buf.buffer,
|
|
4464
|
-
buf.byteOffset,
|
|
4465
|
-
buf.byteLength
|
|
4466
|
-
);
|
|
4467
|
-
const copy = new Float32Array(buf.byteLength / 4);
|
|
4468
|
-
for (let i = 0; i < copy.length; i++) {
|
|
4469
|
-
copy[i] = view.getFloat32(i * 4, true);
|
|
4470
|
-
}
|
|
4471
|
-
return copy;
|
|
4472
|
-
}
|
|
4473
|
-
function reciprocalRankFusion(bm25Ranks, vectorRanks, k = RRF_K) {
|
|
4474
|
-
const allIds = /* @__PURE__ */ new Set([...bm25Ranks.keys(), ...vectorRanks.keys()]);
|
|
4475
|
-
const scored = [];
|
|
4476
|
-
for (const id of allIds) {
|
|
4477
|
-
const bm25Rank = bm25Ranks.get(id);
|
|
4478
|
-
const vecRank = vectorRanks.get(id);
|
|
4479
|
-
let score = 0;
|
|
4480
|
-
if (bm25Rank !== void 0) score += 1 / (k + bm25Rank);
|
|
4481
|
-
if (vecRank !== void 0) score += 1 / (k + vecRank);
|
|
4482
|
-
scored.push([id, score]);
|
|
4483
|
-
}
|
|
4484
|
-
scored.sort((a, b) => b[1] - a[1]);
|
|
4485
|
-
return scored;
|
|
4486
|
-
}
|
|
4487
|
-
|
|
4488
4488
|
// src/codebase-index/writer.ts
|
|
4489
4489
|
var DB_FILE2 = "index.db";
|
|
4490
4490
|
var MAX_STATEMENT_CACHE = 128;
|
|
4491
4491
|
var IndexStore = class _IndexStore {
|
|
4492
4492
|
db;
|
|
4493
|
+
/**
|
|
4494
|
+
* True while an index run owns one outer SQLite transaction. Individual
|
|
4495
|
+
* writer methods normally protect themselves with BEGIN/COMMIT, but during
|
|
4496
|
+
* a refresh they join this transaction so readers observe either the last
|
|
4497
|
+
* completed index or the next completed index, never an in-between batch.
|
|
4498
|
+
*/
|
|
4499
|
+
atomicIndexUpdateActive = false;
|
|
4500
|
+
writeSavepointSequence = 0;
|
|
4493
4501
|
/** Absolute path to this project's index directory. */
|
|
4494
4502
|
indexDir;
|
|
4495
4503
|
/**
|
|
@@ -4570,6 +4578,51 @@ var IndexStore = class _IndexStore {
|
|
|
4570
4578
|
runWithRetry(fn) {
|
|
4571
4579
|
return runSqliteWithRetry(fn);
|
|
4572
4580
|
}
|
|
4581
|
+
/** Run a complete index mutation as one WAL-visible publication. */
|
|
4582
|
+
async runAtomicIndexUpdate(job) {
|
|
4583
|
+
if (this.atomicIndexUpdateActive) return job();
|
|
4584
|
+
this.runWithRetry(() => this.db.exec("BEGIN IMMEDIATE"));
|
|
4585
|
+
this.atomicIndexUpdateActive = true;
|
|
4586
|
+
try {
|
|
4587
|
+
const result = await job();
|
|
4588
|
+
this.db.exec("COMMIT");
|
|
4589
|
+
return result;
|
|
4590
|
+
} catch (error) {
|
|
4591
|
+
try {
|
|
4592
|
+
this.db.exec("ROLLBACK");
|
|
4593
|
+
} catch {
|
|
4594
|
+
}
|
|
4595
|
+
throw error;
|
|
4596
|
+
} finally {
|
|
4597
|
+
this.atomicIndexUpdateActive = false;
|
|
4598
|
+
}
|
|
4599
|
+
}
|
|
4600
|
+
/**
|
|
4601
|
+
* Begin a method-local transaction. Inside an atomic index publication a
|
|
4602
|
+
* SAVEPOINT preserves the old per-batch rollback boundary, which is needed
|
|
4603
|
+
* when commitBatch falls back to per-file writes after one batch fails.
|
|
4604
|
+
*/
|
|
4605
|
+
beginWriteTransaction() {
|
|
4606
|
+
if (this.atomicIndexUpdateActive) {
|
|
4607
|
+
const savepoint = `index_write_${++this.writeSavepointSequence}`;
|
|
4608
|
+
this.db.exec(`SAVEPOINT ${savepoint}`);
|
|
4609
|
+
return savepoint;
|
|
4610
|
+
}
|
|
4611
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
4612
|
+
return null;
|
|
4613
|
+
}
|
|
4614
|
+
commitWriteTransaction(savepoint) {
|
|
4615
|
+
if (savepoint) this.db.exec(`RELEASE SAVEPOINT ${savepoint}`);
|
|
4616
|
+
else this.db.exec("COMMIT");
|
|
4617
|
+
}
|
|
4618
|
+
rollbackWriteTransaction(savepoint) {
|
|
4619
|
+
if (savepoint) {
|
|
4620
|
+
this.db.exec(`ROLLBACK TO SAVEPOINT ${savepoint}`);
|
|
4621
|
+
this.db.exec(`RELEASE SAVEPOINT ${savepoint}`);
|
|
4622
|
+
} else {
|
|
4623
|
+
this.db.exec("ROLLBACK");
|
|
4624
|
+
}
|
|
4625
|
+
}
|
|
4573
4626
|
/**
|
|
4574
4627
|
* Mirror the in-process language→family map into SQLite.
|
|
4575
4628
|
*
|
|
@@ -4804,7 +4857,7 @@ var IndexStore = class _IndexStore {
|
|
|
4804
4857
|
insertSymbols(symbols) {
|
|
4805
4858
|
this.invalidateBm25();
|
|
4806
4859
|
return this.runWithRetry(() => {
|
|
4807
|
-
this.
|
|
4860
|
+
const ownsTransaction = this.beginWriteTransaction();
|
|
4808
4861
|
try {
|
|
4809
4862
|
let nextId = this.allocateSymbolIds(symbols.length);
|
|
4810
4863
|
const result = [];
|
|
@@ -4831,7 +4884,9 @@ var IndexStore = class _IndexStore {
|
|
|
4831
4884
|
}
|
|
4832
4885
|
vectorRows.push({
|
|
4833
4886
|
id,
|
|
4834
|
-
vector: encodeVector(
|
|
4887
|
+
vector: encodeVector(
|
|
4888
|
+
embedText(s.text || buildIndexableText(s.name, s.signature, s.docComment))
|
|
4889
|
+
)
|
|
4835
4890
|
});
|
|
4836
4891
|
result.push({ ...s, id });
|
|
4837
4892
|
}
|
|
@@ -4849,10 +4904,10 @@ var IndexStore = class _IndexStore {
|
|
|
4849
4904
|
vectorRows
|
|
4850
4905
|
);
|
|
4851
4906
|
}
|
|
4852
|
-
this.
|
|
4907
|
+
this.commitWriteTransaction(ownsTransaction);
|
|
4853
4908
|
return result;
|
|
4854
4909
|
} catch (err) {
|
|
4855
|
-
this.
|
|
4910
|
+
this.rollbackWriteTransaction(ownsTransaction);
|
|
4856
4911
|
throw err;
|
|
4857
4912
|
}
|
|
4858
4913
|
});
|
|
@@ -4860,7 +4915,7 @@ var IndexStore = class _IndexStore {
|
|
|
4860
4915
|
deleteSymbolsForFile(file) {
|
|
4861
4916
|
this.invalidateBm25();
|
|
4862
4917
|
this.runWithRetry(() => {
|
|
4863
|
-
this.
|
|
4918
|
+
const ownsTransaction = this.beginWriteTransaction();
|
|
4864
4919
|
try {
|
|
4865
4920
|
const affectedNames = this.invalidateIncomingRefsForFiles([file]);
|
|
4866
4921
|
if (this.ftsAvailable) {
|
|
@@ -4875,9 +4930,9 @@ var IndexStore = class _IndexStore {
|
|
|
4875
4930
|
}
|
|
4876
4931
|
this.stmt("DELETE FROM symbols WHERE file_fk = ?").run(file);
|
|
4877
4932
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
4878
|
-
this.
|
|
4933
|
+
this.commitWriteTransaction(ownsTransaction);
|
|
4879
4934
|
} catch (error) {
|
|
4880
|
-
this.
|
|
4935
|
+
this.rollbackWriteTransaction(ownsTransaction);
|
|
4881
4936
|
throw error;
|
|
4882
4937
|
}
|
|
4883
4938
|
});
|
|
@@ -4890,7 +4945,7 @@ var IndexStore = class _IndexStore {
|
|
|
4890
4945
|
deleteFile(file) {
|
|
4891
4946
|
this.invalidateBm25();
|
|
4892
4947
|
this.runWithRetry(() => {
|
|
4893
|
-
this.
|
|
4948
|
+
const ownsTransaction = this.beginWriteTransaction();
|
|
4894
4949
|
try {
|
|
4895
4950
|
const affectedNames = this.invalidateIncomingRefsForFiles([file]);
|
|
4896
4951
|
if (this.ftsAvailable) {
|
|
@@ -4909,9 +4964,9 @@ var IndexStore = class _IndexStore {
|
|
|
4909
4964
|
this.stmt("DELETE FROM symbols WHERE file_fk = ?").run(file);
|
|
4910
4965
|
this.stmt("DELETE FROM files WHERE file = ?").run(file);
|
|
4911
4966
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
4912
|
-
this.
|
|
4967
|
+
this.commitWriteTransaction(ownsTransaction);
|
|
4913
4968
|
} catch (err) {
|
|
4914
|
-
this.
|
|
4969
|
+
this.rollbackWriteTransaction(ownsTransaction);
|
|
4915
4970
|
throw err;
|
|
4916
4971
|
}
|
|
4917
4972
|
});
|
|
@@ -5298,7 +5353,7 @@ var IndexStore = class _IndexStore {
|
|
|
5298
5353
|
clearAll() {
|
|
5299
5354
|
this.invalidateBm25();
|
|
5300
5355
|
this.runWithRetry(() => {
|
|
5301
|
-
this.
|
|
5356
|
+
const ownsTransaction = this.beginWriteTransaction();
|
|
5302
5357
|
try {
|
|
5303
5358
|
this.db.exec("DROP TABLE IF EXISTS refs");
|
|
5304
5359
|
this.db.exec("DROP TABLE IF EXISTS symbols");
|
|
@@ -5306,15 +5361,15 @@ var IndexStore = class _IndexStore {
|
|
|
5306
5361
|
this.db.exec("DROP TABLE IF EXISTS metadata");
|
|
5307
5362
|
if (this.ftsAvailable) this.db.exec("DROP TABLE IF EXISTS symbols_fts");
|
|
5308
5363
|
this.db.exec("DROP TABLE IF EXISTS symbol_vectors");
|
|
5309
|
-
this.db.exec("COMMIT");
|
|
5310
5364
|
this.stmtCache.clear();
|
|
5311
5365
|
this.initSchema();
|
|
5312
5366
|
this.stmt("INSERT OR REPLACE INTO metadata(key, value) VALUES (?, ?)").run(
|
|
5313
5367
|
_IndexStore.NEXT_SYMBOL_ID_KEY,
|
|
5314
5368
|
"1"
|
|
5315
5369
|
);
|
|
5370
|
+
this.commitWriteTransaction(ownsTransaction);
|
|
5316
5371
|
} catch (err) {
|
|
5317
|
-
this.
|
|
5372
|
+
this.rollbackWriteTransaction(ownsTransaction);
|
|
5318
5373
|
throw err;
|
|
5319
5374
|
}
|
|
5320
5375
|
});
|
|
@@ -5376,7 +5431,7 @@ var IndexStore = class _IndexStore {
|
|
|
5376
5431
|
}
|
|
5377
5432
|
this.invalidateBm25();
|
|
5378
5433
|
return this.runWithRetry(() => {
|
|
5379
|
-
this.
|
|
5434
|
+
const ownsTransaction = this.beginWriteTransaction();
|
|
5380
5435
|
try {
|
|
5381
5436
|
const affectedNames = /* @__PURE__ */ new Set();
|
|
5382
5437
|
for (const entry of entries) {
|
|
@@ -5437,7 +5492,9 @@ var IndexStore = class _IndexStore {
|
|
|
5437
5492
|
}
|
|
5438
5493
|
vectorRows.push({
|
|
5439
5494
|
id,
|
|
5440
|
-
vector: encodeVector(
|
|
5495
|
+
vector: encodeVector(
|
|
5496
|
+
embedText(s.text || buildIndexableText(s.name, s.signature, s.docComment))
|
|
5497
|
+
)
|
|
5441
5498
|
});
|
|
5442
5499
|
const inserted = { ...s, id };
|
|
5443
5500
|
allInserted.push(inserted);
|
|
@@ -5482,10 +5539,10 @@ var IndexStore = class _IndexStore {
|
|
|
5482
5539
|
);
|
|
5483
5540
|
}
|
|
5484
5541
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
5485
|
-
this.
|
|
5542
|
+
this.commitWriteTransaction(ownsTransaction);
|
|
5486
5543
|
return allInserted;
|
|
5487
5544
|
} catch (err) {
|
|
5488
|
-
this.
|
|
5545
|
+
this.rollbackWriteTransaction(ownsTransaction);
|
|
5489
5546
|
throw err;
|
|
5490
5547
|
}
|
|
5491
5548
|
});
|
|
@@ -5562,7 +5619,7 @@ var IndexStore = class _IndexStore {
|
|
|
5562
5619
|
replaceEmptyFile(meta) {
|
|
5563
5620
|
this.invalidateBm25();
|
|
5564
5621
|
this.runWithRetry(() => {
|
|
5565
|
-
this.
|
|
5622
|
+
const ownsTransaction = this.beginWriteTransaction();
|
|
5566
5623
|
try {
|
|
5567
5624
|
const affectedNames = this.invalidateIncomingRefsForFiles([meta.file]);
|
|
5568
5625
|
if (this.ftsAvailable) {
|
|
@@ -5597,9 +5654,9 @@ var IndexStore = class _IndexStore {
|
|
|
5597
5654
|
meta.lastIndexed
|
|
5598
5655
|
);
|
|
5599
5656
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
5600
|
-
this.
|
|
5657
|
+
this.commitWriteTransaction(ownsTransaction);
|
|
5601
5658
|
} catch (err) {
|
|
5602
|
-
this.
|
|
5659
|
+
this.rollbackWriteTransaction(ownsTransaction);
|
|
5603
5660
|
throw err;
|
|
5604
5661
|
}
|
|
5605
5662
|
});
|
|
@@ -6557,6 +6614,7 @@ import { Worker as Worker2 } from "node:worker_threads";
|
|
|
6557
6614
|
// src/codebase-index/indexer.ts
|
|
6558
6615
|
import { expectDefined as expectDefined5 } from "@wrongstack/core/utils";
|
|
6559
6616
|
import { execFile } from "node:child_process";
|
|
6617
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
6560
6618
|
import * as fs11 from "node:fs/promises";
|
|
6561
6619
|
import { availableParallelism } from "node:os";
|
|
6562
6620
|
import * as path13 from "node:path";
|
|
@@ -7397,6 +7455,10 @@ var DEFAULT_IGNORE_FILES = /* @__PURE__ */ new Set(["package-lock.json", "pnpm-l
|
|
|
7397
7455
|
var INDEXABLE_EXTENSION_SET = new Set(INDEXABLE_EXTENSIONS);
|
|
7398
7456
|
var MAX_INDEX_FILE_BYTES = 5 * 1024 * 1024;
|
|
7399
7457
|
var MAX_GIT_FILE_LIST_BYTES = 64 * 1024 * 1024;
|
|
7458
|
+
var GIT_SNAPSHOT_METADATA_KEY = "git_discovery_snapshot";
|
|
7459
|
+
var IndexSourceChangedError = class extends Error {
|
|
7460
|
+
name = "IndexSourceChangedError";
|
|
7461
|
+
};
|
|
7400
7462
|
function isWithinProject(projectRoot, file) {
|
|
7401
7463
|
const rel = path13.relative(projectRoot, file);
|
|
7402
7464
|
return rel !== "" && !rel.startsWith(`..${path13.sep}`) && rel !== ".." && !path13.isAbsolute(rel);
|
|
@@ -7433,7 +7495,7 @@ async function findGitSourceFiles(projectRoot, ignore, signal) {
|
|
|
7433
7495
|
if (normalizeComparablePath(topLevel) !== normalizeComparablePath(projectRoot)) return null;
|
|
7434
7496
|
throwIfAborted(signal);
|
|
7435
7497
|
const ignoreSet = /* @__PURE__ */ new Set([...DEFAULT_IGNORE, ...ignore]);
|
|
7436
|
-
const [output, statusOutput] = await Promise.all([
|
|
7498
|
+
const [output, statusOutput, stagedOutput] = await Promise.all([
|
|
7437
7499
|
gitOutput(projectRoot, ["ls-files", "--cached", "--others", "--exclude-standard", "-z"]),
|
|
7438
7500
|
gitOutput(projectRoot, [
|
|
7439
7501
|
"status",
|
|
@@ -7441,7 +7503,8 @@ async function findGitSourceFiles(projectRoot, ignore, signal) {
|
|
|
7441
7503
|
"-z",
|
|
7442
7504
|
"--untracked-files=all",
|
|
7443
7505
|
"--ignored=no"
|
|
7444
|
-
])
|
|
7506
|
+
]),
|
|
7507
|
+
gitOutput(projectRoot, ["ls-files", "--stage", "-z"])
|
|
7445
7508
|
]);
|
|
7446
7509
|
throwIfAborted(signal);
|
|
7447
7510
|
const dirty = /* @__PURE__ */ new Set();
|
|
@@ -7471,9 +7534,17 @@ async function findGitSourceFiles(projectRoot, ignore, signal) {
|
|
|
7471
7534
|
const ext = path13.extname(relative2).toLowerCase();
|
|
7472
7535
|
if (INDEXABLE_EXTENSION_SET.has(ext) || detectLang(full) !== null) files.push(full);
|
|
7473
7536
|
}
|
|
7537
|
+
const snapshot = createHash2("sha256").update(stagedOutput).update("\0").update(statusOutput);
|
|
7538
|
+
const indexedFiles = new Set(files);
|
|
7539
|
+
for (const dirtyFile of [...dirty].sort()) {
|
|
7540
|
+
if (!indexedFiles.has(dirtyFile) || deleted.has(dirtyFile)) continue;
|
|
7541
|
+
snapshot.update("\0").update(dirtyFile).update("\0");
|
|
7542
|
+
snapshot.update(xxhash64String(await fs11.readFile(dirtyFile, "utf8")));
|
|
7543
|
+
}
|
|
7474
7544
|
return {
|
|
7475
7545
|
files,
|
|
7476
|
-
trustedUnchanged: new Set(files.filter((file) => !dirty.has(file)))
|
|
7546
|
+
trustedUnchanged: new Set(files.filter((file) => !dirty.has(file))),
|
|
7547
|
+
snapshotKey: snapshot.digest("hex")
|
|
7477
7548
|
};
|
|
7478
7549
|
} catch {
|
|
7479
7550
|
return null;
|
|
@@ -7486,7 +7557,8 @@ async function findSourceFiles(projectRoot, ignore, isGitIgnored, signal) {
|
|
|
7486
7557
|
files: gitFiles.files,
|
|
7487
7558
|
complete: true,
|
|
7488
7559
|
errors: [],
|
|
7489
|
-
trustedUnchanged: gitFiles.trustedUnchanged
|
|
7560
|
+
trustedUnchanged: gitFiles.trustedUnchanged,
|
|
7561
|
+
snapshotKey: gitFiles.snapshotKey
|
|
7490
7562
|
};
|
|
7491
7563
|
}
|
|
7492
7564
|
const results = [];
|
|
@@ -7584,6 +7656,17 @@ async function runIndexer(_ctx, opts) {
|
|
|
7584
7656
|
}
|
|
7585
7657
|
}
|
|
7586
7658
|
async function runIndexerWithStore(store, opts) {
|
|
7659
|
+
let result;
|
|
7660
|
+
try {
|
|
7661
|
+
result = await store.runAtomicIndexUpdate(() => runIndexerAtomic(store, opts));
|
|
7662
|
+
} catch (error) {
|
|
7663
|
+
if (!(error instanceof IndexSourceChangedError)) throw error;
|
|
7664
|
+
result = await store.runAtomicIndexUpdate(() => runIndexerAtomic(store, opts));
|
|
7665
|
+
}
|
|
7666
|
+
if (!opts.files) store.compactIfNeeded();
|
|
7667
|
+
return result;
|
|
7668
|
+
}
|
|
7669
|
+
async function runIndexerAtomic(store, opts) {
|
|
7587
7670
|
const { projectRoot, langs, ignore = [], signal } = opts;
|
|
7588
7671
|
const relationGraphVersion = "2";
|
|
7589
7672
|
const refResolutionVersion = "2";
|
|
@@ -7603,6 +7686,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
7603
7686
|
let discoveredFiles = null;
|
|
7604
7687
|
let discoveryComplete = true;
|
|
7605
7688
|
let trustedUnchanged;
|
|
7689
|
+
let discoverySnapshotKey;
|
|
7606
7690
|
if (opts.files && opts.files.length > 0) {
|
|
7607
7691
|
files = opts.files.map((f) => path13.resolve(projectRoot, f)).filter((f) => {
|
|
7608
7692
|
if (!isWithinProject(projectRoot, f)) return false;
|
|
@@ -7616,6 +7700,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
7616
7700
|
discoveryComplete = discovery.complete;
|
|
7617
7701
|
discoveredFiles = new Set(files);
|
|
7618
7702
|
trustedUnchanged = discovery.trustedUnchanged;
|
|
7703
|
+
discoverySnapshotKey = discovery.snapshotKey;
|
|
7619
7704
|
}
|
|
7620
7705
|
if (langs && langs.length > 0) {
|
|
7621
7706
|
const langSet = new Set(langs);
|
|
@@ -7629,6 +7714,8 @@ async function runIndexerWithStore(store, opts) {
|
|
|
7629
7714
|
if (!force) {
|
|
7630
7715
|
for (const meta of store.getAllFileMetas()) existingMeta.set(meta.file, meta);
|
|
7631
7716
|
}
|
|
7717
|
+
const snapshotTrusted = !force && discoverySnapshotKey !== void 0 && store.getMetadata(GIT_SNAPSHOT_METADATA_KEY) === discoverySnapshotKey;
|
|
7718
|
+
if (!snapshotTrusted) trustedUnchanged = void 0;
|
|
7632
7719
|
const totalFilesForProgress = files.length;
|
|
7633
7720
|
let filesPreSkipped = 0;
|
|
7634
7721
|
if (!force && trustedUnchanged) {
|
|
@@ -7691,9 +7778,6 @@ async function runIndexerWithStore(store, opts) {
|
|
|
7691
7778
|
};
|
|
7692
7779
|
}
|
|
7693
7780
|
const meta = existingMeta.get(file);
|
|
7694
|
-
if (!force && meta && meta.mtimeMs === Math.floor(stat2.mtimeMs)) {
|
|
7695
|
-
return { file, stat: stat2, lang, parsed: null, skippedMeta: meta };
|
|
7696
|
-
}
|
|
7697
7781
|
let content;
|
|
7698
7782
|
try {
|
|
7699
7783
|
content = await fs11.readFile(file, { encoding: "utf8", signal });
|
|
@@ -7922,9 +8006,18 @@ async function runIndexerWithStore(store, opts) {
|
|
|
7922
8006
|
});
|
|
7923
8007
|
store.setMetadata("ref_resolution_version", refResolutionVersion);
|
|
7924
8008
|
store.setMetadata("relation_graph_version", relationGraphVersion);
|
|
8009
|
+
const completeProjectScope = !opts.files && (!langs || langs.length === 0) && (!opts.ignore || opts.ignore.length === 0);
|
|
8010
|
+
if (completeProjectScope && discoverySnapshotKey !== void 0) {
|
|
8011
|
+
const finalSnapshot = await findGitSourceFiles(projectRoot, ignore, signal);
|
|
8012
|
+
if (!finalSnapshot || finalSnapshot.snapshotKey !== discoverySnapshotKey) {
|
|
8013
|
+
throw new IndexSourceChangedError(
|
|
8014
|
+
"Project files changed during indexing; retrying before publishing the generation."
|
|
8015
|
+
);
|
|
8016
|
+
}
|
|
8017
|
+
store.setMetadata(GIT_SNAPSHOT_METADATA_KEY, errors.length === 0 ? discoverySnapshotKey : "");
|
|
8018
|
+
}
|
|
7925
8019
|
if (!opts.files || filesIndexed >= 50) store.optimize();
|
|
7926
8020
|
store.setLastIndexed(Date.now());
|
|
7927
|
-
if (!opts.files) store.compactIfNeeded();
|
|
7928
8021
|
const durationMs = Date.now() - startMs;
|
|
7929
8022
|
return {
|
|
7930
8023
|
filesIndexed,
|
|
@@ -8235,6 +8328,13 @@ function callIndexOp(op, args, opts) {
|
|
|
8235
8328
|
});
|
|
8236
8329
|
}
|
|
8237
8330
|
async function callInline(op, args, opts) {
|
|
8331
|
+
if (op !== "index" && _indexing) {
|
|
8332
|
+
const error = new Error(
|
|
8333
|
+
"Codebase index refresh in progress; retry after the completed generation is published."
|
|
8334
|
+
);
|
|
8335
|
+
error.name = "IndexRefreshInProgressError";
|
|
8336
|
+
throw error;
|
|
8337
|
+
}
|
|
8238
8338
|
const ac = new AbortController();
|
|
8239
8339
|
const onOuterAbort = () => ac.abort(opts.signal?.reason ?? new Error("Indexing cancelled"));
|
|
8240
8340
|
if (opts.signal?.aborted) onOuterAbort();
|
|
@@ -8579,12 +8679,12 @@ var codebaseSearchTool = {
|
|
|
8579
8679
|
},
|
|
8580
8680
|
async execute(input, ctx, execOpts) {
|
|
8581
8681
|
const state = getIndexState();
|
|
8582
|
-
if (state.indexing
|
|
8682
|
+
if (state.indexing) {
|
|
8583
8683
|
return {
|
|
8584
8684
|
results: [],
|
|
8585
8685
|
total: 0,
|
|
8586
8686
|
query: input.query,
|
|
8587
|
-
indexStatus: `
|
|
8687
|
+
indexStatus: `Index refresh in progress (${state.currentFile}/${state.totalFiles} files) \u2014 retry after the completed generation is published.`
|
|
8588
8688
|
};
|
|
8589
8689
|
}
|
|
8590
8690
|
if (state.lastError) {
|
|
@@ -8765,12 +8865,12 @@ var codebaseIncomingCallsTool = {
|
|
|
8765
8865
|
},
|
|
8766
8866
|
async execute(input, ctx) {
|
|
8767
8867
|
const state = getIndexState();
|
|
8768
|
-
if (state.indexing
|
|
8868
|
+
if (state.indexing) {
|
|
8769
8869
|
return {
|
|
8770
8870
|
symbol: input.symbol,
|
|
8771
8871
|
calls: [],
|
|
8772
8872
|
total: 0,
|
|
8773
|
-
indexStatus: `
|
|
8873
|
+
indexStatus: `Index refresh in progress (${state.currentFile}/${state.totalFiles} files) \u2014 retry after the completed generation is published.`
|
|
8774
8874
|
};
|
|
8775
8875
|
}
|
|
8776
8876
|
if (state.lastError) {
|
|
@@ -8787,16 +8887,14 @@ var codebaseIncomingCallsTool = {
|
|
|
8787
8887
|
const transitive = input.transitive === true;
|
|
8788
8888
|
let serviced;
|
|
8789
8889
|
try {
|
|
8790
|
-
serviced = await incomingCallsService2(
|
|
8791
|
-
|
|
8792
|
-
|
|
8793
|
-
|
|
8794
|
-
|
|
8795
|
-
|
|
8796
|
-
|
|
8797
|
-
|
|
8798
|
-
}
|
|
8799
|
-
);
|
|
8890
|
+
serviced = await incomingCallsService2({
|
|
8891
|
+
projectRoot: ctx.projectRoot,
|
|
8892
|
+
indexDir: codebaseIndexDirOverride(ctx),
|
|
8893
|
+
symbol: input.symbol,
|
|
8894
|
+
file: input.file,
|
|
8895
|
+
limit,
|
|
8896
|
+
transitive
|
|
8897
|
+
});
|
|
8800
8898
|
} catch (err) {
|
|
8801
8899
|
return {
|
|
8802
8900
|
symbol: input.symbol,
|
|
@@ -8835,10 +8933,14 @@ var codebaseIncomingCallsTool = {
|
|
|
8835
8933
|
}
|
|
8836
8934
|
const notes = [];
|
|
8837
8935
|
if (totalMatches > limit) {
|
|
8838
|
-
notes.push(
|
|
8936
|
+
notes.push(
|
|
8937
|
+
`Results capped at ${limit} of ${totalMatches} call sites. Increase \`limit\` or use \`file\` to narrow.`
|
|
8938
|
+
);
|
|
8839
8939
|
}
|
|
8840
8940
|
if (ambiguous) {
|
|
8841
|
-
notes.push(
|
|
8941
|
+
notes.push(
|
|
8942
|
+
`Symbol "${input.symbol}" exists in multiple files. Results include callers of all same-named symbols. Use codebase-search to find the exact file and pass it as \`file\`.`
|
|
8943
|
+
);
|
|
8842
8944
|
}
|
|
8843
8945
|
return {
|
|
8844
8946
|
symbol: input.symbol,
|
|
@@ -8888,12 +8990,12 @@ var codebaseOutgoingCallsTool = {
|
|
|
8888
8990
|
},
|
|
8889
8991
|
async execute(input, ctx) {
|
|
8890
8992
|
const state = getIndexState();
|
|
8891
|
-
if (state.indexing
|
|
8993
|
+
if (state.indexing) {
|
|
8892
8994
|
return {
|
|
8893
8995
|
symbol: input.symbol,
|
|
8894
8996
|
calls: [],
|
|
8895
8997
|
total: 0,
|
|
8896
|
-
indexStatus: `
|
|
8998
|
+
indexStatus: `Index refresh in progress (${state.currentFile}/${state.totalFiles} files) \u2014 retry after the completed generation is published.`
|
|
8897
8999
|
};
|
|
8898
9000
|
}
|
|
8899
9001
|
if (state.lastError) {
|
|
@@ -8910,16 +9012,14 @@ var codebaseOutgoingCallsTool = {
|
|
|
8910
9012
|
const transitive = input.transitive === true;
|
|
8911
9013
|
let serviced;
|
|
8912
9014
|
try {
|
|
8913
|
-
serviced = await outgoingCallsService2(
|
|
8914
|
-
|
|
8915
|
-
|
|
8916
|
-
|
|
8917
|
-
|
|
8918
|
-
|
|
8919
|
-
|
|
8920
|
-
|
|
8921
|
-
}
|
|
8922
|
-
);
|
|
9015
|
+
serviced = await outgoingCallsService2({
|
|
9016
|
+
projectRoot: ctx.projectRoot,
|
|
9017
|
+
indexDir: codebaseIndexDirOverride(ctx),
|
|
9018
|
+
symbol: input.symbol,
|
|
9019
|
+
file: input.file,
|
|
9020
|
+
limit,
|
|
9021
|
+
transitive
|
|
9022
|
+
});
|
|
8923
9023
|
} catch (err) {
|
|
8924
9024
|
return {
|
|
8925
9025
|
symbol: input.symbol,
|
|
@@ -8958,10 +9058,14 @@ var codebaseOutgoingCallsTool = {
|
|
|
8958
9058
|
}
|
|
8959
9059
|
const notes = [];
|
|
8960
9060
|
if (totalMatches > limit) {
|
|
8961
|
-
notes.push(
|
|
9061
|
+
notes.push(
|
|
9062
|
+
`Results capped at ${limit} of ${totalMatches} call sites. Increase \`limit\` or use \`file\` to narrow.`
|
|
9063
|
+
);
|
|
8962
9064
|
}
|
|
8963
9065
|
if (unresolvedCount > 0) {
|
|
8964
|
-
notes.push(
|
|
9066
|
+
notes.push(
|
|
9067
|
+
`${unresolvedCount} unresolved reference(s) not shown \u2014 their targets could not be resolved during indexing.`
|
|
9068
|
+
);
|
|
8965
9069
|
}
|
|
8966
9070
|
return {
|
|
8967
9071
|
symbol: input.symbol,
|