@wrongstack/tools 0.306.0 → 0.306.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/builtin.js +256 -137
- package/dist/codebase-index/index.js +241 -137
- package/dist/codebase-index/project-server.js +233 -124
- package/dist/codebase-index/worker.js +210 -107
- package/dist/codebase-index/writer.d.ts +18 -0
- package/dist/index.js +258 -138
- package/dist/pack.js +256 -137
- package/dist/plan.js +15 -0
- package/dist/read.js +210 -110
- package/dist/session-kanban.js +3 -1
- package/dist/task.js +15 -0
- package/dist/todo.js +15 -0
- package/dist/tool-tier.js +256 -137
- package/package.json +4 -4
|
@@ -2704,6 +2704,7 @@ import { parentPort } from "node:worker_threads";
|
|
|
2704
2704
|
// src/codebase-index/indexer.ts
|
|
2705
2705
|
import { expectDefined as expectDefined5 } from "@wrongstack/core/utils";
|
|
2706
2706
|
import { execFile } from "node:child_process";
|
|
2707
|
+
import { createHash } from "node:crypto";
|
|
2707
2708
|
import * as fs9 from "node:fs/promises";
|
|
2708
2709
|
import { availableParallelism } from "node:os";
|
|
2709
2710
|
import * as path12 from "node:path";
|
|
@@ -4100,6 +4101,85 @@ function runSqliteWithRetry(fn) {
|
|
|
4100
4101
|
throw lastError;
|
|
4101
4102
|
}
|
|
4102
4103
|
|
|
4104
|
+
// src/codebase-index/vector-search.ts
|
|
4105
|
+
var RRF_K = 60;
|
|
4106
|
+
var VECTOR_DIMENSIONS = 384;
|
|
4107
|
+
var NGRAM_SIZE = 3;
|
|
4108
|
+
function embedText(text) {
|
|
4109
|
+
const vec = new Float32Array(VECTOR_DIMENSIONS);
|
|
4110
|
+
const normalized = text.toLowerCase().trim();
|
|
4111
|
+
if (normalized.length < NGRAM_SIZE) {
|
|
4112
|
+
const padded = ` ${normalized} `.slice(0, Math.max(NGRAM_SIZE, normalized.length + 2));
|
|
4113
|
+
for (let i = 0; i <= padded.length - NGRAM_SIZE; i++) {
|
|
4114
|
+
const ngram = padded.slice(i, i + NGRAM_SIZE);
|
|
4115
|
+
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
4116
|
+
vec[bucket] += 1;
|
|
4117
|
+
}
|
|
4118
|
+
} else {
|
|
4119
|
+
for (let i = 0; i <= normalized.length - NGRAM_SIZE; i++) {
|
|
4120
|
+
const ngram = normalized.slice(i, i + NGRAM_SIZE);
|
|
4121
|
+
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
4122
|
+
vec[bucket] += 1;
|
|
4123
|
+
}
|
|
4124
|
+
}
|
|
4125
|
+
let norm = 0;
|
|
4126
|
+
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
4127
|
+
norm += vec[i] * vec[i];
|
|
4128
|
+
}
|
|
4129
|
+
norm = Math.sqrt(norm);
|
|
4130
|
+
if (norm > 0) {
|
|
4131
|
+
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
4132
|
+
vec[i] /= norm;
|
|
4133
|
+
}
|
|
4134
|
+
}
|
|
4135
|
+
return vec;
|
|
4136
|
+
}
|
|
4137
|
+
function hashNgram(str) {
|
|
4138
|
+
let hash = 2166136261;
|
|
4139
|
+
for (let i = 0; i < str.length; i++) {
|
|
4140
|
+
hash ^= str.charCodeAt(i);
|
|
4141
|
+
hash = Math.imul(hash, 16777619);
|
|
4142
|
+
}
|
|
4143
|
+
return hash >>> 0;
|
|
4144
|
+
}
|
|
4145
|
+
function cosineSimilarity(a, b) {
|
|
4146
|
+
let dot = 0;
|
|
4147
|
+
const len = Math.min(a.length, b.length);
|
|
4148
|
+
for (let i = 0; i < len; i++) {
|
|
4149
|
+
dot += a[i] * b[i];
|
|
4150
|
+
}
|
|
4151
|
+
return dot;
|
|
4152
|
+
}
|
|
4153
|
+
function encodeVector(vec) {
|
|
4154
|
+
return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
|
|
4155
|
+
}
|
|
4156
|
+
function decodeVector(buf) {
|
|
4157
|
+
const view = new DataView(
|
|
4158
|
+
buf.buffer,
|
|
4159
|
+
buf.byteOffset,
|
|
4160
|
+
buf.byteLength
|
|
4161
|
+
);
|
|
4162
|
+
const copy = new Float32Array(buf.byteLength / 4);
|
|
4163
|
+
for (let i = 0; i < copy.length; i++) {
|
|
4164
|
+
copy[i] = view.getFloat32(i * 4, true);
|
|
4165
|
+
}
|
|
4166
|
+
return copy;
|
|
4167
|
+
}
|
|
4168
|
+
function reciprocalRankFusion(bm25Ranks, vectorRanks, k = RRF_K) {
|
|
4169
|
+
const allIds = /* @__PURE__ */ new Set([...bm25Ranks.keys(), ...vectorRanks.keys()]);
|
|
4170
|
+
const scored = [];
|
|
4171
|
+
for (const id of allIds) {
|
|
4172
|
+
const bm25Rank = bm25Ranks.get(id);
|
|
4173
|
+
const vecRank = vectorRanks.get(id);
|
|
4174
|
+
let score = 0;
|
|
4175
|
+
if (bm25Rank !== void 0) score += 1 / (k + bm25Rank);
|
|
4176
|
+
if (vecRank !== void 0) score += 1 / (k + vecRank);
|
|
4177
|
+
scored.push([id, score]);
|
|
4178
|
+
}
|
|
4179
|
+
scored.sort((a, b) => b[1] - a[1]);
|
|
4180
|
+
return scored;
|
|
4181
|
+
}
|
|
4182
|
+
|
|
4103
4183
|
// src/codebase-index/writer-admin.ts
|
|
4104
4184
|
import * as fs7 from "node:fs";
|
|
4105
4185
|
import * as path10 from "node:path";
|
|
@@ -5155,90 +5235,19 @@ var StorePool = class {
|
|
|
5155
5235
|
}
|
|
5156
5236
|
};
|
|
5157
5237
|
|
|
5158
|
-
// src/codebase-index/vector-search.ts
|
|
5159
|
-
var RRF_K = 60;
|
|
5160
|
-
var VECTOR_DIMENSIONS = 384;
|
|
5161
|
-
var NGRAM_SIZE = 3;
|
|
5162
|
-
function embedText(text) {
|
|
5163
|
-
const vec = new Float32Array(VECTOR_DIMENSIONS);
|
|
5164
|
-
const normalized = text.toLowerCase().trim();
|
|
5165
|
-
if (normalized.length < NGRAM_SIZE) {
|
|
5166
|
-
const padded = ` ${normalized} `.slice(0, Math.max(NGRAM_SIZE, normalized.length + 2));
|
|
5167
|
-
for (let i = 0; i <= padded.length - NGRAM_SIZE; i++) {
|
|
5168
|
-
const ngram = padded.slice(i, i + NGRAM_SIZE);
|
|
5169
|
-
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
5170
|
-
vec[bucket] += 1;
|
|
5171
|
-
}
|
|
5172
|
-
} else {
|
|
5173
|
-
for (let i = 0; i <= normalized.length - NGRAM_SIZE; i++) {
|
|
5174
|
-
const ngram = normalized.slice(i, i + NGRAM_SIZE);
|
|
5175
|
-
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
5176
|
-
vec[bucket] += 1;
|
|
5177
|
-
}
|
|
5178
|
-
}
|
|
5179
|
-
let norm = 0;
|
|
5180
|
-
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
5181
|
-
norm += vec[i] * vec[i];
|
|
5182
|
-
}
|
|
5183
|
-
norm = Math.sqrt(norm);
|
|
5184
|
-
if (norm > 0) {
|
|
5185
|
-
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
5186
|
-
vec[i] /= norm;
|
|
5187
|
-
}
|
|
5188
|
-
}
|
|
5189
|
-
return vec;
|
|
5190
|
-
}
|
|
5191
|
-
function hashNgram(str) {
|
|
5192
|
-
let hash = 2166136261;
|
|
5193
|
-
for (let i = 0; i < str.length; i++) {
|
|
5194
|
-
hash ^= str.charCodeAt(i);
|
|
5195
|
-
hash = Math.imul(hash, 16777619);
|
|
5196
|
-
}
|
|
5197
|
-
return hash >>> 0;
|
|
5198
|
-
}
|
|
5199
|
-
function cosineSimilarity(a, b) {
|
|
5200
|
-
let dot = 0;
|
|
5201
|
-
const len = Math.min(a.length, b.length);
|
|
5202
|
-
for (let i = 0; i < len; i++) {
|
|
5203
|
-
dot += a[i] * b[i];
|
|
5204
|
-
}
|
|
5205
|
-
return dot;
|
|
5206
|
-
}
|
|
5207
|
-
function encodeVector(vec) {
|
|
5208
|
-
return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
|
|
5209
|
-
}
|
|
5210
|
-
function decodeVector(buf) {
|
|
5211
|
-
const view = new DataView(
|
|
5212
|
-
buf.buffer,
|
|
5213
|
-
buf.byteOffset,
|
|
5214
|
-
buf.byteLength
|
|
5215
|
-
);
|
|
5216
|
-
const copy = new Float32Array(buf.byteLength / 4);
|
|
5217
|
-
for (let i = 0; i < copy.length; i++) {
|
|
5218
|
-
copy[i] = view.getFloat32(i * 4, true);
|
|
5219
|
-
}
|
|
5220
|
-
return copy;
|
|
5221
|
-
}
|
|
5222
|
-
function reciprocalRankFusion(bm25Ranks, vectorRanks, k = RRF_K) {
|
|
5223
|
-
const allIds = /* @__PURE__ */ new Set([...bm25Ranks.keys(), ...vectorRanks.keys()]);
|
|
5224
|
-
const scored = [];
|
|
5225
|
-
for (const id of allIds) {
|
|
5226
|
-
const bm25Rank = bm25Ranks.get(id);
|
|
5227
|
-
const vecRank = vectorRanks.get(id);
|
|
5228
|
-
let score = 0;
|
|
5229
|
-
if (bm25Rank !== void 0) score += 1 / (k + bm25Rank);
|
|
5230
|
-
if (vecRank !== void 0) score += 1 / (k + vecRank);
|
|
5231
|
-
scored.push([id, score]);
|
|
5232
|
-
}
|
|
5233
|
-
scored.sort((a, b) => b[1] - a[1]);
|
|
5234
|
-
return scored;
|
|
5235
|
-
}
|
|
5236
|
-
|
|
5237
5238
|
// src/codebase-index/writer.ts
|
|
5238
5239
|
var DB_FILE2 = "index.db";
|
|
5239
5240
|
var MAX_STATEMENT_CACHE = 128;
|
|
5240
5241
|
var IndexStore = class _IndexStore {
|
|
5241
5242
|
db;
|
|
5243
|
+
/**
|
|
5244
|
+
* True while an index run owns one outer SQLite transaction. Individual
|
|
5245
|
+
* writer methods normally protect themselves with BEGIN/COMMIT, but during
|
|
5246
|
+
* a refresh they join this transaction so readers observe either the last
|
|
5247
|
+
* completed index or the next completed index, never an in-between batch.
|
|
5248
|
+
*/
|
|
5249
|
+
atomicIndexUpdateActive = false;
|
|
5250
|
+
writeSavepointSequence = 0;
|
|
5242
5251
|
/** Absolute path to this project's index directory. */
|
|
5243
5252
|
indexDir;
|
|
5244
5253
|
/**
|
|
@@ -5319,6 +5328,51 @@ var IndexStore = class _IndexStore {
|
|
|
5319
5328
|
runWithRetry(fn) {
|
|
5320
5329
|
return runSqliteWithRetry(fn);
|
|
5321
5330
|
}
|
|
5331
|
+
/** Run a complete index mutation as one WAL-visible publication. */
|
|
5332
|
+
async runAtomicIndexUpdate(job) {
|
|
5333
|
+
if (this.atomicIndexUpdateActive) return job();
|
|
5334
|
+
this.runWithRetry(() => this.db.exec("BEGIN IMMEDIATE"));
|
|
5335
|
+
this.atomicIndexUpdateActive = true;
|
|
5336
|
+
try {
|
|
5337
|
+
const result = await job();
|
|
5338
|
+
this.db.exec("COMMIT");
|
|
5339
|
+
return result;
|
|
5340
|
+
} catch (error) {
|
|
5341
|
+
try {
|
|
5342
|
+
this.db.exec("ROLLBACK");
|
|
5343
|
+
} catch {
|
|
5344
|
+
}
|
|
5345
|
+
throw error;
|
|
5346
|
+
} finally {
|
|
5347
|
+
this.atomicIndexUpdateActive = false;
|
|
5348
|
+
}
|
|
5349
|
+
}
|
|
5350
|
+
/**
|
|
5351
|
+
* Begin a method-local transaction. Inside an atomic index publication a
|
|
5352
|
+
* SAVEPOINT preserves the old per-batch rollback boundary, which is needed
|
|
5353
|
+
* when commitBatch falls back to per-file writes after one batch fails.
|
|
5354
|
+
*/
|
|
5355
|
+
beginWriteTransaction() {
|
|
5356
|
+
if (this.atomicIndexUpdateActive) {
|
|
5357
|
+
const savepoint = `index_write_${++this.writeSavepointSequence}`;
|
|
5358
|
+
this.db.exec(`SAVEPOINT ${savepoint}`);
|
|
5359
|
+
return savepoint;
|
|
5360
|
+
}
|
|
5361
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
5362
|
+
return null;
|
|
5363
|
+
}
|
|
5364
|
+
commitWriteTransaction(savepoint) {
|
|
5365
|
+
if (savepoint) this.db.exec(`RELEASE SAVEPOINT ${savepoint}`);
|
|
5366
|
+
else this.db.exec("COMMIT");
|
|
5367
|
+
}
|
|
5368
|
+
rollbackWriteTransaction(savepoint) {
|
|
5369
|
+
if (savepoint) {
|
|
5370
|
+
this.db.exec(`ROLLBACK TO SAVEPOINT ${savepoint}`);
|
|
5371
|
+
this.db.exec(`RELEASE SAVEPOINT ${savepoint}`);
|
|
5372
|
+
} else {
|
|
5373
|
+
this.db.exec("ROLLBACK");
|
|
5374
|
+
}
|
|
5375
|
+
}
|
|
5322
5376
|
/**
|
|
5323
5377
|
* Mirror the in-process language→family map into SQLite.
|
|
5324
5378
|
*
|
|
@@ -5553,7 +5607,7 @@ var IndexStore = class _IndexStore {
|
|
|
5553
5607
|
insertSymbols(symbols) {
|
|
5554
5608
|
this.invalidateBm25();
|
|
5555
5609
|
return this.runWithRetry(() => {
|
|
5556
|
-
this.
|
|
5610
|
+
const ownsTransaction = this.beginWriteTransaction();
|
|
5557
5611
|
try {
|
|
5558
5612
|
let nextId = this.allocateSymbolIds(symbols.length);
|
|
5559
5613
|
const result = [];
|
|
@@ -5580,7 +5634,9 @@ var IndexStore = class _IndexStore {
|
|
|
5580
5634
|
}
|
|
5581
5635
|
vectorRows.push({
|
|
5582
5636
|
id,
|
|
5583
|
-
vector: encodeVector(
|
|
5637
|
+
vector: encodeVector(
|
|
5638
|
+
embedText(s.text || buildIndexableText(s.name, s.signature, s.docComment))
|
|
5639
|
+
)
|
|
5584
5640
|
});
|
|
5585
5641
|
result.push({ ...s, id });
|
|
5586
5642
|
}
|
|
@@ -5598,10 +5654,10 @@ var IndexStore = class _IndexStore {
|
|
|
5598
5654
|
vectorRows
|
|
5599
5655
|
);
|
|
5600
5656
|
}
|
|
5601
|
-
this.
|
|
5657
|
+
this.commitWriteTransaction(ownsTransaction);
|
|
5602
5658
|
return result;
|
|
5603
5659
|
} catch (err) {
|
|
5604
|
-
this.
|
|
5660
|
+
this.rollbackWriteTransaction(ownsTransaction);
|
|
5605
5661
|
throw err;
|
|
5606
5662
|
}
|
|
5607
5663
|
});
|
|
@@ -5609,7 +5665,7 @@ var IndexStore = class _IndexStore {
|
|
|
5609
5665
|
deleteSymbolsForFile(file) {
|
|
5610
5666
|
this.invalidateBm25();
|
|
5611
5667
|
this.runWithRetry(() => {
|
|
5612
|
-
this.
|
|
5668
|
+
const ownsTransaction = this.beginWriteTransaction();
|
|
5613
5669
|
try {
|
|
5614
5670
|
const affectedNames = this.invalidateIncomingRefsForFiles([file]);
|
|
5615
5671
|
if (this.ftsAvailable) {
|
|
@@ -5624,9 +5680,9 @@ var IndexStore = class _IndexStore {
|
|
|
5624
5680
|
}
|
|
5625
5681
|
this.stmt("DELETE FROM symbols WHERE file_fk = ?").run(file);
|
|
5626
5682
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
5627
|
-
this.
|
|
5683
|
+
this.commitWriteTransaction(ownsTransaction);
|
|
5628
5684
|
} catch (error) {
|
|
5629
|
-
this.
|
|
5685
|
+
this.rollbackWriteTransaction(ownsTransaction);
|
|
5630
5686
|
throw error;
|
|
5631
5687
|
}
|
|
5632
5688
|
});
|
|
@@ -5639,7 +5695,7 @@ var IndexStore = class _IndexStore {
|
|
|
5639
5695
|
deleteFile(file) {
|
|
5640
5696
|
this.invalidateBm25();
|
|
5641
5697
|
this.runWithRetry(() => {
|
|
5642
|
-
this.
|
|
5698
|
+
const ownsTransaction = this.beginWriteTransaction();
|
|
5643
5699
|
try {
|
|
5644
5700
|
const affectedNames = this.invalidateIncomingRefsForFiles([file]);
|
|
5645
5701
|
if (this.ftsAvailable) {
|
|
@@ -5658,9 +5714,9 @@ var IndexStore = class _IndexStore {
|
|
|
5658
5714
|
this.stmt("DELETE FROM symbols WHERE file_fk = ?").run(file);
|
|
5659
5715
|
this.stmt("DELETE FROM files WHERE file = ?").run(file);
|
|
5660
5716
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
5661
|
-
this.
|
|
5717
|
+
this.commitWriteTransaction(ownsTransaction);
|
|
5662
5718
|
} catch (err) {
|
|
5663
|
-
this.
|
|
5719
|
+
this.rollbackWriteTransaction(ownsTransaction);
|
|
5664
5720
|
throw err;
|
|
5665
5721
|
}
|
|
5666
5722
|
});
|
|
@@ -6047,7 +6103,7 @@ var IndexStore = class _IndexStore {
|
|
|
6047
6103
|
clearAll() {
|
|
6048
6104
|
this.invalidateBm25();
|
|
6049
6105
|
this.runWithRetry(() => {
|
|
6050
|
-
this.
|
|
6106
|
+
const ownsTransaction = this.beginWriteTransaction();
|
|
6051
6107
|
try {
|
|
6052
6108
|
this.db.exec("DROP TABLE IF EXISTS refs");
|
|
6053
6109
|
this.db.exec("DROP TABLE IF EXISTS symbols");
|
|
@@ -6055,15 +6111,15 @@ var IndexStore = class _IndexStore {
|
|
|
6055
6111
|
this.db.exec("DROP TABLE IF EXISTS metadata");
|
|
6056
6112
|
if (this.ftsAvailable) this.db.exec("DROP TABLE IF EXISTS symbols_fts");
|
|
6057
6113
|
this.db.exec("DROP TABLE IF EXISTS symbol_vectors");
|
|
6058
|
-
this.db.exec("COMMIT");
|
|
6059
6114
|
this.stmtCache.clear();
|
|
6060
6115
|
this.initSchema();
|
|
6061
6116
|
this.stmt("INSERT OR REPLACE INTO metadata(key, value) VALUES (?, ?)").run(
|
|
6062
6117
|
_IndexStore.NEXT_SYMBOL_ID_KEY,
|
|
6063
6118
|
"1"
|
|
6064
6119
|
);
|
|
6120
|
+
this.commitWriteTransaction(ownsTransaction);
|
|
6065
6121
|
} catch (err) {
|
|
6066
|
-
this.
|
|
6122
|
+
this.rollbackWriteTransaction(ownsTransaction);
|
|
6067
6123
|
throw err;
|
|
6068
6124
|
}
|
|
6069
6125
|
});
|
|
@@ -6125,7 +6181,7 @@ var IndexStore = class _IndexStore {
|
|
|
6125
6181
|
}
|
|
6126
6182
|
this.invalidateBm25();
|
|
6127
6183
|
return this.runWithRetry(() => {
|
|
6128
|
-
this.
|
|
6184
|
+
const ownsTransaction = this.beginWriteTransaction();
|
|
6129
6185
|
try {
|
|
6130
6186
|
const affectedNames = /* @__PURE__ */ new Set();
|
|
6131
6187
|
for (const entry of entries) {
|
|
@@ -6186,7 +6242,9 @@ var IndexStore = class _IndexStore {
|
|
|
6186
6242
|
}
|
|
6187
6243
|
vectorRows.push({
|
|
6188
6244
|
id,
|
|
6189
|
-
vector: encodeVector(
|
|
6245
|
+
vector: encodeVector(
|
|
6246
|
+
embedText(s.text || buildIndexableText(s.name, s.signature, s.docComment))
|
|
6247
|
+
)
|
|
6190
6248
|
});
|
|
6191
6249
|
const inserted = { ...s, id };
|
|
6192
6250
|
allInserted.push(inserted);
|
|
@@ -6231,10 +6289,10 @@ var IndexStore = class _IndexStore {
|
|
|
6231
6289
|
);
|
|
6232
6290
|
}
|
|
6233
6291
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
6234
|
-
this.
|
|
6292
|
+
this.commitWriteTransaction(ownsTransaction);
|
|
6235
6293
|
return allInserted;
|
|
6236
6294
|
} catch (err) {
|
|
6237
|
-
this.
|
|
6295
|
+
this.rollbackWriteTransaction(ownsTransaction);
|
|
6238
6296
|
throw err;
|
|
6239
6297
|
}
|
|
6240
6298
|
});
|
|
@@ -6311,7 +6369,7 @@ var IndexStore = class _IndexStore {
|
|
|
6311
6369
|
replaceEmptyFile(meta) {
|
|
6312
6370
|
this.invalidateBm25();
|
|
6313
6371
|
this.runWithRetry(() => {
|
|
6314
|
-
this.
|
|
6372
|
+
const ownsTransaction = this.beginWriteTransaction();
|
|
6315
6373
|
try {
|
|
6316
6374
|
const affectedNames = this.invalidateIncomingRefsForFiles([meta.file]);
|
|
6317
6375
|
if (this.ftsAvailable) {
|
|
@@ -6346,9 +6404,9 @@ var IndexStore = class _IndexStore {
|
|
|
6346
6404
|
meta.lastIndexed
|
|
6347
6405
|
);
|
|
6348
6406
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
6349
|
-
this.
|
|
6407
|
+
this.commitWriteTransaction(ownsTransaction);
|
|
6350
6408
|
} catch (err) {
|
|
6351
|
-
this.
|
|
6409
|
+
this.rollbackWriteTransaction(ownsTransaction);
|
|
6352
6410
|
throw err;
|
|
6353
6411
|
}
|
|
6354
6412
|
});
|
|
@@ -6541,6 +6599,10 @@ var DEFAULT_IGNORE_FILES = /* @__PURE__ */ new Set(["package-lock.json", "pnpm-l
|
|
|
6541
6599
|
var INDEXABLE_EXTENSION_SET = new Set(INDEXABLE_EXTENSIONS);
|
|
6542
6600
|
var MAX_INDEX_FILE_BYTES = 5 * 1024 * 1024;
|
|
6543
6601
|
var MAX_GIT_FILE_LIST_BYTES = 64 * 1024 * 1024;
|
|
6602
|
+
var GIT_SNAPSHOT_METADATA_KEY = "git_discovery_snapshot";
|
|
6603
|
+
var IndexSourceChangedError = class extends Error {
|
|
6604
|
+
name = "IndexSourceChangedError";
|
|
6605
|
+
};
|
|
6544
6606
|
function isWithinProject(projectRoot, file) {
|
|
6545
6607
|
const rel = path12.relative(projectRoot, file);
|
|
6546
6608
|
return rel !== "" && !rel.startsWith(`..${path12.sep}`) && rel !== ".." && !path12.isAbsolute(rel);
|
|
@@ -6577,7 +6639,7 @@ async function findGitSourceFiles(projectRoot, ignore, signal) {
|
|
|
6577
6639
|
if (normalizeComparablePath(topLevel) !== normalizeComparablePath(projectRoot)) return null;
|
|
6578
6640
|
throwIfAborted(signal);
|
|
6579
6641
|
const ignoreSet = /* @__PURE__ */ new Set([...DEFAULT_IGNORE, ...ignore]);
|
|
6580
|
-
const [output, statusOutput] = await Promise.all([
|
|
6642
|
+
const [output, statusOutput, stagedOutput] = await Promise.all([
|
|
6581
6643
|
gitOutput(projectRoot, ["ls-files", "--cached", "--others", "--exclude-standard", "-z"]),
|
|
6582
6644
|
gitOutput(projectRoot, [
|
|
6583
6645
|
"status",
|
|
@@ -6585,7 +6647,8 @@ async function findGitSourceFiles(projectRoot, ignore, signal) {
|
|
|
6585
6647
|
"-z",
|
|
6586
6648
|
"--untracked-files=all",
|
|
6587
6649
|
"--ignored=no"
|
|
6588
|
-
])
|
|
6650
|
+
]),
|
|
6651
|
+
gitOutput(projectRoot, ["ls-files", "--stage", "-z"])
|
|
6589
6652
|
]);
|
|
6590
6653
|
throwIfAborted(signal);
|
|
6591
6654
|
const dirty = /* @__PURE__ */ new Set();
|
|
@@ -6615,9 +6678,17 @@ async function findGitSourceFiles(projectRoot, ignore, signal) {
|
|
|
6615
6678
|
const ext = path12.extname(relative2).toLowerCase();
|
|
6616
6679
|
if (INDEXABLE_EXTENSION_SET.has(ext) || detectLang(full) !== null) files.push(full);
|
|
6617
6680
|
}
|
|
6681
|
+
const snapshot = createHash("sha256").update(stagedOutput).update("\0").update(statusOutput);
|
|
6682
|
+
const indexedFiles = new Set(files);
|
|
6683
|
+
for (const dirtyFile of [...dirty].sort()) {
|
|
6684
|
+
if (!indexedFiles.has(dirtyFile) || deleted.has(dirtyFile)) continue;
|
|
6685
|
+
snapshot.update("\0").update(dirtyFile).update("\0");
|
|
6686
|
+
snapshot.update(xxhash64String(await fs9.readFile(dirtyFile, "utf8")));
|
|
6687
|
+
}
|
|
6618
6688
|
return {
|
|
6619
6689
|
files,
|
|
6620
|
-
trustedUnchanged: new Set(files.filter((file) => !dirty.has(file)))
|
|
6690
|
+
trustedUnchanged: new Set(files.filter((file) => !dirty.has(file))),
|
|
6691
|
+
snapshotKey: snapshot.digest("hex")
|
|
6621
6692
|
};
|
|
6622
6693
|
} catch {
|
|
6623
6694
|
return null;
|
|
@@ -6630,7 +6701,8 @@ async function findSourceFiles(projectRoot, ignore, isGitIgnored, signal) {
|
|
|
6630
6701
|
files: gitFiles.files,
|
|
6631
6702
|
complete: true,
|
|
6632
6703
|
errors: [],
|
|
6633
|
-
trustedUnchanged: gitFiles.trustedUnchanged
|
|
6704
|
+
trustedUnchanged: gitFiles.trustedUnchanged,
|
|
6705
|
+
snapshotKey: gitFiles.snapshotKey
|
|
6634
6706
|
};
|
|
6635
6707
|
}
|
|
6636
6708
|
const results = [];
|
|
@@ -6717,6 +6789,17 @@ async function resolveProjectRelations(store, projectRoot, opts) {
|
|
|
6717
6789
|
}
|
|
6718
6790
|
}
|
|
6719
6791
|
async function runIndexerWithStore(store, opts) {
|
|
6792
|
+
let result;
|
|
6793
|
+
try {
|
|
6794
|
+
result = await store.runAtomicIndexUpdate(() => runIndexerAtomic(store, opts));
|
|
6795
|
+
} catch (error) {
|
|
6796
|
+
if (!(error instanceof IndexSourceChangedError)) throw error;
|
|
6797
|
+
result = await store.runAtomicIndexUpdate(() => runIndexerAtomic(store, opts));
|
|
6798
|
+
}
|
|
6799
|
+
if (!opts.files) store.compactIfNeeded();
|
|
6800
|
+
return result;
|
|
6801
|
+
}
|
|
6802
|
+
async function runIndexerAtomic(store, opts) {
|
|
6720
6803
|
const { projectRoot, langs, ignore = [], signal } = opts;
|
|
6721
6804
|
const relationGraphVersion = "2";
|
|
6722
6805
|
const refResolutionVersion = "2";
|
|
@@ -6736,6 +6819,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
6736
6819
|
let discoveredFiles = null;
|
|
6737
6820
|
let discoveryComplete = true;
|
|
6738
6821
|
let trustedUnchanged;
|
|
6822
|
+
let discoverySnapshotKey;
|
|
6739
6823
|
if (opts.files && opts.files.length > 0) {
|
|
6740
6824
|
files = opts.files.map((f) => path12.resolve(projectRoot, f)).filter((f) => {
|
|
6741
6825
|
if (!isWithinProject(projectRoot, f)) return false;
|
|
@@ -6749,6 +6833,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
6749
6833
|
discoveryComplete = discovery.complete;
|
|
6750
6834
|
discoveredFiles = new Set(files);
|
|
6751
6835
|
trustedUnchanged = discovery.trustedUnchanged;
|
|
6836
|
+
discoverySnapshotKey = discovery.snapshotKey;
|
|
6752
6837
|
}
|
|
6753
6838
|
if (langs && langs.length > 0) {
|
|
6754
6839
|
const langSet = new Set(langs);
|
|
@@ -6762,6 +6847,8 @@ async function runIndexerWithStore(store, opts) {
|
|
|
6762
6847
|
if (!force) {
|
|
6763
6848
|
for (const meta of store.getAllFileMetas()) existingMeta.set(meta.file, meta);
|
|
6764
6849
|
}
|
|
6850
|
+
const snapshotTrusted = !force && discoverySnapshotKey !== void 0 && store.getMetadata(GIT_SNAPSHOT_METADATA_KEY) === discoverySnapshotKey;
|
|
6851
|
+
if (!snapshotTrusted) trustedUnchanged = void 0;
|
|
6765
6852
|
const totalFilesForProgress = files.length;
|
|
6766
6853
|
let filesPreSkipped = 0;
|
|
6767
6854
|
if (!force && trustedUnchanged) {
|
|
@@ -6824,9 +6911,6 @@ async function runIndexerWithStore(store, opts) {
|
|
|
6824
6911
|
};
|
|
6825
6912
|
}
|
|
6826
6913
|
const meta = existingMeta.get(file);
|
|
6827
|
-
if (!force && meta && meta.mtimeMs === Math.floor(stat2.mtimeMs)) {
|
|
6828
|
-
return { file, stat: stat2, lang, parsed: null, skippedMeta: meta };
|
|
6829
|
-
}
|
|
6830
6914
|
let content;
|
|
6831
6915
|
try {
|
|
6832
6916
|
content = await fs9.readFile(file, { encoding: "utf8", signal });
|
|
@@ -7055,9 +7139,18 @@ async function runIndexerWithStore(store, opts) {
|
|
|
7055
7139
|
});
|
|
7056
7140
|
store.setMetadata("ref_resolution_version", refResolutionVersion);
|
|
7057
7141
|
store.setMetadata("relation_graph_version", relationGraphVersion);
|
|
7142
|
+
const completeProjectScope = !opts.files && (!langs || langs.length === 0) && (!opts.ignore || opts.ignore.length === 0);
|
|
7143
|
+
if (completeProjectScope && discoverySnapshotKey !== void 0) {
|
|
7144
|
+
const finalSnapshot = await findGitSourceFiles(projectRoot, ignore, signal);
|
|
7145
|
+
if (!finalSnapshot || finalSnapshot.snapshotKey !== discoverySnapshotKey) {
|
|
7146
|
+
throw new IndexSourceChangedError(
|
|
7147
|
+
"Project files changed during indexing; retrying before publishing the generation."
|
|
7148
|
+
);
|
|
7149
|
+
}
|
|
7150
|
+
store.setMetadata(GIT_SNAPSHOT_METADATA_KEY, errors.length === 0 ? discoverySnapshotKey : "");
|
|
7151
|
+
}
|
|
7058
7152
|
if (!opts.files || filesIndexed >= 50) store.optimize();
|
|
7059
7153
|
store.setLastIndexed(Date.now());
|
|
7060
|
-
if (!opts.files) store.compactIfNeeded();
|
|
7061
7154
|
const durationMs = Date.now() - startMs;
|
|
7062
7155
|
return {
|
|
7063
7156
|
filesIndexed,
|
|
@@ -7168,20 +7261,30 @@ function outgoingCallsService(args) {
|
|
|
7168
7261
|
if (!parentPort) throw new Error("codebase-index worker must be started as a worker thread");
|
|
7169
7262
|
var port = parentPort;
|
|
7170
7263
|
var inFlight = /* @__PURE__ */ new Map();
|
|
7264
|
+
var activeIndexes = 0;
|
|
7171
7265
|
function post(msg) {
|
|
7172
7266
|
port.postMessage(msg);
|
|
7173
7267
|
}
|
|
7174
7268
|
async function dispatch2(msg) {
|
|
7269
|
+
if (msg.op !== "index" && activeIndexes > 0) {
|
|
7270
|
+
const error = new Error(
|
|
7271
|
+
"Codebase index refresh in progress; retry after the completed generation is published."
|
|
7272
|
+
);
|
|
7273
|
+
error.name = "IndexRefreshInProgressError";
|
|
7274
|
+
throw error;
|
|
7275
|
+
}
|
|
7175
7276
|
switch (msg.op) {
|
|
7176
7277
|
case "index": {
|
|
7177
7278
|
const ac = new AbortController();
|
|
7178
7279
|
inFlight.set(msg.id, ac);
|
|
7280
|
+
activeIndexes++;
|
|
7179
7281
|
try {
|
|
7180
7282
|
return await indexService(msg.args, {
|
|
7181
7283
|
signal: ac.signal,
|
|
7182
7284
|
onProgress: (current, total) => post({ type: "progress", id: msg.id, current, total })
|
|
7183
7285
|
});
|
|
7184
7286
|
} finally {
|
|
7287
|
+
activeIndexes--;
|
|
7185
7288
|
inFlight.delete(msg.id);
|
|
7186
7289
|
}
|
|
7187
7290
|
}
|
|
@@ -5,6 +5,14 @@ export { codebaseIndexDirOverride, resolveIndexDir } from './writer-helpers.js';
|
|
|
5
5
|
export { StorePool } from './writer-store-pool.js';
|
|
6
6
|
export declare class IndexStore {
|
|
7
7
|
private db;
|
|
8
|
+
/**
|
|
9
|
+
* True while an index run owns one outer SQLite transaction. Individual
|
|
10
|
+
* writer methods normally protect themselves with BEGIN/COMMIT, but during
|
|
11
|
+
* a refresh they join this transaction so readers observe either the last
|
|
12
|
+
* completed index or the next completed index, never an in-between batch.
|
|
13
|
+
*/
|
|
14
|
+
private atomicIndexUpdateActive;
|
|
15
|
+
private writeSavepointSequence;
|
|
8
16
|
/** Absolute path to this project's index directory. */
|
|
9
17
|
private readonly indexDir;
|
|
10
18
|
/**
|
|
@@ -62,6 +70,16 @@ export declare class IndexStore {
|
|
|
62
70
|
indexDir?: string | undefined;
|
|
63
71
|
});
|
|
64
72
|
runWithRetry<T>(fn: () => T): T;
|
|
73
|
+
/** Run a complete index mutation as one WAL-visible publication. */
|
|
74
|
+
runAtomicIndexUpdate<T>(job: () => Promise<T>): Promise<T>;
|
|
75
|
+
/**
|
|
76
|
+
* Begin a method-local transaction. Inside an atomic index publication a
|
|
77
|
+
* SAVEPOINT preserves the old per-batch rollback boundary, which is needed
|
|
78
|
+
* when commitBatch falls back to per-file writes after one batch fails.
|
|
79
|
+
*/
|
|
80
|
+
private beginWriteTransaction;
|
|
81
|
+
private commitWriteTransaction;
|
|
82
|
+
private rollbackWriteTransaction;
|
|
65
83
|
/**
|
|
66
84
|
* Mirror the in-process language→family map into SQLite.
|
|
67
85
|
*
|