@wrongstack/tools 0.306.0 → 0.306.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/builtin.js +256 -137
- package/dist/codebase-index/index.js +241 -137
- package/dist/codebase-index/project-server.js +233 -124
- package/dist/codebase-index/worker.js +210 -107
- package/dist/codebase-index/writer.d.ts +18 -0
- package/dist/index.js +258 -138
- package/dist/pack.js +256 -137
- package/dist/plan.js +15 -0
- package/dist/read.js +210 -110
- package/dist/session-kanban.js +3 -1
- package/dist/task.js +15 -0
- package/dist/todo.js +15 -0
- package/dist/tool-tier.js +256 -137
- package/package.json +4 -4
|
@@ -2707,17 +2707,19 @@ import { randomBytes } from "node:crypto";
|
|
|
2707
2707
|
import * as fs11 from "node:fs";
|
|
2708
2708
|
import * as net from "node:net";
|
|
2709
2709
|
import * as path14 from "node:path";
|
|
2710
|
+
import { restrictFilePermissions } from "@wrongstack/core/security";
|
|
2710
2711
|
import {
|
|
2711
2712
|
DEFAULT_WALK_IGNORE_SET,
|
|
2712
2713
|
startSharedHeapWatchdog,
|
|
2713
2714
|
useDaemonPerfDefaults,
|
|
2714
2715
|
watchProjectTree
|
|
2715
2716
|
} from "@wrongstack/core/utils";
|
|
2716
|
-
import { bindProjectEndpoint } from "@wrongstack/persistence";
|
|
2717
|
+
import { atomicWrite, bindProjectEndpoint } from "@wrongstack/persistence";
|
|
2717
2718
|
|
|
2718
2719
|
// src/codebase-index/indexer.ts
|
|
2719
2720
|
import { expectDefined as expectDefined5 } from "@wrongstack/core/utils";
|
|
2720
2721
|
import { execFile } from "node:child_process";
|
|
2722
|
+
import { createHash } from "node:crypto";
|
|
2721
2723
|
import * as fs9 from "node:fs/promises";
|
|
2722
2724
|
import { availableParallelism } from "node:os";
|
|
2723
2725
|
import * as path12 from "node:path";
|
|
@@ -4114,6 +4116,85 @@ function runSqliteWithRetry(fn) {
|
|
|
4114
4116
|
throw lastError;
|
|
4115
4117
|
}
|
|
4116
4118
|
|
|
4119
|
+
// src/codebase-index/vector-search.ts
|
|
4120
|
+
var RRF_K = 60;
|
|
4121
|
+
var VECTOR_DIMENSIONS = 384;
|
|
4122
|
+
var NGRAM_SIZE = 3;
|
|
4123
|
+
function embedText(text) {
|
|
4124
|
+
const vec = new Float32Array(VECTOR_DIMENSIONS);
|
|
4125
|
+
const normalized = text.toLowerCase().trim();
|
|
4126
|
+
if (normalized.length < NGRAM_SIZE) {
|
|
4127
|
+
const padded = ` ${normalized} `.slice(0, Math.max(NGRAM_SIZE, normalized.length + 2));
|
|
4128
|
+
for (let i = 0; i <= padded.length - NGRAM_SIZE; i++) {
|
|
4129
|
+
const ngram = padded.slice(i, i + NGRAM_SIZE);
|
|
4130
|
+
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
4131
|
+
vec[bucket] += 1;
|
|
4132
|
+
}
|
|
4133
|
+
} else {
|
|
4134
|
+
for (let i = 0; i <= normalized.length - NGRAM_SIZE; i++) {
|
|
4135
|
+
const ngram = normalized.slice(i, i + NGRAM_SIZE);
|
|
4136
|
+
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
4137
|
+
vec[bucket] += 1;
|
|
4138
|
+
}
|
|
4139
|
+
}
|
|
4140
|
+
let norm = 0;
|
|
4141
|
+
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
4142
|
+
norm += vec[i] * vec[i];
|
|
4143
|
+
}
|
|
4144
|
+
norm = Math.sqrt(norm);
|
|
4145
|
+
if (norm > 0) {
|
|
4146
|
+
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
4147
|
+
vec[i] /= norm;
|
|
4148
|
+
}
|
|
4149
|
+
}
|
|
4150
|
+
return vec;
|
|
4151
|
+
}
|
|
4152
|
+
function hashNgram(str) {
|
|
4153
|
+
let hash = 2166136261;
|
|
4154
|
+
for (let i = 0; i < str.length; i++) {
|
|
4155
|
+
hash ^= str.charCodeAt(i);
|
|
4156
|
+
hash = Math.imul(hash, 16777619);
|
|
4157
|
+
}
|
|
4158
|
+
return hash >>> 0;
|
|
4159
|
+
}
|
|
4160
|
+
function cosineSimilarity(a, b) {
|
|
4161
|
+
let dot = 0;
|
|
4162
|
+
const len = Math.min(a.length, b.length);
|
|
4163
|
+
for (let i = 0; i < len; i++) {
|
|
4164
|
+
dot += a[i] * b[i];
|
|
4165
|
+
}
|
|
4166
|
+
return dot;
|
|
4167
|
+
}
|
|
4168
|
+
function encodeVector(vec) {
|
|
4169
|
+
return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
|
|
4170
|
+
}
|
|
4171
|
+
function decodeVector(buf) {
|
|
4172
|
+
const view = new DataView(
|
|
4173
|
+
buf.buffer,
|
|
4174
|
+
buf.byteOffset,
|
|
4175
|
+
buf.byteLength
|
|
4176
|
+
);
|
|
4177
|
+
const copy = new Float32Array(buf.byteLength / 4);
|
|
4178
|
+
for (let i = 0; i < copy.length; i++) {
|
|
4179
|
+
copy[i] = view.getFloat32(i * 4, true);
|
|
4180
|
+
}
|
|
4181
|
+
return copy;
|
|
4182
|
+
}
|
|
4183
|
+
function reciprocalRankFusion(bm25Ranks, vectorRanks, k = RRF_K) {
|
|
4184
|
+
const allIds = /* @__PURE__ */ new Set([...bm25Ranks.keys(), ...vectorRanks.keys()]);
|
|
4185
|
+
const scored = [];
|
|
4186
|
+
for (const id of allIds) {
|
|
4187
|
+
const bm25Rank = bm25Ranks.get(id);
|
|
4188
|
+
const vecRank = vectorRanks.get(id);
|
|
4189
|
+
let score = 0;
|
|
4190
|
+
if (bm25Rank !== void 0) score += 1 / (k + bm25Rank);
|
|
4191
|
+
if (vecRank !== void 0) score += 1 / (k + vecRank);
|
|
4192
|
+
scored.push([id, score]);
|
|
4193
|
+
}
|
|
4194
|
+
scored.sort((a, b) => b[1] - a[1]);
|
|
4195
|
+
return scored;
|
|
4196
|
+
}
|
|
4197
|
+
|
|
4117
4198
|
// src/codebase-index/writer-admin.ts
|
|
4118
4199
|
import * as fs7 from "node:fs";
|
|
4119
4200
|
import * as path10 from "node:path";
|
|
@@ -5169,90 +5250,19 @@ var StorePool = class {
|
|
|
5169
5250
|
}
|
|
5170
5251
|
};
|
|
5171
5252
|
|
|
5172
|
-
// src/codebase-index/vector-search.ts
|
|
5173
|
-
var RRF_K = 60;
|
|
5174
|
-
var VECTOR_DIMENSIONS = 384;
|
|
5175
|
-
var NGRAM_SIZE = 3;
|
|
5176
|
-
function embedText(text) {
|
|
5177
|
-
const vec = new Float32Array(VECTOR_DIMENSIONS);
|
|
5178
|
-
const normalized = text.toLowerCase().trim();
|
|
5179
|
-
if (normalized.length < NGRAM_SIZE) {
|
|
5180
|
-
const padded = ` ${normalized} `.slice(0, Math.max(NGRAM_SIZE, normalized.length + 2));
|
|
5181
|
-
for (let i = 0; i <= padded.length - NGRAM_SIZE; i++) {
|
|
5182
|
-
const ngram = padded.slice(i, i + NGRAM_SIZE);
|
|
5183
|
-
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
5184
|
-
vec[bucket] += 1;
|
|
5185
|
-
}
|
|
5186
|
-
} else {
|
|
5187
|
-
for (let i = 0; i <= normalized.length - NGRAM_SIZE; i++) {
|
|
5188
|
-
const ngram = normalized.slice(i, i + NGRAM_SIZE);
|
|
5189
|
-
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
5190
|
-
vec[bucket] += 1;
|
|
5191
|
-
}
|
|
5192
|
-
}
|
|
5193
|
-
let norm = 0;
|
|
5194
|
-
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
5195
|
-
norm += vec[i] * vec[i];
|
|
5196
|
-
}
|
|
5197
|
-
norm = Math.sqrt(norm);
|
|
5198
|
-
if (norm > 0) {
|
|
5199
|
-
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
5200
|
-
vec[i] /= norm;
|
|
5201
|
-
}
|
|
5202
|
-
}
|
|
5203
|
-
return vec;
|
|
5204
|
-
}
|
|
5205
|
-
function hashNgram(str) {
|
|
5206
|
-
let hash = 2166136261;
|
|
5207
|
-
for (let i = 0; i < str.length; i++) {
|
|
5208
|
-
hash ^= str.charCodeAt(i);
|
|
5209
|
-
hash = Math.imul(hash, 16777619);
|
|
5210
|
-
}
|
|
5211
|
-
return hash >>> 0;
|
|
5212
|
-
}
|
|
5213
|
-
function cosineSimilarity(a, b) {
|
|
5214
|
-
let dot = 0;
|
|
5215
|
-
const len = Math.min(a.length, b.length);
|
|
5216
|
-
for (let i = 0; i < len; i++) {
|
|
5217
|
-
dot += a[i] * b[i];
|
|
5218
|
-
}
|
|
5219
|
-
return dot;
|
|
5220
|
-
}
|
|
5221
|
-
function encodeVector(vec) {
|
|
5222
|
-
return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
|
|
5223
|
-
}
|
|
5224
|
-
function decodeVector(buf) {
|
|
5225
|
-
const view = new DataView(
|
|
5226
|
-
buf.buffer,
|
|
5227
|
-
buf.byteOffset,
|
|
5228
|
-
buf.byteLength
|
|
5229
|
-
);
|
|
5230
|
-
const copy = new Float32Array(buf.byteLength / 4);
|
|
5231
|
-
for (let i = 0; i < copy.length; i++) {
|
|
5232
|
-
copy[i] = view.getFloat32(i * 4, true);
|
|
5233
|
-
}
|
|
5234
|
-
return copy;
|
|
5235
|
-
}
|
|
5236
|
-
function reciprocalRankFusion(bm25Ranks, vectorRanks, k = RRF_K) {
|
|
5237
|
-
const allIds = /* @__PURE__ */ new Set([...bm25Ranks.keys(), ...vectorRanks.keys()]);
|
|
5238
|
-
const scored = [];
|
|
5239
|
-
for (const id of allIds) {
|
|
5240
|
-
const bm25Rank = bm25Ranks.get(id);
|
|
5241
|
-
const vecRank = vectorRanks.get(id);
|
|
5242
|
-
let score = 0;
|
|
5243
|
-
if (bm25Rank !== void 0) score += 1 / (k + bm25Rank);
|
|
5244
|
-
if (vecRank !== void 0) score += 1 / (k + vecRank);
|
|
5245
|
-
scored.push([id, score]);
|
|
5246
|
-
}
|
|
5247
|
-
scored.sort((a, b) => b[1] - a[1]);
|
|
5248
|
-
return scored;
|
|
5249
|
-
}
|
|
5250
|
-
|
|
5251
5253
|
// src/codebase-index/writer.ts
|
|
5252
5254
|
var DB_FILE2 = "index.db";
|
|
5253
5255
|
var MAX_STATEMENT_CACHE = 128;
|
|
5254
5256
|
var IndexStore = class _IndexStore {
|
|
5255
5257
|
db;
|
|
5258
|
+
/**
|
|
5259
|
+
* True while an index run owns one outer SQLite transaction. Individual
|
|
5260
|
+
* writer methods normally protect themselves with BEGIN/COMMIT, but during
|
|
5261
|
+
* a refresh they join this transaction so readers observe either the last
|
|
5262
|
+
* completed index or the next completed index, never an in-between batch.
|
|
5263
|
+
*/
|
|
5264
|
+
atomicIndexUpdateActive = false;
|
|
5265
|
+
writeSavepointSequence = 0;
|
|
5256
5266
|
/** Absolute path to this project's index directory. */
|
|
5257
5267
|
indexDir;
|
|
5258
5268
|
/**
|
|
@@ -5333,6 +5343,51 @@ var IndexStore = class _IndexStore {
|
|
|
5333
5343
|
runWithRetry(fn) {
|
|
5334
5344
|
return runSqliteWithRetry(fn);
|
|
5335
5345
|
}
|
|
5346
|
+
/** Run a complete index mutation as one WAL-visible publication. */
|
|
5347
|
+
async runAtomicIndexUpdate(job) {
|
|
5348
|
+
if (this.atomicIndexUpdateActive) return job();
|
|
5349
|
+
this.runWithRetry(() => this.db.exec("BEGIN IMMEDIATE"));
|
|
5350
|
+
this.atomicIndexUpdateActive = true;
|
|
5351
|
+
try {
|
|
5352
|
+
const result = await job();
|
|
5353
|
+
this.db.exec("COMMIT");
|
|
5354
|
+
return result;
|
|
5355
|
+
} catch (error) {
|
|
5356
|
+
try {
|
|
5357
|
+
this.db.exec("ROLLBACK");
|
|
5358
|
+
} catch {
|
|
5359
|
+
}
|
|
5360
|
+
throw error;
|
|
5361
|
+
} finally {
|
|
5362
|
+
this.atomicIndexUpdateActive = false;
|
|
5363
|
+
}
|
|
5364
|
+
}
|
|
5365
|
+
/**
|
|
5366
|
+
* Begin a method-local transaction. Inside an atomic index publication a
|
|
5367
|
+
* SAVEPOINT preserves the old per-batch rollback boundary, which is needed
|
|
5368
|
+
* when commitBatch falls back to per-file writes after one batch fails.
|
|
5369
|
+
*/
|
|
5370
|
+
beginWriteTransaction() {
|
|
5371
|
+
if (this.atomicIndexUpdateActive) {
|
|
5372
|
+
const savepoint = `index_write_${++this.writeSavepointSequence}`;
|
|
5373
|
+
this.db.exec(`SAVEPOINT ${savepoint}`);
|
|
5374
|
+
return savepoint;
|
|
5375
|
+
}
|
|
5376
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
5377
|
+
return null;
|
|
5378
|
+
}
|
|
5379
|
+
commitWriteTransaction(savepoint) {
|
|
5380
|
+
if (savepoint) this.db.exec(`RELEASE SAVEPOINT ${savepoint}`);
|
|
5381
|
+
else this.db.exec("COMMIT");
|
|
5382
|
+
}
|
|
5383
|
+
rollbackWriteTransaction(savepoint) {
|
|
5384
|
+
if (savepoint) {
|
|
5385
|
+
this.db.exec(`ROLLBACK TO SAVEPOINT ${savepoint}`);
|
|
5386
|
+
this.db.exec(`RELEASE SAVEPOINT ${savepoint}`);
|
|
5387
|
+
} else {
|
|
5388
|
+
this.db.exec("ROLLBACK");
|
|
5389
|
+
}
|
|
5390
|
+
}
|
|
5336
5391
|
/**
|
|
5337
5392
|
* Mirror the in-process language→family map into SQLite.
|
|
5338
5393
|
*
|
|
@@ -5567,7 +5622,7 @@ var IndexStore = class _IndexStore {
|
|
|
5567
5622
|
insertSymbols(symbols) {
|
|
5568
5623
|
this.invalidateBm25();
|
|
5569
5624
|
return this.runWithRetry(() => {
|
|
5570
|
-
this.
|
|
5625
|
+
const ownsTransaction = this.beginWriteTransaction();
|
|
5571
5626
|
try {
|
|
5572
5627
|
let nextId = this.allocateSymbolIds(symbols.length);
|
|
5573
5628
|
const result = [];
|
|
@@ -5594,7 +5649,9 @@ var IndexStore = class _IndexStore {
|
|
|
5594
5649
|
}
|
|
5595
5650
|
vectorRows.push({
|
|
5596
5651
|
id,
|
|
5597
|
-
vector: encodeVector(
|
|
5652
|
+
vector: encodeVector(
|
|
5653
|
+
embedText(s.text || buildIndexableText(s.name, s.signature, s.docComment))
|
|
5654
|
+
)
|
|
5598
5655
|
});
|
|
5599
5656
|
result.push({ ...s, id });
|
|
5600
5657
|
}
|
|
@@ -5612,10 +5669,10 @@ var IndexStore = class _IndexStore {
|
|
|
5612
5669
|
vectorRows
|
|
5613
5670
|
);
|
|
5614
5671
|
}
|
|
5615
|
-
this.
|
|
5672
|
+
this.commitWriteTransaction(ownsTransaction);
|
|
5616
5673
|
return result;
|
|
5617
5674
|
} catch (err) {
|
|
5618
|
-
this.
|
|
5675
|
+
this.rollbackWriteTransaction(ownsTransaction);
|
|
5619
5676
|
throw err;
|
|
5620
5677
|
}
|
|
5621
5678
|
});
|
|
@@ -5623,7 +5680,7 @@ var IndexStore = class _IndexStore {
|
|
|
5623
5680
|
deleteSymbolsForFile(file) {
|
|
5624
5681
|
this.invalidateBm25();
|
|
5625
5682
|
this.runWithRetry(() => {
|
|
5626
|
-
this.
|
|
5683
|
+
const ownsTransaction = this.beginWriteTransaction();
|
|
5627
5684
|
try {
|
|
5628
5685
|
const affectedNames = this.invalidateIncomingRefsForFiles([file]);
|
|
5629
5686
|
if (this.ftsAvailable) {
|
|
@@ -5638,9 +5695,9 @@ var IndexStore = class _IndexStore {
|
|
|
5638
5695
|
}
|
|
5639
5696
|
this.stmt("DELETE FROM symbols WHERE file_fk = ?").run(file);
|
|
5640
5697
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
5641
|
-
this.
|
|
5698
|
+
this.commitWriteTransaction(ownsTransaction);
|
|
5642
5699
|
} catch (error) {
|
|
5643
|
-
this.
|
|
5700
|
+
this.rollbackWriteTransaction(ownsTransaction);
|
|
5644
5701
|
throw error;
|
|
5645
5702
|
}
|
|
5646
5703
|
});
|
|
@@ -5653,7 +5710,7 @@ var IndexStore = class _IndexStore {
|
|
|
5653
5710
|
deleteFile(file) {
|
|
5654
5711
|
this.invalidateBm25();
|
|
5655
5712
|
this.runWithRetry(() => {
|
|
5656
|
-
this.
|
|
5713
|
+
const ownsTransaction = this.beginWriteTransaction();
|
|
5657
5714
|
try {
|
|
5658
5715
|
const affectedNames = this.invalidateIncomingRefsForFiles([file]);
|
|
5659
5716
|
if (this.ftsAvailable) {
|
|
@@ -5672,9 +5729,9 @@ var IndexStore = class _IndexStore {
|
|
|
5672
5729
|
this.stmt("DELETE FROM symbols WHERE file_fk = ?").run(file);
|
|
5673
5730
|
this.stmt("DELETE FROM files WHERE file = ?").run(file);
|
|
5674
5731
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
5675
|
-
this.
|
|
5732
|
+
this.commitWriteTransaction(ownsTransaction);
|
|
5676
5733
|
} catch (err) {
|
|
5677
|
-
this.
|
|
5734
|
+
this.rollbackWriteTransaction(ownsTransaction);
|
|
5678
5735
|
throw err;
|
|
5679
5736
|
}
|
|
5680
5737
|
});
|
|
@@ -6061,7 +6118,7 @@ var IndexStore = class _IndexStore {
|
|
|
6061
6118
|
clearAll() {
|
|
6062
6119
|
this.invalidateBm25();
|
|
6063
6120
|
this.runWithRetry(() => {
|
|
6064
|
-
this.
|
|
6121
|
+
const ownsTransaction = this.beginWriteTransaction();
|
|
6065
6122
|
try {
|
|
6066
6123
|
this.db.exec("DROP TABLE IF EXISTS refs");
|
|
6067
6124
|
this.db.exec("DROP TABLE IF EXISTS symbols");
|
|
@@ -6069,15 +6126,15 @@ var IndexStore = class _IndexStore {
|
|
|
6069
6126
|
this.db.exec("DROP TABLE IF EXISTS metadata");
|
|
6070
6127
|
if (this.ftsAvailable) this.db.exec("DROP TABLE IF EXISTS symbols_fts");
|
|
6071
6128
|
this.db.exec("DROP TABLE IF EXISTS symbol_vectors");
|
|
6072
|
-
this.db.exec("COMMIT");
|
|
6073
6129
|
this.stmtCache.clear();
|
|
6074
6130
|
this.initSchema();
|
|
6075
6131
|
this.stmt("INSERT OR REPLACE INTO metadata(key, value) VALUES (?, ?)").run(
|
|
6076
6132
|
_IndexStore.NEXT_SYMBOL_ID_KEY,
|
|
6077
6133
|
"1"
|
|
6078
6134
|
);
|
|
6135
|
+
this.commitWriteTransaction(ownsTransaction);
|
|
6079
6136
|
} catch (err) {
|
|
6080
|
-
this.
|
|
6137
|
+
this.rollbackWriteTransaction(ownsTransaction);
|
|
6081
6138
|
throw err;
|
|
6082
6139
|
}
|
|
6083
6140
|
});
|
|
@@ -6139,7 +6196,7 @@ var IndexStore = class _IndexStore {
|
|
|
6139
6196
|
}
|
|
6140
6197
|
this.invalidateBm25();
|
|
6141
6198
|
return this.runWithRetry(() => {
|
|
6142
|
-
this.
|
|
6199
|
+
const ownsTransaction = this.beginWriteTransaction();
|
|
6143
6200
|
try {
|
|
6144
6201
|
const affectedNames = /* @__PURE__ */ new Set();
|
|
6145
6202
|
for (const entry of entries) {
|
|
@@ -6200,7 +6257,9 @@ var IndexStore = class _IndexStore {
|
|
|
6200
6257
|
}
|
|
6201
6258
|
vectorRows.push({
|
|
6202
6259
|
id,
|
|
6203
|
-
vector: encodeVector(
|
|
6260
|
+
vector: encodeVector(
|
|
6261
|
+
embedText(s.text || buildIndexableText(s.name, s.signature, s.docComment))
|
|
6262
|
+
)
|
|
6204
6263
|
});
|
|
6205
6264
|
const inserted = { ...s, id };
|
|
6206
6265
|
allInserted.push(inserted);
|
|
@@ -6245,10 +6304,10 @@ var IndexStore = class _IndexStore {
|
|
|
6245
6304
|
);
|
|
6246
6305
|
}
|
|
6247
6306
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
6248
|
-
this.
|
|
6307
|
+
this.commitWriteTransaction(ownsTransaction);
|
|
6249
6308
|
return allInserted;
|
|
6250
6309
|
} catch (err) {
|
|
6251
|
-
this.
|
|
6310
|
+
this.rollbackWriteTransaction(ownsTransaction);
|
|
6252
6311
|
throw err;
|
|
6253
6312
|
}
|
|
6254
6313
|
});
|
|
@@ -6325,7 +6384,7 @@ var IndexStore = class _IndexStore {
|
|
|
6325
6384
|
replaceEmptyFile(meta) {
|
|
6326
6385
|
this.invalidateBm25();
|
|
6327
6386
|
this.runWithRetry(() => {
|
|
6328
|
-
this.
|
|
6387
|
+
const ownsTransaction = this.beginWriteTransaction();
|
|
6329
6388
|
try {
|
|
6330
6389
|
const affectedNames = this.invalidateIncomingRefsForFiles([meta.file]);
|
|
6331
6390
|
if (this.ftsAvailable) {
|
|
@@ -6360,9 +6419,9 @@ var IndexStore = class _IndexStore {
|
|
|
6360
6419
|
meta.lastIndexed
|
|
6361
6420
|
);
|
|
6362
6421
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
6363
|
-
this.
|
|
6422
|
+
this.commitWriteTransaction(ownsTransaction);
|
|
6364
6423
|
} catch (err) {
|
|
6365
|
-
this.
|
|
6424
|
+
this.rollbackWriteTransaction(ownsTransaction);
|
|
6366
6425
|
throw err;
|
|
6367
6426
|
}
|
|
6368
6427
|
});
|
|
@@ -6555,6 +6614,10 @@ var DEFAULT_IGNORE_FILES = /* @__PURE__ */ new Set(["package-lock.json", "pnpm-l
|
|
|
6555
6614
|
var INDEXABLE_EXTENSION_SET = new Set(INDEXABLE_EXTENSIONS);
|
|
6556
6615
|
var MAX_INDEX_FILE_BYTES = 5 * 1024 * 1024;
|
|
6557
6616
|
var MAX_GIT_FILE_LIST_BYTES = 64 * 1024 * 1024;
|
|
6617
|
+
var GIT_SNAPSHOT_METADATA_KEY = "git_discovery_snapshot";
|
|
6618
|
+
var IndexSourceChangedError = class extends Error {
|
|
6619
|
+
name = "IndexSourceChangedError";
|
|
6620
|
+
};
|
|
6558
6621
|
function isWithinProject(projectRoot2, file) {
|
|
6559
6622
|
const rel = path12.relative(projectRoot2, file);
|
|
6560
6623
|
return rel !== "" && !rel.startsWith(`..${path12.sep}`) && rel !== ".." && !path12.isAbsolute(rel);
|
|
@@ -6591,7 +6654,7 @@ async function findGitSourceFiles(projectRoot2, ignore, signal) {
|
|
|
6591
6654
|
if (normalizeComparablePath(topLevel) !== normalizeComparablePath(projectRoot2)) return null;
|
|
6592
6655
|
throwIfAborted(signal);
|
|
6593
6656
|
const ignoreSet = /* @__PURE__ */ new Set([...DEFAULT_IGNORE, ...ignore]);
|
|
6594
|
-
const [output, statusOutput] = await Promise.all([
|
|
6657
|
+
const [output, statusOutput, stagedOutput] = await Promise.all([
|
|
6595
6658
|
gitOutput(projectRoot2, ["ls-files", "--cached", "--others", "--exclude-standard", "-z"]),
|
|
6596
6659
|
gitOutput(projectRoot2, [
|
|
6597
6660
|
"status",
|
|
@@ -6599,7 +6662,8 @@ async function findGitSourceFiles(projectRoot2, ignore, signal) {
|
|
|
6599
6662
|
"-z",
|
|
6600
6663
|
"--untracked-files=all",
|
|
6601
6664
|
"--ignored=no"
|
|
6602
|
-
])
|
|
6665
|
+
]),
|
|
6666
|
+
gitOutput(projectRoot2, ["ls-files", "--stage", "-z"])
|
|
6603
6667
|
]);
|
|
6604
6668
|
throwIfAborted(signal);
|
|
6605
6669
|
const dirty = /* @__PURE__ */ new Set();
|
|
@@ -6629,9 +6693,17 @@ async function findGitSourceFiles(projectRoot2, ignore, signal) {
|
|
|
6629
6693
|
const ext = path12.extname(relative3).toLowerCase();
|
|
6630
6694
|
if (INDEXABLE_EXTENSION_SET.has(ext) || detectLang(full) !== null) files.push(full);
|
|
6631
6695
|
}
|
|
6696
|
+
const snapshot = createHash("sha256").update(stagedOutput).update("\0").update(statusOutput);
|
|
6697
|
+
const indexedFiles = new Set(files);
|
|
6698
|
+
for (const dirtyFile of [...dirty].sort()) {
|
|
6699
|
+
if (!indexedFiles.has(dirtyFile) || deleted.has(dirtyFile)) continue;
|
|
6700
|
+
snapshot.update("\0").update(dirtyFile).update("\0");
|
|
6701
|
+
snapshot.update(xxhash64String(await fs9.readFile(dirtyFile, "utf8")));
|
|
6702
|
+
}
|
|
6632
6703
|
return {
|
|
6633
6704
|
files,
|
|
6634
|
-
trustedUnchanged: new Set(files.filter((file) => !dirty.has(file)))
|
|
6705
|
+
trustedUnchanged: new Set(files.filter((file) => !dirty.has(file))),
|
|
6706
|
+
snapshotKey: snapshot.digest("hex")
|
|
6635
6707
|
};
|
|
6636
6708
|
} catch {
|
|
6637
6709
|
return null;
|
|
@@ -6644,7 +6716,8 @@ async function findSourceFiles(projectRoot2, ignore, isGitIgnored, signal) {
|
|
|
6644
6716
|
files: gitFiles.files,
|
|
6645
6717
|
complete: true,
|
|
6646
6718
|
errors: [],
|
|
6647
|
-
trustedUnchanged: gitFiles.trustedUnchanged
|
|
6719
|
+
trustedUnchanged: gitFiles.trustedUnchanged,
|
|
6720
|
+
snapshotKey: gitFiles.snapshotKey
|
|
6648
6721
|
};
|
|
6649
6722
|
}
|
|
6650
6723
|
const results = [];
|
|
@@ -6731,6 +6804,17 @@ async function resolveProjectRelations(store, projectRoot2, opts) {
|
|
|
6731
6804
|
}
|
|
6732
6805
|
}
|
|
6733
6806
|
async function runIndexerWithStore(store, opts) {
|
|
6807
|
+
let result;
|
|
6808
|
+
try {
|
|
6809
|
+
result = await store.runAtomicIndexUpdate(() => runIndexerAtomic(store, opts));
|
|
6810
|
+
} catch (error) {
|
|
6811
|
+
if (!(error instanceof IndexSourceChangedError)) throw error;
|
|
6812
|
+
result = await store.runAtomicIndexUpdate(() => runIndexerAtomic(store, opts));
|
|
6813
|
+
}
|
|
6814
|
+
if (!opts.files) store.compactIfNeeded();
|
|
6815
|
+
return result;
|
|
6816
|
+
}
|
|
6817
|
+
async function runIndexerAtomic(store, opts) {
|
|
6734
6818
|
const { projectRoot: projectRoot2, langs, ignore = [], signal } = opts;
|
|
6735
6819
|
const relationGraphVersion = "2";
|
|
6736
6820
|
const refResolutionVersion = "2";
|
|
@@ -6750,6 +6834,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
6750
6834
|
let discoveredFiles = null;
|
|
6751
6835
|
let discoveryComplete = true;
|
|
6752
6836
|
let trustedUnchanged;
|
|
6837
|
+
let discoverySnapshotKey;
|
|
6753
6838
|
if (opts.files && opts.files.length > 0) {
|
|
6754
6839
|
files = opts.files.map((f) => path12.resolve(projectRoot2, f)).filter((f) => {
|
|
6755
6840
|
if (!isWithinProject(projectRoot2, f)) return false;
|
|
@@ -6763,6 +6848,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
6763
6848
|
discoveryComplete = discovery.complete;
|
|
6764
6849
|
discoveredFiles = new Set(files);
|
|
6765
6850
|
trustedUnchanged = discovery.trustedUnchanged;
|
|
6851
|
+
discoverySnapshotKey = discovery.snapshotKey;
|
|
6766
6852
|
}
|
|
6767
6853
|
if (langs && langs.length > 0) {
|
|
6768
6854
|
const langSet = new Set(langs);
|
|
@@ -6776,6 +6862,8 @@ async function runIndexerWithStore(store, opts) {
|
|
|
6776
6862
|
if (!force) {
|
|
6777
6863
|
for (const meta of store.getAllFileMetas()) existingMeta.set(meta.file, meta);
|
|
6778
6864
|
}
|
|
6865
|
+
const snapshotTrusted = !force && discoverySnapshotKey !== void 0 && store.getMetadata(GIT_SNAPSHOT_METADATA_KEY) === discoverySnapshotKey;
|
|
6866
|
+
if (!snapshotTrusted) trustedUnchanged = void 0;
|
|
6779
6867
|
const totalFilesForProgress = files.length;
|
|
6780
6868
|
let filesPreSkipped = 0;
|
|
6781
6869
|
if (!force && trustedUnchanged) {
|
|
@@ -6838,9 +6926,6 @@ async function runIndexerWithStore(store, opts) {
|
|
|
6838
6926
|
};
|
|
6839
6927
|
}
|
|
6840
6928
|
const meta = existingMeta.get(file);
|
|
6841
|
-
if (!force && meta && meta.mtimeMs === Math.floor(stat2.mtimeMs)) {
|
|
6842
|
-
return { file, stat: stat2, lang, parsed: null, skippedMeta: meta };
|
|
6843
|
-
}
|
|
6844
6929
|
let content;
|
|
6845
6930
|
try {
|
|
6846
6931
|
content = await fs9.readFile(file, { encoding: "utf8", signal });
|
|
@@ -7069,9 +7154,18 @@ async function runIndexerWithStore(store, opts) {
|
|
|
7069
7154
|
});
|
|
7070
7155
|
store.setMetadata("ref_resolution_version", refResolutionVersion);
|
|
7071
7156
|
store.setMetadata("relation_graph_version", relationGraphVersion);
|
|
7157
|
+
const completeProjectScope = !opts.files && (!langs || langs.length === 0) && (!opts.ignore || opts.ignore.length === 0);
|
|
7158
|
+
if (completeProjectScope && discoverySnapshotKey !== void 0) {
|
|
7159
|
+
const finalSnapshot = await findGitSourceFiles(projectRoot2, ignore, signal);
|
|
7160
|
+
if (!finalSnapshot || finalSnapshot.snapshotKey !== discoverySnapshotKey) {
|
|
7161
|
+
throw new IndexSourceChangedError(
|
|
7162
|
+
"Project files changed during indexing; retrying before publishing the generation."
|
|
7163
|
+
);
|
|
7164
|
+
}
|
|
7165
|
+
store.setMetadata(GIT_SNAPSHOT_METADATA_KEY, errors.length === 0 ? discoverySnapshotKey : "");
|
|
7166
|
+
}
|
|
7072
7167
|
if (!opts.files || filesIndexed >= 50) store.optimize();
|
|
7073
7168
|
store.setLastIndexed(Date.now());
|
|
7074
|
-
if (!opts.files) store.compactIfNeeded();
|
|
7075
7169
|
const durationMs = Date.now() - startMs;
|
|
7076
7170
|
return {
|
|
7077
7171
|
filesIndexed,
|
|
@@ -7214,7 +7308,7 @@ var GenerationLruCache = class {
|
|
|
7214
7308
|
};
|
|
7215
7309
|
|
|
7216
7310
|
// src/codebase-index/project-server-endpoint.ts
|
|
7217
|
-
import { createHash } from "node:crypto";
|
|
7311
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
7218
7312
|
import * as fs10 from "node:fs";
|
|
7219
7313
|
import * as os3 from "node:os";
|
|
7220
7314
|
import * as path13 from "node:path";
|
|
@@ -7232,7 +7326,7 @@ function projectIndexServerBuildId(entrypoint) {
|
|
|
7232
7326
|
if (buildIdCache?.file === file && buildIdCache.mtimeMs === stat2.mtimeMs && buildIdCache.size === stat2.size) {
|
|
7233
7327
|
return buildIdCache.buildId;
|
|
7234
7328
|
}
|
|
7235
|
-
const buildId =
|
|
7329
|
+
const buildId = createHash2("sha256").update(fs10.readFileSync(file)).digest("hex").slice(0, 24);
|
|
7236
7330
|
buildIdCache = { file, mtimeMs: stat2.mtimeMs, size: stat2.size, buildId };
|
|
7237
7331
|
return buildId;
|
|
7238
7332
|
} catch {
|
|
@@ -7245,7 +7339,7 @@ function normalizeLocalPath(value) {
|
|
|
7245
7339
|
}
|
|
7246
7340
|
function projectIndexServerKey(projectRoot2, indexDir2) {
|
|
7247
7341
|
const resolvedIndexDir = normalizeLocalPath(resolveIndexDir(projectRoot2, indexDir2));
|
|
7248
|
-
return
|
|
7342
|
+
return createHash2("sha256").update(resolvedIndexDir).digest("hex").slice(0, 24);
|
|
7249
7343
|
}
|
|
7250
7344
|
function projectIndexServerEndpoint(projectRoot2, indexDir2) {
|
|
7251
7345
|
const key = projectIndexServerKey(projectRoot2, indexDir2);
|
|
@@ -7368,11 +7462,18 @@ function clearQueryCaches() {
|
|
|
7368
7462
|
outgoingCallsCache.clear();
|
|
7369
7463
|
}
|
|
7370
7464
|
function cachedRead(cache, key, load) {
|
|
7465
|
+
if (indexActivity.indexing) {
|
|
7466
|
+
const error = new Error(
|
|
7467
|
+
`Codebase index refresh in progress (${indexActivity.currentFile}/${indexActivity.totalFiles} files); retry after the completed generation is published.`
|
|
7468
|
+
);
|
|
7469
|
+
error.name = "IndexRefreshInProgressError";
|
|
7470
|
+
throw error;
|
|
7471
|
+
}
|
|
7371
7472
|
const generation = indexActivity.generation;
|
|
7372
7473
|
const cached = cache.get(key, generation);
|
|
7373
7474
|
if (cached !== void 0) return cached;
|
|
7374
7475
|
const value = load();
|
|
7375
|
-
return
|
|
7476
|
+
return cache.set(key, generation, value);
|
|
7376
7477
|
}
|
|
7377
7478
|
var MAX_CLIENT_WRITE_BUFFER_BYTES = 8 * 1024 * 1024;
|
|
7378
7479
|
function send(state, message) {
|
|
@@ -7575,12 +7676,22 @@ async function dispatchOperation(state, message) {
|
|
|
7575
7676
|
);
|
|
7576
7677
|
case "incomingCalls": {
|
|
7577
7678
|
const callArgs = fixedArgs(message.args);
|
|
7578
|
-
const cacheKey = JSON.stringify([
|
|
7679
|
+
const cacheKey = JSON.stringify([
|
|
7680
|
+
callArgs.symbol,
|
|
7681
|
+
callArgs.file ?? "",
|
|
7682
|
+
callArgs.limit ?? 100,
|
|
7683
|
+
callArgs.transitive ?? false
|
|
7684
|
+
]);
|
|
7579
7685
|
return cachedRead(incomingCallsCache, cacheKey, () => incomingCallsService(callArgs));
|
|
7580
7686
|
}
|
|
7581
7687
|
case "outgoingCalls": {
|
|
7582
7688
|
const callArgs = fixedArgs(message.args);
|
|
7583
|
-
const cacheKey = JSON.stringify([
|
|
7689
|
+
const cacheKey = JSON.stringify([
|
|
7690
|
+
callArgs.symbol,
|
|
7691
|
+
callArgs.file ?? "",
|
|
7692
|
+
callArgs.limit ?? 100,
|
|
7693
|
+
callArgs.transitive ?? false
|
|
7694
|
+
]);
|
|
7584
7695
|
return cachedRead(outgoingCallsCache, cacheKey, () => outgoingCallsService(callArgs));
|
|
7585
7696
|
}
|
|
7586
7697
|
default:
|
|
@@ -7785,18 +7896,16 @@ function removeMetadataIfOwned() {
|
|
|
7785
7896
|
} catch {
|
|
7786
7897
|
}
|
|
7787
7898
|
}
|
|
7788
|
-
function writeMetadata() {
|
|
7899
|
+
async function writeMetadata() {
|
|
7789
7900
|
fs11.mkdirSync(path14.dirname(metadataPath), { recursive: true });
|
|
7790
|
-
const temporary = `${metadataPath}.${process.pid}.tmp`;
|
|
7791
7901
|
const metadata = { ...serverInfo, authToken };
|
|
7792
|
-
|
|
7902
|
+
await atomicWrite(metadataPath, `${JSON.stringify(metadata, null, 2)}
|
|
7793
7903
|
`, { mode: 384 });
|
|
7794
|
-
|
|
7795
|
-
|
|
7796
|
-
|
|
7797
|
-
|
|
7798
|
-
|
|
7799
|
-
}
|
|
7904
|
+
await restrictFilePermissions(metadataPath, {
|
|
7905
|
+
label: "codebase-index-metadata",
|
|
7906
|
+
warn: (message) => process.stderr.write(`${message}
|
|
7907
|
+
`)
|
|
7908
|
+
});
|
|
7800
7909
|
}
|
|
7801
7910
|
var server = net.createServer((socket) => {
|
|
7802
7911
|
if (idleTimer) clearTimeout(idleTimer);
|
|
@@ -7893,7 +8002,7 @@ void (async () => {
|
|
|
7893
8002
|
`);
|
|
7894
8003
|
process.exitCode = 1;
|
|
7895
8004
|
});
|
|
7896
|
-
writeMetadata();
|
|
8005
|
+
await writeMetadata();
|
|
7897
8006
|
markMetadataWritten?.();
|
|
7898
8007
|
scheduleIdleStop();
|
|
7899
8008
|
})();
|