@wrongstack/tools 0.306.2 → 0.306.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/builtin.js +241 -137
- package/dist/codebase-index/index.js +241 -137
- package/dist/codebase-index/project-server.js +224 -114
- package/dist/codebase-index/worker.js +210 -107
- package/dist/codebase-index/writer.d.ts +18 -0
- package/dist/index.js +241 -137
- package/dist/pack.js +241 -137
- package/dist/read.js +210 -110
- package/dist/tool-tier.js +241 -137
- package/package.json +4 -4
|
@@ -2707,18 +2707,19 @@ import { randomBytes } from "node:crypto";
|
|
|
2707
2707
|
import * as fs11 from "node:fs";
|
|
2708
2708
|
import * as net from "node:net";
|
|
2709
2709
|
import * as path14 from "node:path";
|
|
2710
|
+
import { restrictFilePermissions } from "@wrongstack/core/security";
|
|
2710
2711
|
import {
|
|
2711
2712
|
DEFAULT_WALK_IGNORE_SET,
|
|
2712
2713
|
startSharedHeapWatchdog,
|
|
2713
2714
|
useDaemonPerfDefaults,
|
|
2714
2715
|
watchProjectTree
|
|
2715
2716
|
} from "@wrongstack/core/utils";
|
|
2716
|
-
import { restrictFilePermissions } from "@wrongstack/core/security";
|
|
2717
2717
|
import { atomicWrite, bindProjectEndpoint } from "@wrongstack/persistence";
|
|
2718
2718
|
|
|
2719
2719
|
// src/codebase-index/indexer.ts
|
|
2720
2720
|
import { expectDefined as expectDefined5 } from "@wrongstack/core/utils";
|
|
2721
2721
|
import { execFile } from "node:child_process";
|
|
2722
|
+
import { createHash } from "node:crypto";
|
|
2722
2723
|
import * as fs9 from "node:fs/promises";
|
|
2723
2724
|
import { availableParallelism } from "node:os";
|
|
2724
2725
|
import * as path12 from "node:path";
|
|
@@ -4115,6 +4116,85 @@ function runSqliteWithRetry(fn) {
|
|
|
4115
4116
|
throw lastError;
|
|
4116
4117
|
}
|
|
4117
4118
|
|
|
4119
|
+
// src/codebase-index/vector-search.ts
|
|
4120
|
+
var RRF_K = 60;
|
|
4121
|
+
var VECTOR_DIMENSIONS = 384;
|
|
4122
|
+
var NGRAM_SIZE = 3;
|
|
4123
|
+
function embedText(text) {
|
|
4124
|
+
const vec = new Float32Array(VECTOR_DIMENSIONS);
|
|
4125
|
+
const normalized = text.toLowerCase().trim();
|
|
4126
|
+
if (normalized.length < NGRAM_SIZE) {
|
|
4127
|
+
const padded = ` ${normalized} `.slice(0, Math.max(NGRAM_SIZE, normalized.length + 2));
|
|
4128
|
+
for (let i = 0; i <= padded.length - NGRAM_SIZE; i++) {
|
|
4129
|
+
const ngram = padded.slice(i, i + NGRAM_SIZE);
|
|
4130
|
+
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
4131
|
+
vec[bucket] += 1;
|
|
4132
|
+
}
|
|
4133
|
+
} else {
|
|
4134
|
+
for (let i = 0; i <= normalized.length - NGRAM_SIZE; i++) {
|
|
4135
|
+
const ngram = normalized.slice(i, i + NGRAM_SIZE);
|
|
4136
|
+
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
4137
|
+
vec[bucket] += 1;
|
|
4138
|
+
}
|
|
4139
|
+
}
|
|
4140
|
+
let norm = 0;
|
|
4141
|
+
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
4142
|
+
norm += vec[i] * vec[i];
|
|
4143
|
+
}
|
|
4144
|
+
norm = Math.sqrt(norm);
|
|
4145
|
+
if (norm > 0) {
|
|
4146
|
+
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
4147
|
+
vec[i] /= norm;
|
|
4148
|
+
}
|
|
4149
|
+
}
|
|
4150
|
+
return vec;
|
|
4151
|
+
}
|
|
4152
|
+
function hashNgram(str) {
|
|
4153
|
+
let hash = 2166136261;
|
|
4154
|
+
for (let i = 0; i < str.length; i++) {
|
|
4155
|
+
hash ^= str.charCodeAt(i);
|
|
4156
|
+
hash = Math.imul(hash, 16777619);
|
|
4157
|
+
}
|
|
4158
|
+
return hash >>> 0;
|
|
4159
|
+
}
|
|
4160
|
+
function cosineSimilarity(a, b) {
|
|
4161
|
+
let dot = 0;
|
|
4162
|
+
const len = Math.min(a.length, b.length);
|
|
4163
|
+
for (let i = 0; i < len; i++) {
|
|
4164
|
+
dot += a[i] * b[i];
|
|
4165
|
+
}
|
|
4166
|
+
return dot;
|
|
4167
|
+
}
|
|
4168
|
+
function encodeVector(vec) {
|
|
4169
|
+
return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
|
|
4170
|
+
}
|
|
4171
|
+
function decodeVector(buf) {
|
|
4172
|
+
const view = new DataView(
|
|
4173
|
+
buf.buffer,
|
|
4174
|
+
buf.byteOffset,
|
|
4175
|
+
buf.byteLength
|
|
4176
|
+
);
|
|
4177
|
+
const copy = new Float32Array(buf.byteLength / 4);
|
|
4178
|
+
for (let i = 0; i < copy.length; i++) {
|
|
4179
|
+
copy[i] = view.getFloat32(i * 4, true);
|
|
4180
|
+
}
|
|
4181
|
+
return copy;
|
|
4182
|
+
}
|
|
4183
|
+
function reciprocalRankFusion(bm25Ranks, vectorRanks, k = RRF_K) {
|
|
4184
|
+
const allIds = /* @__PURE__ */ new Set([...bm25Ranks.keys(), ...vectorRanks.keys()]);
|
|
4185
|
+
const scored = [];
|
|
4186
|
+
for (const id of allIds) {
|
|
4187
|
+
const bm25Rank = bm25Ranks.get(id);
|
|
4188
|
+
const vecRank = vectorRanks.get(id);
|
|
4189
|
+
let score = 0;
|
|
4190
|
+
if (bm25Rank !== void 0) score += 1 / (k + bm25Rank);
|
|
4191
|
+
if (vecRank !== void 0) score += 1 / (k + vecRank);
|
|
4192
|
+
scored.push([id, score]);
|
|
4193
|
+
}
|
|
4194
|
+
scored.sort((a, b) => b[1] - a[1]);
|
|
4195
|
+
return scored;
|
|
4196
|
+
}
|
|
4197
|
+
|
|
4118
4198
|
// src/codebase-index/writer-admin.ts
|
|
4119
4199
|
import * as fs7 from "node:fs";
|
|
4120
4200
|
import * as path10 from "node:path";
|
|
@@ -5170,90 +5250,19 @@ var StorePool = class {
|
|
|
5170
5250
|
}
|
|
5171
5251
|
};
|
|
5172
5252
|
|
|
5173
|
-
// src/codebase-index/vector-search.ts
|
|
5174
|
-
var RRF_K = 60;
|
|
5175
|
-
var VECTOR_DIMENSIONS = 384;
|
|
5176
|
-
var NGRAM_SIZE = 3;
|
|
5177
|
-
function embedText(text) {
|
|
5178
|
-
const vec = new Float32Array(VECTOR_DIMENSIONS);
|
|
5179
|
-
const normalized = text.toLowerCase().trim();
|
|
5180
|
-
if (normalized.length < NGRAM_SIZE) {
|
|
5181
|
-
const padded = ` ${normalized} `.slice(0, Math.max(NGRAM_SIZE, normalized.length + 2));
|
|
5182
|
-
for (let i = 0; i <= padded.length - NGRAM_SIZE; i++) {
|
|
5183
|
-
const ngram = padded.slice(i, i + NGRAM_SIZE);
|
|
5184
|
-
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
5185
|
-
vec[bucket] += 1;
|
|
5186
|
-
}
|
|
5187
|
-
} else {
|
|
5188
|
-
for (let i = 0; i <= normalized.length - NGRAM_SIZE; i++) {
|
|
5189
|
-
const ngram = normalized.slice(i, i + NGRAM_SIZE);
|
|
5190
|
-
const bucket = hashNgram(ngram) % VECTOR_DIMENSIONS;
|
|
5191
|
-
vec[bucket] += 1;
|
|
5192
|
-
}
|
|
5193
|
-
}
|
|
5194
|
-
let norm = 0;
|
|
5195
|
-
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
5196
|
-
norm += vec[i] * vec[i];
|
|
5197
|
-
}
|
|
5198
|
-
norm = Math.sqrt(norm);
|
|
5199
|
-
if (norm > 0) {
|
|
5200
|
-
for (let i = 0; i < VECTOR_DIMENSIONS; i++) {
|
|
5201
|
-
vec[i] /= norm;
|
|
5202
|
-
}
|
|
5203
|
-
}
|
|
5204
|
-
return vec;
|
|
5205
|
-
}
|
|
5206
|
-
function hashNgram(str) {
|
|
5207
|
-
let hash = 2166136261;
|
|
5208
|
-
for (let i = 0; i < str.length; i++) {
|
|
5209
|
-
hash ^= str.charCodeAt(i);
|
|
5210
|
-
hash = Math.imul(hash, 16777619);
|
|
5211
|
-
}
|
|
5212
|
-
return hash >>> 0;
|
|
5213
|
-
}
|
|
5214
|
-
function cosineSimilarity(a, b) {
|
|
5215
|
-
let dot = 0;
|
|
5216
|
-
const len = Math.min(a.length, b.length);
|
|
5217
|
-
for (let i = 0; i < len; i++) {
|
|
5218
|
-
dot += a[i] * b[i];
|
|
5219
|
-
}
|
|
5220
|
-
return dot;
|
|
5221
|
-
}
|
|
5222
|
-
function encodeVector(vec) {
|
|
5223
|
-
return Buffer.from(vec.buffer, vec.byteOffset, vec.byteLength);
|
|
5224
|
-
}
|
|
5225
|
-
function decodeVector(buf) {
|
|
5226
|
-
const view = new DataView(
|
|
5227
|
-
buf.buffer,
|
|
5228
|
-
buf.byteOffset,
|
|
5229
|
-
buf.byteLength
|
|
5230
|
-
);
|
|
5231
|
-
const copy = new Float32Array(buf.byteLength / 4);
|
|
5232
|
-
for (let i = 0; i < copy.length; i++) {
|
|
5233
|
-
copy[i] = view.getFloat32(i * 4, true);
|
|
5234
|
-
}
|
|
5235
|
-
return copy;
|
|
5236
|
-
}
|
|
5237
|
-
function reciprocalRankFusion(bm25Ranks, vectorRanks, k = RRF_K) {
|
|
5238
|
-
const allIds = /* @__PURE__ */ new Set([...bm25Ranks.keys(), ...vectorRanks.keys()]);
|
|
5239
|
-
const scored = [];
|
|
5240
|
-
for (const id of allIds) {
|
|
5241
|
-
const bm25Rank = bm25Ranks.get(id);
|
|
5242
|
-
const vecRank = vectorRanks.get(id);
|
|
5243
|
-
let score = 0;
|
|
5244
|
-
if (bm25Rank !== void 0) score += 1 / (k + bm25Rank);
|
|
5245
|
-
if (vecRank !== void 0) score += 1 / (k + vecRank);
|
|
5246
|
-
scored.push([id, score]);
|
|
5247
|
-
}
|
|
5248
|
-
scored.sort((a, b) => b[1] - a[1]);
|
|
5249
|
-
return scored;
|
|
5250
|
-
}
|
|
5251
|
-
|
|
5252
5253
|
// src/codebase-index/writer.ts
|
|
5253
5254
|
var DB_FILE2 = "index.db";
|
|
5254
5255
|
var MAX_STATEMENT_CACHE = 128;
|
|
5255
5256
|
var IndexStore = class _IndexStore {
|
|
5256
5257
|
db;
|
|
5258
|
+
/**
|
|
5259
|
+
* True while an index run owns one outer SQLite transaction. Individual
|
|
5260
|
+
* writer methods normally protect themselves with BEGIN/COMMIT, but during
|
|
5261
|
+
* a refresh they join this transaction so readers observe either the last
|
|
5262
|
+
* completed index or the next completed index, never an in-between batch.
|
|
5263
|
+
*/
|
|
5264
|
+
atomicIndexUpdateActive = false;
|
|
5265
|
+
writeSavepointSequence = 0;
|
|
5257
5266
|
/** Absolute path to this project's index directory. */
|
|
5258
5267
|
indexDir;
|
|
5259
5268
|
/**
|
|
@@ -5334,6 +5343,51 @@ var IndexStore = class _IndexStore {
|
|
|
5334
5343
|
runWithRetry(fn) {
|
|
5335
5344
|
return runSqliteWithRetry(fn);
|
|
5336
5345
|
}
|
|
5346
|
+
/** Run a complete index mutation as one WAL-visible publication. */
|
|
5347
|
+
async runAtomicIndexUpdate(job) {
|
|
5348
|
+
if (this.atomicIndexUpdateActive) return job();
|
|
5349
|
+
this.runWithRetry(() => this.db.exec("BEGIN IMMEDIATE"));
|
|
5350
|
+
this.atomicIndexUpdateActive = true;
|
|
5351
|
+
try {
|
|
5352
|
+
const result = await job();
|
|
5353
|
+
this.db.exec("COMMIT");
|
|
5354
|
+
return result;
|
|
5355
|
+
} catch (error) {
|
|
5356
|
+
try {
|
|
5357
|
+
this.db.exec("ROLLBACK");
|
|
5358
|
+
} catch {
|
|
5359
|
+
}
|
|
5360
|
+
throw error;
|
|
5361
|
+
} finally {
|
|
5362
|
+
this.atomicIndexUpdateActive = false;
|
|
5363
|
+
}
|
|
5364
|
+
}
|
|
5365
|
+
/**
|
|
5366
|
+
* Begin a method-local transaction. Inside an atomic index publication a
|
|
5367
|
+
* SAVEPOINT preserves the old per-batch rollback boundary, which is needed
|
|
5368
|
+
* when commitBatch falls back to per-file writes after one batch fails.
|
|
5369
|
+
*/
|
|
5370
|
+
beginWriteTransaction() {
|
|
5371
|
+
if (this.atomicIndexUpdateActive) {
|
|
5372
|
+
const savepoint = `index_write_${++this.writeSavepointSequence}`;
|
|
5373
|
+
this.db.exec(`SAVEPOINT ${savepoint}`);
|
|
5374
|
+
return savepoint;
|
|
5375
|
+
}
|
|
5376
|
+
this.db.exec("BEGIN IMMEDIATE");
|
|
5377
|
+
return null;
|
|
5378
|
+
}
|
|
5379
|
+
commitWriteTransaction(savepoint) {
|
|
5380
|
+
if (savepoint) this.db.exec(`RELEASE SAVEPOINT ${savepoint}`);
|
|
5381
|
+
else this.db.exec("COMMIT");
|
|
5382
|
+
}
|
|
5383
|
+
rollbackWriteTransaction(savepoint) {
|
|
5384
|
+
if (savepoint) {
|
|
5385
|
+
this.db.exec(`ROLLBACK TO SAVEPOINT ${savepoint}`);
|
|
5386
|
+
this.db.exec(`RELEASE SAVEPOINT ${savepoint}`);
|
|
5387
|
+
} else {
|
|
5388
|
+
this.db.exec("ROLLBACK");
|
|
5389
|
+
}
|
|
5390
|
+
}
|
|
5337
5391
|
/**
|
|
5338
5392
|
* Mirror the in-process language→family map into SQLite.
|
|
5339
5393
|
*
|
|
@@ -5568,7 +5622,7 @@ var IndexStore = class _IndexStore {
|
|
|
5568
5622
|
insertSymbols(symbols) {
|
|
5569
5623
|
this.invalidateBm25();
|
|
5570
5624
|
return this.runWithRetry(() => {
|
|
5571
|
-
this.
|
|
5625
|
+
const ownsTransaction = this.beginWriteTransaction();
|
|
5572
5626
|
try {
|
|
5573
5627
|
let nextId = this.allocateSymbolIds(symbols.length);
|
|
5574
5628
|
const result = [];
|
|
@@ -5595,7 +5649,9 @@ var IndexStore = class _IndexStore {
|
|
|
5595
5649
|
}
|
|
5596
5650
|
vectorRows.push({
|
|
5597
5651
|
id,
|
|
5598
|
-
vector: encodeVector(
|
|
5652
|
+
vector: encodeVector(
|
|
5653
|
+
embedText(s.text || buildIndexableText(s.name, s.signature, s.docComment))
|
|
5654
|
+
)
|
|
5599
5655
|
});
|
|
5600
5656
|
result.push({ ...s, id });
|
|
5601
5657
|
}
|
|
@@ -5613,10 +5669,10 @@ var IndexStore = class _IndexStore {
|
|
|
5613
5669
|
vectorRows
|
|
5614
5670
|
);
|
|
5615
5671
|
}
|
|
5616
|
-
this.
|
|
5672
|
+
this.commitWriteTransaction(ownsTransaction);
|
|
5617
5673
|
return result;
|
|
5618
5674
|
} catch (err) {
|
|
5619
|
-
this.
|
|
5675
|
+
this.rollbackWriteTransaction(ownsTransaction);
|
|
5620
5676
|
throw err;
|
|
5621
5677
|
}
|
|
5622
5678
|
});
|
|
@@ -5624,7 +5680,7 @@ var IndexStore = class _IndexStore {
|
|
|
5624
5680
|
deleteSymbolsForFile(file) {
|
|
5625
5681
|
this.invalidateBm25();
|
|
5626
5682
|
this.runWithRetry(() => {
|
|
5627
|
-
this.
|
|
5683
|
+
const ownsTransaction = this.beginWriteTransaction();
|
|
5628
5684
|
try {
|
|
5629
5685
|
const affectedNames = this.invalidateIncomingRefsForFiles([file]);
|
|
5630
5686
|
if (this.ftsAvailable) {
|
|
@@ -5639,9 +5695,9 @@ var IndexStore = class _IndexStore {
|
|
|
5639
5695
|
}
|
|
5640
5696
|
this.stmt("DELETE FROM symbols WHERE file_fk = ?").run(file);
|
|
5641
5697
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
5642
|
-
this.
|
|
5698
|
+
this.commitWriteTransaction(ownsTransaction);
|
|
5643
5699
|
} catch (error) {
|
|
5644
|
-
this.
|
|
5700
|
+
this.rollbackWriteTransaction(ownsTransaction);
|
|
5645
5701
|
throw error;
|
|
5646
5702
|
}
|
|
5647
5703
|
});
|
|
@@ -5654,7 +5710,7 @@ var IndexStore = class _IndexStore {
|
|
|
5654
5710
|
deleteFile(file) {
|
|
5655
5711
|
this.invalidateBm25();
|
|
5656
5712
|
this.runWithRetry(() => {
|
|
5657
|
-
this.
|
|
5713
|
+
const ownsTransaction = this.beginWriteTransaction();
|
|
5658
5714
|
try {
|
|
5659
5715
|
const affectedNames = this.invalidateIncomingRefsForFiles([file]);
|
|
5660
5716
|
if (this.ftsAvailable) {
|
|
@@ -5673,9 +5729,9 @@ var IndexStore = class _IndexStore {
|
|
|
5673
5729
|
this.stmt("DELETE FROM symbols WHERE file_fk = ?").run(file);
|
|
5674
5730
|
this.stmt("DELETE FROM files WHERE file = ?").run(file);
|
|
5675
5731
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
5676
|
-
this.
|
|
5732
|
+
this.commitWriteTransaction(ownsTransaction);
|
|
5677
5733
|
} catch (err) {
|
|
5678
|
-
this.
|
|
5734
|
+
this.rollbackWriteTransaction(ownsTransaction);
|
|
5679
5735
|
throw err;
|
|
5680
5736
|
}
|
|
5681
5737
|
});
|
|
@@ -6062,7 +6118,7 @@ var IndexStore = class _IndexStore {
|
|
|
6062
6118
|
clearAll() {
|
|
6063
6119
|
this.invalidateBm25();
|
|
6064
6120
|
this.runWithRetry(() => {
|
|
6065
|
-
this.
|
|
6121
|
+
const ownsTransaction = this.beginWriteTransaction();
|
|
6066
6122
|
try {
|
|
6067
6123
|
this.db.exec("DROP TABLE IF EXISTS refs");
|
|
6068
6124
|
this.db.exec("DROP TABLE IF EXISTS symbols");
|
|
@@ -6070,15 +6126,15 @@ var IndexStore = class _IndexStore {
|
|
|
6070
6126
|
this.db.exec("DROP TABLE IF EXISTS metadata");
|
|
6071
6127
|
if (this.ftsAvailable) this.db.exec("DROP TABLE IF EXISTS symbols_fts");
|
|
6072
6128
|
this.db.exec("DROP TABLE IF EXISTS symbol_vectors");
|
|
6073
|
-
this.db.exec("COMMIT");
|
|
6074
6129
|
this.stmtCache.clear();
|
|
6075
6130
|
this.initSchema();
|
|
6076
6131
|
this.stmt("INSERT OR REPLACE INTO metadata(key, value) VALUES (?, ?)").run(
|
|
6077
6132
|
_IndexStore.NEXT_SYMBOL_ID_KEY,
|
|
6078
6133
|
"1"
|
|
6079
6134
|
);
|
|
6135
|
+
this.commitWriteTransaction(ownsTransaction);
|
|
6080
6136
|
} catch (err) {
|
|
6081
|
-
this.
|
|
6137
|
+
this.rollbackWriteTransaction(ownsTransaction);
|
|
6082
6138
|
throw err;
|
|
6083
6139
|
}
|
|
6084
6140
|
});
|
|
@@ -6140,7 +6196,7 @@ var IndexStore = class _IndexStore {
|
|
|
6140
6196
|
}
|
|
6141
6197
|
this.invalidateBm25();
|
|
6142
6198
|
return this.runWithRetry(() => {
|
|
6143
|
-
this.
|
|
6199
|
+
const ownsTransaction = this.beginWriteTransaction();
|
|
6144
6200
|
try {
|
|
6145
6201
|
const affectedNames = /* @__PURE__ */ new Set();
|
|
6146
6202
|
for (const entry of entries) {
|
|
@@ -6201,7 +6257,9 @@ var IndexStore = class _IndexStore {
|
|
|
6201
6257
|
}
|
|
6202
6258
|
vectorRows.push({
|
|
6203
6259
|
id,
|
|
6204
|
-
vector: encodeVector(
|
|
6260
|
+
vector: encodeVector(
|
|
6261
|
+
embedText(s.text || buildIndexableText(s.name, s.signature, s.docComment))
|
|
6262
|
+
)
|
|
6205
6263
|
});
|
|
6206
6264
|
const inserted = { ...s, id };
|
|
6207
6265
|
allInserted.push(inserted);
|
|
@@ -6246,10 +6304,10 @@ var IndexStore = class _IndexStore {
|
|
|
6246
6304
|
);
|
|
6247
6305
|
}
|
|
6248
6306
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
6249
|
-
this.
|
|
6307
|
+
this.commitWriteTransaction(ownsTransaction);
|
|
6250
6308
|
return allInserted;
|
|
6251
6309
|
} catch (err) {
|
|
6252
|
-
this.
|
|
6310
|
+
this.rollbackWriteTransaction(ownsTransaction);
|
|
6253
6311
|
throw err;
|
|
6254
6312
|
}
|
|
6255
6313
|
});
|
|
@@ -6326,7 +6384,7 @@ var IndexStore = class _IndexStore {
|
|
|
6326
6384
|
replaceEmptyFile(meta) {
|
|
6327
6385
|
this.invalidateBm25();
|
|
6328
6386
|
this.runWithRetry(() => {
|
|
6329
|
-
this.
|
|
6387
|
+
const ownsTransaction = this.beginWriteTransaction();
|
|
6330
6388
|
try {
|
|
6331
6389
|
const affectedNames = this.invalidateIncomingRefsForFiles([meta.file]);
|
|
6332
6390
|
if (this.ftsAvailable) {
|
|
@@ -6361,9 +6419,9 @@ var IndexStore = class _IndexStore {
|
|
|
6361
6419
|
meta.lastIndexed
|
|
6362
6420
|
);
|
|
6363
6421
|
this.resolveRefsForNamesUnsafe(affectedNames);
|
|
6364
|
-
this.
|
|
6422
|
+
this.commitWriteTransaction(ownsTransaction);
|
|
6365
6423
|
} catch (err) {
|
|
6366
|
-
this.
|
|
6424
|
+
this.rollbackWriteTransaction(ownsTransaction);
|
|
6367
6425
|
throw err;
|
|
6368
6426
|
}
|
|
6369
6427
|
});
|
|
@@ -6556,6 +6614,10 @@ var DEFAULT_IGNORE_FILES = /* @__PURE__ */ new Set(["package-lock.json", "pnpm-l
|
|
|
6556
6614
|
var INDEXABLE_EXTENSION_SET = new Set(INDEXABLE_EXTENSIONS);
|
|
6557
6615
|
var MAX_INDEX_FILE_BYTES = 5 * 1024 * 1024;
|
|
6558
6616
|
var MAX_GIT_FILE_LIST_BYTES = 64 * 1024 * 1024;
|
|
6617
|
+
var GIT_SNAPSHOT_METADATA_KEY = "git_discovery_snapshot";
|
|
6618
|
+
var IndexSourceChangedError = class extends Error {
|
|
6619
|
+
name = "IndexSourceChangedError";
|
|
6620
|
+
};
|
|
6559
6621
|
function isWithinProject(projectRoot2, file) {
|
|
6560
6622
|
const rel = path12.relative(projectRoot2, file);
|
|
6561
6623
|
return rel !== "" && !rel.startsWith(`..${path12.sep}`) && rel !== ".." && !path12.isAbsolute(rel);
|
|
@@ -6592,7 +6654,7 @@ async function findGitSourceFiles(projectRoot2, ignore, signal) {
|
|
|
6592
6654
|
if (normalizeComparablePath(topLevel) !== normalizeComparablePath(projectRoot2)) return null;
|
|
6593
6655
|
throwIfAborted(signal);
|
|
6594
6656
|
const ignoreSet = /* @__PURE__ */ new Set([...DEFAULT_IGNORE, ...ignore]);
|
|
6595
|
-
const [output, statusOutput] = await Promise.all([
|
|
6657
|
+
const [output, statusOutput, stagedOutput] = await Promise.all([
|
|
6596
6658
|
gitOutput(projectRoot2, ["ls-files", "--cached", "--others", "--exclude-standard", "-z"]),
|
|
6597
6659
|
gitOutput(projectRoot2, [
|
|
6598
6660
|
"status",
|
|
@@ -6600,7 +6662,8 @@ async function findGitSourceFiles(projectRoot2, ignore, signal) {
|
|
|
6600
6662
|
"-z",
|
|
6601
6663
|
"--untracked-files=all",
|
|
6602
6664
|
"--ignored=no"
|
|
6603
|
-
])
|
|
6665
|
+
]),
|
|
6666
|
+
gitOutput(projectRoot2, ["ls-files", "--stage", "-z"])
|
|
6604
6667
|
]);
|
|
6605
6668
|
throwIfAborted(signal);
|
|
6606
6669
|
const dirty = /* @__PURE__ */ new Set();
|
|
@@ -6630,9 +6693,17 @@ async function findGitSourceFiles(projectRoot2, ignore, signal) {
|
|
|
6630
6693
|
const ext = path12.extname(relative3).toLowerCase();
|
|
6631
6694
|
if (INDEXABLE_EXTENSION_SET.has(ext) || detectLang(full) !== null) files.push(full);
|
|
6632
6695
|
}
|
|
6696
|
+
const snapshot = createHash("sha256").update(stagedOutput).update("\0").update(statusOutput);
|
|
6697
|
+
const indexedFiles = new Set(files);
|
|
6698
|
+
for (const dirtyFile of [...dirty].sort()) {
|
|
6699
|
+
if (!indexedFiles.has(dirtyFile) || deleted.has(dirtyFile)) continue;
|
|
6700
|
+
snapshot.update("\0").update(dirtyFile).update("\0");
|
|
6701
|
+
snapshot.update(xxhash64String(await fs9.readFile(dirtyFile, "utf8")));
|
|
6702
|
+
}
|
|
6633
6703
|
return {
|
|
6634
6704
|
files,
|
|
6635
|
-
trustedUnchanged: new Set(files.filter((file) => !dirty.has(file)))
|
|
6705
|
+
trustedUnchanged: new Set(files.filter((file) => !dirty.has(file))),
|
|
6706
|
+
snapshotKey: snapshot.digest("hex")
|
|
6636
6707
|
};
|
|
6637
6708
|
} catch {
|
|
6638
6709
|
return null;
|
|
@@ -6645,7 +6716,8 @@ async function findSourceFiles(projectRoot2, ignore, isGitIgnored, signal) {
|
|
|
6645
6716
|
files: gitFiles.files,
|
|
6646
6717
|
complete: true,
|
|
6647
6718
|
errors: [],
|
|
6648
|
-
trustedUnchanged: gitFiles.trustedUnchanged
|
|
6719
|
+
trustedUnchanged: gitFiles.trustedUnchanged,
|
|
6720
|
+
snapshotKey: gitFiles.snapshotKey
|
|
6649
6721
|
};
|
|
6650
6722
|
}
|
|
6651
6723
|
const results = [];
|
|
@@ -6732,6 +6804,17 @@ async function resolveProjectRelations(store, projectRoot2, opts) {
|
|
|
6732
6804
|
}
|
|
6733
6805
|
}
|
|
6734
6806
|
async function runIndexerWithStore(store, opts) {
|
|
6807
|
+
let result;
|
|
6808
|
+
try {
|
|
6809
|
+
result = await store.runAtomicIndexUpdate(() => runIndexerAtomic(store, opts));
|
|
6810
|
+
} catch (error) {
|
|
6811
|
+
if (!(error instanceof IndexSourceChangedError)) throw error;
|
|
6812
|
+
result = await store.runAtomicIndexUpdate(() => runIndexerAtomic(store, opts));
|
|
6813
|
+
}
|
|
6814
|
+
if (!opts.files) store.compactIfNeeded();
|
|
6815
|
+
return result;
|
|
6816
|
+
}
|
|
6817
|
+
async function runIndexerAtomic(store, opts) {
|
|
6735
6818
|
const { projectRoot: projectRoot2, langs, ignore = [], signal } = opts;
|
|
6736
6819
|
const relationGraphVersion = "2";
|
|
6737
6820
|
const refResolutionVersion = "2";
|
|
@@ -6751,6 +6834,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
6751
6834
|
let discoveredFiles = null;
|
|
6752
6835
|
let discoveryComplete = true;
|
|
6753
6836
|
let trustedUnchanged;
|
|
6837
|
+
let discoverySnapshotKey;
|
|
6754
6838
|
if (opts.files && opts.files.length > 0) {
|
|
6755
6839
|
files = opts.files.map((f) => path12.resolve(projectRoot2, f)).filter((f) => {
|
|
6756
6840
|
if (!isWithinProject(projectRoot2, f)) return false;
|
|
@@ -6764,6 +6848,7 @@ async function runIndexerWithStore(store, opts) {
|
|
|
6764
6848
|
discoveryComplete = discovery.complete;
|
|
6765
6849
|
discoveredFiles = new Set(files);
|
|
6766
6850
|
trustedUnchanged = discovery.trustedUnchanged;
|
|
6851
|
+
discoverySnapshotKey = discovery.snapshotKey;
|
|
6767
6852
|
}
|
|
6768
6853
|
if (langs && langs.length > 0) {
|
|
6769
6854
|
const langSet = new Set(langs);
|
|
@@ -6777,6 +6862,8 @@ async function runIndexerWithStore(store, opts) {
|
|
|
6777
6862
|
if (!force) {
|
|
6778
6863
|
for (const meta of store.getAllFileMetas()) existingMeta.set(meta.file, meta);
|
|
6779
6864
|
}
|
|
6865
|
+
const snapshotTrusted = !force && discoverySnapshotKey !== void 0 && store.getMetadata(GIT_SNAPSHOT_METADATA_KEY) === discoverySnapshotKey;
|
|
6866
|
+
if (!snapshotTrusted) trustedUnchanged = void 0;
|
|
6780
6867
|
const totalFilesForProgress = files.length;
|
|
6781
6868
|
let filesPreSkipped = 0;
|
|
6782
6869
|
if (!force && trustedUnchanged) {
|
|
@@ -6839,9 +6926,6 @@ async function runIndexerWithStore(store, opts) {
|
|
|
6839
6926
|
};
|
|
6840
6927
|
}
|
|
6841
6928
|
const meta = existingMeta.get(file);
|
|
6842
|
-
if (!force && meta && meta.mtimeMs === Math.floor(stat2.mtimeMs)) {
|
|
6843
|
-
return { file, stat: stat2, lang, parsed: null, skippedMeta: meta };
|
|
6844
|
-
}
|
|
6845
6929
|
let content;
|
|
6846
6930
|
try {
|
|
6847
6931
|
content = await fs9.readFile(file, { encoding: "utf8", signal });
|
|
@@ -7070,9 +7154,18 @@ async function runIndexerWithStore(store, opts) {
|
|
|
7070
7154
|
});
|
|
7071
7155
|
store.setMetadata("ref_resolution_version", refResolutionVersion);
|
|
7072
7156
|
store.setMetadata("relation_graph_version", relationGraphVersion);
|
|
7157
|
+
const completeProjectScope = !opts.files && (!langs || langs.length === 0) && (!opts.ignore || opts.ignore.length === 0);
|
|
7158
|
+
if (completeProjectScope && discoverySnapshotKey !== void 0) {
|
|
7159
|
+
const finalSnapshot = await findGitSourceFiles(projectRoot2, ignore, signal);
|
|
7160
|
+
if (!finalSnapshot || finalSnapshot.snapshotKey !== discoverySnapshotKey) {
|
|
7161
|
+
throw new IndexSourceChangedError(
|
|
7162
|
+
"Project files changed during indexing; retrying before publishing the generation."
|
|
7163
|
+
);
|
|
7164
|
+
}
|
|
7165
|
+
store.setMetadata(GIT_SNAPSHOT_METADATA_KEY, errors.length === 0 ? discoverySnapshotKey : "");
|
|
7166
|
+
}
|
|
7073
7167
|
if (!opts.files || filesIndexed >= 50) store.optimize();
|
|
7074
7168
|
store.setLastIndexed(Date.now());
|
|
7075
|
-
if (!opts.files) store.compactIfNeeded();
|
|
7076
7169
|
const durationMs = Date.now() - startMs;
|
|
7077
7170
|
return {
|
|
7078
7171
|
filesIndexed,
|
|
@@ -7215,7 +7308,7 @@ var GenerationLruCache = class {
|
|
|
7215
7308
|
};
|
|
7216
7309
|
|
|
7217
7310
|
// src/codebase-index/project-server-endpoint.ts
|
|
7218
|
-
import { createHash } from "node:crypto";
|
|
7311
|
+
import { createHash as createHash2 } from "node:crypto";
|
|
7219
7312
|
import * as fs10 from "node:fs";
|
|
7220
7313
|
import * as os3 from "node:os";
|
|
7221
7314
|
import * as path13 from "node:path";
|
|
@@ -7233,7 +7326,7 @@ function projectIndexServerBuildId(entrypoint) {
|
|
|
7233
7326
|
if (buildIdCache?.file === file && buildIdCache.mtimeMs === stat2.mtimeMs && buildIdCache.size === stat2.size) {
|
|
7234
7327
|
return buildIdCache.buildId;
|
|
7235
7328
|
}
|
|
7236
|
-
const buildId =
|
|
7329
|
+
const buildId = createHash2("sha256").update(fs10.readFileSync(file)).digest("hex").slice(0, 24);
|
|
7237
7330
|
buildIdCache = { file, mtimeMs: stat2.mtimeMs, size: stat2.size, buildId };
|
|
7238
7331
|
return buildId;
|
|
7239
7332
|
} catch {
|
|
@@ -7246,7 +7339,7 @@ function normalizeLocalPath(value) {
|
|
|
7246
7339
|
}
|
|
7247
7340
|
function projectIndexServerKey(projectRoot2, indexDir2) {
|
|
7248
7341
|
const resolvedIndexDir = normalizeLocalPath(resolveIndexDir(projectRoot2, indexDir2));
|
|
7249
|
-
return
|
|
7342
|
+
return createHash2("sha256").update(resolvedIndexDir).digest("hex").slice(0, 24);
|
|
7250
7343
|
}
|
|
7251
7344
|
function projectIndexServerEndpoint(projectRoot2, indexDir2) {
|
|
7252
7345
|
const key = projectIndexServerKey(projectRoot2, indexDir2);
|
|
@@ -7369,11 +7462,18 @@ function clearQueryCaches() {
|
|
|
7369
7462
|
outgoingCallsCache.clear();
|
|
7370
7463
|
}
|
|
7371
7464
|
function cachedRead(cache, key, load) {
|
|
7465
|
+
if (indexActivity.indexing) {
|
|
7466
|
+
const error = new Error(
|
|
7467
|
+
`Codebase index refresh in progress (${indexActivity.currentFile}/${indexActivity.totalFiles} files); retry after the completed generation is published.`
|
|
7468
|
+
);
|
|
7469
|
+
error.name = "IndexRefreshInProgressError";
|
|
7470
|
+
throw error;
|
|
7471
|
+
}
|
|
7372
7472
|
const generation = indexActivity.generation;
|
|
7373
7473
|
const cached = cache.get(key, generation);
|
|
7374
7474
|
if (cached !== void 0) return cached;
|
|
7375
7475
|
const value = load();
|
|
7376
|
-
return
|
|
7476
|
+
return cache.set(key, generation, value);
|
|
7377
7477
|
}
|
|
7378
7478
|
var MAX_CLIENT_WRITE_BUFFER_BYTES = 8 * 1024 * 1024;
|
|
7379
7479
|
function send(state, message) {
|
|
@@ -7576,12 +7676,22 @@ async function dispatchOperation(state, message) {
|
|
|
7576
7676
|
);
|
|
7577
7677
|
case "incomingCalls": {
|
|
7578
7678
|
const callArgs = fixedArgs(message.args);
|
|
7579
|
-
const cacheKey = JSON.stringify([
|
|
7679
|
+
const cacheKey = JSON.stringify([
|
|
7680
|
+
callArgs.symbol,
|
|
7681
|
+
callArgs.file ?? "",
|
|
7682
|
+
callArgs.limit ?? 100,
|
|
7683
|
+
callArgs.transitive ?? false
|
|
7684
|
+
]);
|
|
7580
7685
|
return cachedRead(incomingCallsCache, cacheKey, () => incomingCallsService(callArgs));
|
|
7581
7686
|
}
|
|
7582
7687
|
case "outgoingCalls": {
|
|
7583
7688
|
const callArgs = fixedArgs(message.args);
|
|
7584
|
-
const cacheKey = JSON.stringify([
|
|
7689
|
+
const cacheKey = JSON.stringify([
|
|
7690
|
+
callArgs.symbol,
|
|
7691
|
+
callArgs.file ?? "",
|
|
7692
|
+
callArgs.limit ?? 100,
|
|
7693
|
+
callArgs.transitive ?? false
|
|
7694
|
+
]);
|
|
7585
7695
|
return cachedRead(outgoingCallsCache, cacheKey, () => outgoingCallsService(callArgs));
|
|
7586
7696
|
}
|
|
7587
7697
|
default:
|