grepmax 0.24.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +11 -6
- package/dist/commands/add.js +38 -31
- package/dist/commands/config.js +16 -15
- package/dist/commands/context.js +38 -13
- package/dist/commands/doctor.js +76 -83
- package/dist/commands/extract.js +12 -1
- package/dist/commands/index.js +23 -24
- package/dist/commands/list.js +23 -14
- package/dist/commands/mcp.js +278 -143
- package/dist/commands/peek.js +16 -5
- package/dist/commands/repair.js +54 -120
- package/dist/commands/search-output.js +30 -9
- package/dist/commands/search-run.js +75 -47
- package/dist/commands/search-skeletons.js +28 -18
- package/dist/commands/search.js +45 -49
- package/dist/commands/serve.js +415 -373
- package/dist/commands/setup.js +2 -2
- package/dist/commands/similar.js +5 -5
- package/dist/commands/skeleton.js +67 -41
- package/dist/commands/status.js +5 -2
- package/dist/commands/summarize.js +6 -0
- package/dist/commands/watch.js +102 -38
- package/dist/config.js +3 -0
- package/dist/lib/daemon/daemon.js +1101 -379
- package/dist/lib/daemon/ipc-handler.js +122 -13
- package/dist/lib/daemon/mlx-server-manager.js +268 -125
- package/dist/lib/daemon/process-manager.js +7 -4
- package/dist/lib/daemon/search-handler.js +23 -9
- package/dist/lib/daemon/watcher-manager.js +353 -110
- package/dist/lib/index/batch-processor.js +231 -67
- package/dist/lib/index/embedding-generation.js +109 -0
- package/dist/lib/index/embedding-status.js +23 -0
- package/dist/lib/index/file-policy.js +273 -0
- package/dist/lib/index/ignore-patterns.js +4 -0
- package/dist/lib/index/index-config.js +18 -4
- package/dist/lib/index/syncer.js +256 -79
- package/dist/lib/index/walker.js +66 -86
- package/dist/lib/index/watcher-batch.js +9 -0
- package/dist/lib/index/watcher.js +129 -3
- package/dist/lib/llm/server.js +6 -0
- package/dist/lib/search/searcher.js +30 -19
- package/dist/lib/store/store-lease.js +473 -0
- package/dist/lib/store/vector-db.js +302 -57
- package/dist/lib/utils/blocked-roots.js +63 -0
- package/dist/lib/utils/cross-project.js +7 -3
- package/dist/lib/utils/daemon-client.js +24 -1
- package/dist/lib/utils/daemon-launcher.js +38 -13
- package/dist/lib/utils/doctor-status.js +76 -0
- package/dist/lib/utils/file-utils.js +74 -4
- package/dist/lib/utils/keyed-mutex.js +101 -0
- package/dist/lib/utils/logger.js +57 -1
- package/dist/lib/utils/mlx-hf-cache.js +114 -0
- package/dist/lib/utils/operation-coordinator.js +146 -0
- package/dist/lib/utils/path-containment.js +106 -0
- package/dist/lib/utils/process.js +44 -0
- package/dist/lib/utils/project-registry.js +351 -3
- package/dist/lib/utils/scope-filter.js +3 -9
- package/dist/lib/utils/stale-hint.js +12 -19
- package/dist/lib/utils/watcher-launcher.js +5 -1
- package/dist/lib/utils/watcher-store.js +2 -2
- package/dist/lib/workers/colbert-math.js +15 -12
- package/dist/lib/workers/embeddings/colbert.js +3 -2
- package/dist/lib/workers/embeddings/granite.js +4 -3
- package/dist/lib/workers/embeddings/mlx-client.js +59 -16
- package/dist/lib/workers/orchestrator.js +39 -8
- package/dist/lib/workers/pool.js +146 -82
- package/dist/lib/workers/process-child.js +11 -2
- package/dist/lib/workers/serialized-handler.js +10 -0
- package/dist/lib/workers/worker.js +13 -4
- package/mlx-embed-server/server.py +21 -3
- package/mlx-embed-server/summarizer.py +8 -0
- package/package.json +4 -3
- package/plugins/grepmax/.claude-plugin/plugin.json +1 -1
- package/plugins/grepmax/hooks/start.js +3 -170
|
@@ -44,6 +44,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
44
44
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
45
45
|
exports.VectorDB = exports.DiskPressureError = void 0;
|
|
46
46
|
exports.isLanceCorruptionError = isLanceCorruptionError;
|
|
47
|
+
exports.isMissingTableError = isMissingTableError;
|
|
47
48
|
const fs = __importStar(require("node:fs"));
|
|
48
49
|
const lancedb = __importStar(require("@lancedb/lancedb"));
|
|
49
50
|
const apache_arrow_1 = require("apache-arrow");
|
|
@@ -52,6 +53,7 @@ const index_config_1 = require("../index/index-config");
|
|
|
52
53
|
const cleanup_1 = require("../utils/cleanup");
|
|
53
54
|
const filter_builder_1 = require("../utils/filter-builder");
|
|
54
55
|
const logger_1 = require("../utils/logger");
|
|
56
|
+
const store_lease_1 = require("./store-lease");
|
|
55
57
|
class DiskPressureError extends Error {
|
|
56
58
|
constructor(message = "Disk critically low — writes suspended") {
|
|
57
59
|
super(message);
|
|
@@ -69,6 +71,9 @@ function isLanceCorruptionError(err) {
|
|
|
69
71
|
return /Not found:.*\.lance(?:[^a-z]|$)/i.test(msg);
|
|
70
72
|
}
|
|
71
73
|
function isMissingTableError(err) {
|
|
74
|
+
const code = err === null || err === void 0 ? void 0 : err.code;
|
|
75
|
+
if (code === "EACCES" || code === "EIO")
|
|
76
|
+
return false;
|
|
72
77
|
const msg = err instanceof Error ? err.message : String(err);
|
|
73
78
|
const tableMissing = /table.*chunks.*(?:not found|does not exist)/i.test(msg) ||
|
|
74
79
|
/(?:not found|does not exist).*table.*chunks/i.test(msg) ||
|
|
@@ -82,22 +87,34 @@ function isMissingTableError(err) {
|
|
|
82
87
|
const TABLE_NAME = "chunks";
|
|
83
88
|
const MAINTENANCE_INTERVAL_MS = 5 * 60 * 1000;
|
|
84
89
|
class VectorDB {
|
|
85
|
-
constructor(lancedbDir, vectorDim) {
|
|
90
|
+
constructor(lancedbDir, vectorDim, suppliedLease) {
|
|
86
91
|
this.lancedbDir = lancedbDir;
|
|
92
|
+
this.suppliedLease = suppliedLease;
|
|
87
93
|
this.db = null;
|
|
88
94
|
this.closed = false;
|
|
89
95
|
this.maintenanceRunning = false;
|
|
90
96
|
this.maintenancePromise = null;
|
|
91
97
|
this.maintenanceTimer = null;
|
|
98
|
+
this.maintenanceRunner = (fn) => fn();
|
|
92
99
|
this.lastCorruptionLogMs = 0;
|
|
93
100
|
this.diskPressure = "ok";
|
|
94
101
|
this.lastDiskCheckMs = 0;
|
|
95
102
|
this.lastLoggedPressure = "ok";
|
|
103
|
+
this.leasePromise = null;
|
|
104
|
+
this.leaseAbort = new AbortController();
|
|
105
|
+
this.closePromise = null;
|
|
106
|
+
this.releaseLeaseOnClose = true;
|
|
107
|
+
this.requireClosedOnClose = false;
|
|
108
|
+
this.leaseTransitionTail = Promise.resolve();
|
|
109
|
+
this.leaseHolder = {};
|
|
96
110
|
// Write gate: async read-write lock where writes are "readers" (shared)
|
|
97
111
|
// and compaction is the "writer" (exclusive).
|
|
98
112
|
this.activeWrites = 0;
|
|
99
|
-
this.
|
|
113
|
+
this.writeDrainResolvers = new Set();
|
|
100
114
|
this.compactingPromise = null;
|
|
115
|
+
this.exclusiveMutationPromise = null;
|
|
116
|
+
this.activeCompactions = 0;
|
|
117
|
+
this.compactionDrainResolvers = new Set();
|
|
101
118
|
// Default to the configured tier's dim (not the hard-wired small-tier 384)
|
|
102
119
|
// so a `standard`-tier index actually stores 768d vectors. An explicit
|
|
103
120
|
// arg still wins (eval scripts, tests).
|
|
@@ -108,7 +125,9 @@ class VectorDB {
|
|
|
108
125
|
* Start a periodic maintenance timer (FTS rebuild + optimize).
|
|
109
126
|
* Call once from the daemon — replaces per-processor maintenance intervals.
|
|
110
127
|
*/
|
|
111
|
-
startMaintenanceLoop() {
|
|
128
|
+
startMaintenanceLoop(runOperation) {
|
|
129
|
+
if (runOperation)
|
|
130
|
+
this.maintenanceRunner = runOperation;
|
|
112
131
|
if (this.maintenanceTimer)
|
|
113
132
|
return;
|
|
114
133
|
this.maintenanceTimer = setInterval(() => {
|
|
@@ -118,7 +137,7 @@ class VectorDB {
|
|
|
118
137
|
// promise to await instead of a chain.
|
|
119
138
|
if (this.maintenancePromise)
|
|
120
139
|
return;
|
|
121
|
-
const run = (() => __awaiter(this, void 0, void 0, function* () {
|
|
140
|
+
const run = this.maintenanceRunner(() => __awaiter(this, void 0, void 0, function* () {
|
|
122
141
|
try {
|
|
123
142
|
yield this.runMaintenance();
|
|
124
143
|
}
|
|
@@ -143,7 +162,7 @@ class VectorDB {
|
|
|
143
162
|
}
|
|
144
163
|
(0, logger_1.log)("vectordb", `Periodic maintenance failed: ${err}`);
|
|
145
164
|
}
|
|
146
|
-
}))
|
|
165
|
+
}));
|
|
147
166
|
this.maintenancePromise = run.finally(() => {
|
|
148
167
|
if (this.maintenancePromise === run)
|
|
149
168
|
this.maintenancePromise = null;
|
|
@@ -169,6 +188,10 @@ class VectorDB {
|
|
|
169
188
|
}
|
|
170
189
|
getDb() {
|
|
171
190
|
return __awaiter(this, void 0, void 0, function* () {
|
|
191
|
+
if (this.closed) {
|
|
192
|
+
throw new Error("VectorDB connection is closed");
|
|
193
|
+
}
|
|
194
|
+
yield this.getLease();
|
|
172
195
|
if (this.closed) {
|
|
173
196
|
throw new Error("VectorDB connection is closed");
|
|
174
197
|
}
|
|
@@ -179,6 +202,88 @@ class VectorDB {
|
|
|
179
202
|
return this.db;
|
|
180
203
|
});
|
|
181
204
|
}
|
|
205
|
+
getLease() {
|
|
206
|
+
if (!this.leasePromise) {
|
|
207
|
+
const lease = this.suppliedLease
|
|
208
|
+
? Promise.resolve(this.suppliedLease)
|
|
209
|
+
: store_lease_1.StoreLease.acquireShared({
|
|
210
|
+
storeDir: this.lancedbDir,
|
|
211
|
+
role: process.title || "gmax",
|
|
212
|
+
signal: this.leaseAbort.signal,
|
|
213
|
+
});
|
|
214
|
+
this.leasePromise = lease.then((resolved) => {
|
|
215
|
+
resolved.claim(this.leaseHolder, this.lancedbDir);
|
|
216
|
+
return resolved;
|
|
217
|
+
});
|
|
218
|
+
}
|
|
219
|
+
return this.leasePromise;
|
|
220
|
+
}
|
|
221
|
+
withLeaseTransition(fn) {
|
|
222
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
223
|
+
const previous = this.leaseTransitionTail;
|
|
224
|
+
let release;
|
|
225
|
+
this.leaseTransitionTail = new Promise((resolve) => {
|
|
226
|
+
release = resolve;
|
|
227
|
+
});
|
|
228
|
+
yield previous;
|
|
229
|
+
try {
|
|
230
|
+
return yield fn();
|
|
231
|
+
}
|
|
232
|
+
finally {
|
|
233
|
+
release();
|
|
234
|
+
}
|
|
235
|
+
});
|
|
236
|
+
}
|
|
237
|
+
/** Upgrade and retain this instance's current lease for transfer to a new DB. */
|
|
238
|
+
upgradeStoreLease(signal) {
|
|
239
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
240
|
+
if (this.closed)
|
|
241
|
+
throw new Error("VectorDB connection is closed");
|
|
242
|
+
if (this.exclusiveMutationPromise)
|
|
243
|
+
yield this.exclusiveMutationPromise;
|
|
244
|
+
return this.withLeaseTransition(() => __awaiter(this, void 0, void 0, function* () {
|
|
245
|
+
const current = yield this.getLease();
|
|
246
|
+
if (current.mode === "exclusive")
|
|
247
|
+
return current;
|
|
248
|
+
const upgrade = current.upgrade({ signal });
|
|
249
|
+
this.leasePromise = upgrade;
|
|
250
|
+
try {
|
|
251
|
+
const upgraded = yield upgrade;
|
|
252
|
+
upgraded.claim(this.leaseHolder, this.lancedbDir);
|
|
253
|
+
return upgraded;
|
|
254
|
+
}
|
|
255
|
+
catch (error) {
|
|
256
|
+
this.leasePromise = Promise.resolve(current);
|
|
257
|
+
throw error;
|
|
258
|
+
}
|
|
259
|
+
}));
|
|
260
|
+
});
|
|
261
|
+
}
|
|
262
|
+
/** Downgrade a transferred exclusive lease and retain the replacement token. */
|
|
263
|
+
downgradeStoreLease() {
|
|
264
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
265
|
+
if (this.closed)
|
|
266
|
+
throw new Error("VectorDB connection is closed");
|
|
267
|
+
if (this.exclusiveMutationPromise)
|
|
268
|
+
yield this.exclusiveMutationPromise;
|
|
269
|
+
return this.withLeaseTransition(() => __awaiter(this, void 0, void 0, function* () {
|
|
270
|
+
const current = yield this.getLease();
|
|
271
|
+
if (current.mode === "shared")
|
|
272
|
+
return current;
|
|
273
|
+
const downgrade = current.downgrade();
|
|
274
|
+
this.leasePromise = downgrade;
|
|
275
|
+
try {
|
|
276
|
+
const downgraded = yield downgrade;
|
|
277
|
+
downgraded.claim(this.leaseHolder, this.lancedbDir);
|
|
278
|
+
return downgraded;
|
|
279
|
+
}
|
|
280
|
+
catch (error) {
|
|
281
|
+
this.leasePromise = Promise.resolve(current);
|
|
282
|
+
throw error;
|
|
283
|
+
}
|
|
284
|
+
}));
|
|
285
|
+
});
|
|
286
|
+
}
|
|
182
287
|
getAvailableBytes() {
|
|
183
288
|
try {
|
|
184
289
|
const stats = fs.statfsSync(this.lancedbDir);
|
|
@@ -233,29 +338,86 @@ class VectorDB {
|
|
|
233
338
|
*/
|
|
234
339
|
withWriteGate(fn) {
|
|
235
340
|
return __awaiter(this, void 0, void 0, function* () {
|
|
236
|
-
|
|
237
|
-
|
|
341
|
+
while (this.exclusiveMutationPromise || this.compactingPromise) {
|
|
342
|
+
if (this.closed)
|
|
343
|
+
throw new Error("VectorDB connection is closed");
|
|
344
|
+
yield Promise.all([this.exclusiveMutationPromise, this.compactingPromise].filter((promise) => promise !== null));
|
|
238
345
|
}
|
|
346
|
+
if (this.closed)
|
|
347
|
+
throw new Error("VectorDB connection is closed");
|
|
239
348
|
this.activeWrites++;
|
|
240
349
|
try {
|
|
241
350
|
return yield fn();
|
|
242
351
|
}
|
|
243
352
|
finally {
|
|
244
353
|
this.activeWrites--;
|
|
245
|
-
if (this.activeWrites === 0
|
|
246
|
-
this.
|
|
247
|
-
|
|
354
|
+
if (this.activeWrites === 0) {
|
|
355
|
+
for (const resolve of this.writeDrainResolvers)
|
|
356
|
+
resolve();
|
|
357
|
+
this.writeDrainResolvers.clear();
|
|
248
358
|
}
|
|
249
359
|
}
|
|
250
360
|
});
|
|
251
361
|
}
|
|
362
|
+
drainCompactions() {
|
|
363
|
+
if (this.activeCompactions === 0)
|
|
364
|
+
return Promise.resolve();
|
|
365
|
+
return new Promise((resolve) => {
|
|
366
|
+
this.compactionDrainResolvers.add(resolve);
|
|
367
|
+
});
|
|
368
|
+
}
|
|
369
|
+
/**
|
|
370
|
+
* Run a destructive table mutation while holding the process-wide exclusive
|
|
371
|
+
* store intent. New local writes stop before existing activity is drained.
|
|
372
|
+
*/
|
|
373
|
+
withExclusiveTableMutation(mutation) {
|
|
374
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
375
|
+
if (this.closed)
|
|
376
|
+
throw new Error("VectorDB connection is closed");
|
|
377
|
+
if (this.exclusiveMutationPromise) {
|
|
378
|
+
throw new Error("An exclusive table mutation is already in progress");
|
|
379
|
+
}
|
|
380
|
+
let resolveMutation;
|
|
381
|
+
this.exclusiveMutationPromise = new Promise((resolve) => {
|
|
382
|
+
resolveMutation = resolve;
|
|
383
|
+
});
|
|
384
|
+
try {
|
|
385
|
+
return yield this.withLeaseTransition(() => __awaiter(this, void 0, void 0, function* () {
|
|
386
|
+
let temporaryExclusiveLease = null;
|
|
387
|
+
try {
|
|
388
|
+
const currentLease = yield this.getLease();
|
|
389
|
+
if (currentLease.mode === "shared") {
|
|
390
|
+
temporaryExclusiveLease = yield store_lease_1.StoreLease.acquireExclusive({
|
|
391
|
+
storeDir: this.lancedbDir,
|
|
392
|
+
pid: currentLease.owner.pid,
|
|
393
|
+
processStart: currentLease.owner.processStart,
|
|
394
|
+
role: `${currentLease.owner.role}:exclusive-mutation`,
|
|
395
|
+
ignoreNonces: new Set([currentLease.owner.nonce]),
|
|
396
|
+
signal: this.leaseAbort.signal,
|
|
397
|
+
});
|
|
398
|
+
}
|
|
399
|
+
yield Promise.all([this.drainWrites(), this.drainCompactions()]);
|
|
400
|
+
const db = yield this.getDb();
|
|
401
|
+
return yield mutation(db);
|
|
402
|
+
}
|
|
403
|
+
finally {
|
|
404
|
+
yield (temporaryExclusiveLease === null || temporaryExclusiveLease === void 0 ? void 0 : temporaryExclusiveLease.release());
|
|
405
|
+
}
|
|
406
|
+
}));
|
|
407
|
+
}
|
|
408
|
+
finally {
|
|
409
|
+
this.exclusiveMutationPromise = null;
|
|
410
|
+
resolveMutation();
|
|
411
|
+
}
|
|
412
|
+
});
|
|
413
|
+
}
|
|
252
414
|
/** Wait for all in-flight writes to complete before compaction. */
|
|
253
415
|
drainWrites() {
|
|
254
416
|
if (this.activeWrites === 0)
|
|
255
417
|
return Promise.resolve();
|
|
256
418
|
(0, logger_1.debug)("vectordb", `Draining ${this.activeWrites} in-flight write(s) before compaction`);
|
|
257
419
|
return new Promise((resolve) => {
|
|
258
|
-
this.
|
|
420
|
+
this.writeDrainResolvers.add(resolve);
|
|
259
421
|
});
|
|
260
422
|
}
|
|
261
423
|
seedRow() {
|
|
@@ -305,8 +467,10 @@ class VectorDB {
|
|
|
305
467
|
try {
|
|
306
468
|
table = yield db.openTable(TABLE_NAME);
|
|
307
469
|
}
|
|
308
|
-
catch (
|
|
309
|
-
|
|
470
|
+
catch (err) {
|
|
471
|
+
if (isMissingTableError(err))
|
|
472
|
+
return null;
|
|
473
|
+
throw err;
|
|
310
474
|
}
|
|
311
475
|
const schema = yield table.schema();
|
|
312
476
|
const field = schema.fields.find((f) => f.name === "vector");
|
|
@@ -398,6 +562,11 @@ class VectorDB {
|
|
|
398
562
|
});
|
|
399
563
|
}
|
|
400
564
|
ensureTable() {
|
|
565
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
566
|
+
return this.withWriteGate(() => this.ensureTableUnsafe());
|
|
567
|
+
});
|
|
568
|
+
}
|
|
569
|
+
ensureTableUnsafe() {
|
|
401
570
|
return __awaiter(this, void 0, void 0, function* () {
|
|
402
571
|
const db = yield this.getDb();
|
|
403
572
|
let table;
|
|
@@ -426,7 +595,6 @@ class VectorDB {
|
|
|
426
595
|
if (!records.length)
|
|
427
596
|
return;
|
|
428
597
|
this.ensureDiskOk();
|
|
429
|
-
const table = yield this.ensureTable();
|
|
430
598
|
const toBuffer = (val) => {
|
|
431
599
|
var _a, _b;
|
|
432
600
|
if (Buffer.isBuffer(val))
|
|
@@ -475,8 +643,7 @@ class VectorDB {
|
|
|
475
643
|
if (vec.length !== this.vectorDim) {
|
|
476
644
|
throw new Error(`Vector dimension mismatch: got ${vec.length}d, expected ${this.vectorDim}d. ` +
|
|
477
645
|
"The embedding model tier likely changed without a rebuild — the shared " +
|
|
478
|
-
|
|
479
|
-
"`gmax index --reset` cannot change the table width).");
|
|
646
|
+
"table is fixed-width. Run `gmax repair --rebuild` to rebuild the whole corpus.");
|
|
480
647
|
}
|
|
481
648
|
rec.vector = vec;
|
|
482
649
|
rec.colbert = toBuffer(rec.colbert);
|
|
@@ -510,11 +677,15 @@ class VectorDB {
|
|
|
510
677
|
rec.summary = (_r = rec.summary) !== null && _r !== void 0 ? _r : null;
|
|
511
678
|
}
|
|
512
679
|
try {
|
|
513
|
-
yield this.withWriteGate(() =>
|
|
680
|
+
yield this.withWriteGate(() => __awaiter(this, void 0, void 0, function* () {
|
|
681
|
+
const table = yield this.ensureTableUnsafe();
|
|
682
|
+
yield table.add(records);
|
|
683
|
+
}));
|
|
514
684
|
}
|
|
515
685
|
catch (err) {
|
|
516
686
|
const msg = err instanceof Error ? err.message : String(err);
|
|
517
687
|
if (msg.toLowerCase().includes("found field not in schema")) {
|
|
688
|
+
const table = yield this.ensureTable();
|
|
518
689
|
const schema = yield table.schema();
|
|
519
690
|
const schemaFields = schema.fields.map((f) => f.name);
|
|
520
691
|
throw new Error(`[vector-db] schema mismatch detected (fields: ${schemaFields.join(", ")}). Please run "gmax index --reset" to rebuild the index.`);
|
|
@@ -525,7 +696,12 @@ class VectorDB {
|
|
|
525
696
|
}
|
|
526
697
|
createFTSIndex() {
|
|
527
698
|
return __awaiter(this, arguments, void 0, function* (rebuild = false, retries = 5) {
|
|
528
|
-
|
|
699
|
+
return this.withWriteGate(() => this.createFTSIndexUnsafe(rebuild, retries));
|
|
700
|
+
});
|
|
701
|
+
}
|
|
702
|
+
createFTSIndexUnsafe() {
|
|
703
|
+
return __awaiter(this, arguments, void 0, function* (rebuild = false, retries = 5) {
|
|
704
|
+
const table = yield this.ensureTableUnsafe();
|
|
529
705
|
if (rebuild) {
|
|
530
706
|
try {
|
|
531
707
|
yield table.dropIndex("content_idx");
|
|
@@ -572,18 +748,26 @@ class VectorDB {
|
|
|
572
748
|
});
|
|
573
749
|
}
|
|
574
750
|
optimize() {
|
|
575
|
-
return __awaiter(this, arguments, void 0, function* (retries = 5, retentionMs = 0) {
|
|
751
|
+
return __awaiter(this, arguments, void 0, function* (retries = 5, retentionMs = 0, bypassExclusiveMutation = false) {
|
|
752
|
+
if (!bypassExclusiveMutation) {
|
|
753
|
+
while (this.exclusiveMutationPromise)
|
|
754
|
+
yield this.exclusiveMutationPromise;
|
|
755
|
+
if (this.closed)
|
|
756
|
+
throw new Error("VectorDB connection is closed");
|
|
757
|
+
}
|
|
576
758
|
if (this.compactingPromise) {
|
|
577
759
|
(0, logger_1.debug)("vectordb", "Optimize already in progress, skipping");
|
|
760
|
+
yield this.compactingPromise;
|
|
578
761
|
return;
|
|
579
762
|
}
|
|
580
|
-
const table = yield this.ensureTable();
|
|
581
|
-
const cutoff = new Date(Date.now() - retentionMs);
|
|
582
763
|
let resolveCompacting;
|
|
583
|
-
this.compactingPromise = new Promise((
|
|
584
|
-
resolveCompacting =
|
|
764
|
+
this.compactingPromise = new Promise((resolve) => {
|
|
765
|
+
resolveCompacting = resolve;
|
|
585
766
|
});
|
|
767
|
+
this.activeCompactions++;
|
|
586
768
|
try {
|
|
769
|
+
const table = yield this.ensureTableUnsafe();
|
|
770
|
+
const cutoff = new Date(Date.now() - retentionMs);
|
|
587
771
|
for (let attempt = 1; attempt <= retries; attempt++) {
|
|
588
772
|
yield this.drainWrites();
|
|
589
773
|
try {
|
|
@@ -633,6 +817,12 @@ class VectorDB {
|
|
|
633
817
|
finally {
|
|
634
818
|
this.compactingPromise = null;
|
|
635
819
|
resolveCompacting();
|
|
820
|
+
this.activeCompactions--;
|
|
821
|
+
if (this.activeCompactions === 0) {
|
|
822
|
+
for (const resolve of this.compactionDrainResolvers)
|
|
823
|
+
resolve();
|
|
824
|
+
this.compactionDrainResolvers.clear();
|
|
825
|
+
}
|
|
636
826
|
}
|
|
637
827
|
});
|
|
638
828
|
}
|
|
@@ -658,11 +848,11 @@ class VectorDB {
|
|
|
658
848
|
yield this.createFTSIndex();
|
|
659
849
|
if (pressure === "low") {
|
|
660
850
|
(0, logger_1.log)("vectordb", `Low disk — single-pass optimize (no bloat retry)`);
|
|
661
|
-
yield this.optimize(1);
|
|
851
|
+
yield this.optimize(1, 0, true);
|
|
662
852
|
return;
|
|
663
853
|
}
|
|
664
854
|
// Normal maintenance: full optimize + bloat check
|
|
665
|
-
yield this.optimize();
|
|
855
|
+
yield this.optimize(5, 0, true);
|
|
666
856
|
const table = yield this.ensureTable();
|
|
667
857
|
const stats = yield table.stats();
|
|
668
858
|
const diskSize = this.getDirectorySize(this.lancedbDir);
|
|
@@ -672,7 +862,7 @@ class VectorDB {
|
|
|
672
862
|
if (bloatRatio > 2.0 && this.checkDiskPressure() === "ok") {
|
|
673
863
|
(0, logger_1.log)("vectordb", `Bloat detected after optimize: ${(diskSize / 1024 / 1024).toFixed(0)}MB disk vs ${(logicalSize / 1024 / 1024).toFixed(0)}MB logical (${bloatRatio.toFixed(1)}x) — retrying`);
|
|
674
864
|
yield new Promise((r) => setTimeout(r, 2000));
|
|
675
|
-
yield this.optimize();
|
|
865
|
+
yield this.optimize(5, 0, true);
|
|
676
866
|
}
|
|
677
867
|
}
|
|
678
868
|
finally {
|
|
@@ -684,26 +874,24 @@ class VectorDB {
|
|
|
684
874
|
return __awaiter(this, arguments, void 0, function* (threshold = config_1.FRAGMENT_COMPACT_THRESHOLD) {
|
|
685
875
|
if (this.maintenanceRunning)
|
|
686
876
|
return false;
|
|
687
|
-
|
|
688
|
-
return false;
|
|
877
|
+
this.maintenanceRunning = true;
|
|
689
878
|
try {
|
|
879
|
+
if (this.checkDiskPressure() !== "ok")
|
|
880
|
+
return false;
|
|
690
881
|
const table = yield this.ensureTable();
|
|
691
882
|
const stats = yield table.stats();
|
|
692
883
|
if (stats.fragmentStats.numSmallFragments > threshold) {
|
|
693
884
|
(0, logger_1.log)("vectordb", `Fragment threshold exceeded (${stats.fragmentStats.numSmallFragments} > ${threshold}) — compacting`);
|
|
694
|
-
this.
|
|
695
|
-
try {
|
|
696
|
-
yield this.optimize(2);
|
|
697
|
-
}
|
|
698
|
-
finally {
|
|
699
|
-
this.maintenanceRunning = false;
|
|
700
|
-
}
|
|
885
|
+
yield this.optimize(2, 0, true);
|
|
701
886
|
return true;
|
|
702
887
|
}
|
|
703
888
|
}
|
|
704
889
|
catch (err) {
|
|
705
890
|
(0, logger_1.debug)("vectordb", `compactIfNeeded check failed: ${err}`);
|
|
706
891
|
}
|
|
892
|
+
finally {
|
|
893
|
+
this.maintenanceRunning = false;
|
|
894
|
+
}
|
|
707
895
|
return false;
|
|
708
896
|
});
|
|
709
897
|
}
|
|
@@ -795,10 +983,10 @@ class VectorDB {
|
|
|
795
983
|
if (!paths.length)
|
|
796
984
|
return;
|
|
797
985
|
this.ensureDiskOk();
|
|
798
|
-
const table = yield this.ensureTable();
|
|
799
986
|
const unique = Array.from(new Set(paths));
|
|
800
987
|
const batchSize = 500;
|
|
801
988
|
yield this.withWriteGate(() => __awaiter(this, void 0, void 0, function* () {
|
|
989
|
+
const table = yield this.ensureTableUnsafe();
|
|
802
990
|
for (let i = 0; i < unique.length; i += batchSize) {
|
|
803
991
|
const slice = unique.slice(i, i + batchSize);
|
|
804
992
|
const values = slice.map((p) => `'${(0, filter_builder_1.escapeSqlString)(p)}'`).join(",");
|
|
@@ -821,9 +1009,9 @@ class VectorDB {
|
|
|
821
1009
|
return __awaiter(this, void 0, void 0, function* () {
|
|
822
1010
|
if (!ids.length)
|
|
823
1011
|
return;
|
|
824
|
-
const table = yield this.ensureTable();
|
|
825
1012
|
yield this.withWriteGate(() => __awaiter(this, void 0, void 0, function* () {
|
|
826
1013
|
var _a;
|
|
1014
|
+
const table = yield this.ensureTableUnsafe();
|
|
827
1015
|
for (let i = 0; i < ids.length; i++) {
|
|
828
1016
|
const escaped = (0, filter_builder_1.escapeSqlString)(ids[i]);
|
|
829
1017
|
yield table.update({
|
|
@@ -839,13 +1027,13 @@ class VectorDB {
|
|
|
839
1027
|
if (!paths.length)
|
|
840
1028
|
return;
|
|
841
1029
|
this.ensureDiskOk();
|
|
842
|
-
const table = yield this.ensureTable();
|
|
843
1030
|
const unique = Array.from(new Set(paths));
|
|
844
1031
|
const batchSize = 500;
|
|
845
1032
|
const idExclusion = excludeIds.length > 0
|
|
846
1033
|
? ` AND id NOT IN (${excludeIds.map((id) => `'${(0, filter_builder_1.escapeSqlString)(id)}'`).join(",")})`
|
|
847
1034
|
: "";
|
|
848
1035
|
yield this.withWriteGate(() => __awaiter(this, void 0, void 0, function* () {
|
|
1036
|
+
const table = yield this.ensureTableUnsafe();
|
|
849
1037
|
for (let i = 0; i < unique.length; i += batchSize) {
|
|
850
1038
|
const slice = unique.slice(i, i + batchSize);
|
|
851
1039
|
const values = slice.map((p) => `'${(0, filter_builder_1.escapeSqlString)(p)}'`).join(",");
|
|
@@ -866,36 +1054,59 @@ class VectorDB {
|
|
|
866
1054
|
deletePathsWithPrefix(prefix) {
|
|
867
1055
|
return __awaiter(this, void 0, void 0, function* () {
|
|
868
1056
|
this.ensureDiskOk();
|
|
869
|
-
const table = yield this.ensureTable();
|
|
870
1057
|
// Slash-terminate so a project root can't bleed into a sibling
|
|
871
1058
|
// (`/repo/app` must not delete `/repo/app2`), and use starts_with so `_`/`%`
|
|
872
1059
|
// in the path are literal, not LIKE wildcards. Destructive path — keep this
|
|
873
1060
|
// self-protective even if a caller forgets to normalize.
|
|
874
1061
|
const dirPrefix = prefix.endsWith("/") ? prefix : `${prefix}/`;
|
|
875
|
-
yield this.withWriteGate(() =>
|
|
1062
|
+
yield this.withWriteGate(() => __awaiter(this, void 0, void 0, function* () {
|
|
1063
|
+
const table = yield this.ensureTableUnsafe();
|
|
1064
|
+
yield table.delete((0, filter_builder_1.pathStartsWith)(dirPrefix));
|
|
1065
|
+
}));
|
|
876
1066
|
});
|
|
877
1067
|
}
|
|
878
1068
|
drop() {
|
|
879
1069
|
return __awaiter(this, void 0, void 0, function* () {
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
1070
|
+
yield this.withExclusiveTableMutation((db) => __awaiter(this, void 0, void 0, function* () {
|
|
1071
|
+
try {
|
|
1072
|
+
yield db.dropTable(TABLE_NAME);
|
|
1073
|
+
}
|
|
1074
|
+
catch (err) {
|
|
1075
|
+
if (!isMissingTableError(err))
|
|
1076
|
+
throw err;
|
|
1077
|
+
}
|
|
1078
|
+
}));
|
|
887
1079
|
});
|
|
888
1080
|
}
|
|
889
1081
|
close() {
|
|
890
|
-
return __awaiter(this,
|
|
891
|
-
var _a;
|
|
892
|
-
if (this.
|
|
893
|
-
|
|
1082
|
+
return __awaiter(this, arguments, void 0, function* (options = {}) {
|
|
1083
|
+
var _a, _b;
|
|
1084
|
+
if (this.closePromise) {
|
|
1085
|
+
if ((options.requireClosed === true && !this.requireClosedOnClose) ||
|
|
1086
|
+
(options.releaseLease === false && this.releaseLeaseOnClose)) {
|
|
1087
|
+
throw new Error("VectorDB close already started with weaker ownership semantics");
|
|
1088
|
+
}
|
|
1089
|
+
return this.closePromise;
|
|
1090
|
+
}
|
|
1091
|
+
this.releaseLeaseOnClose = (_a = options.releaseLease) !== null && _a !== void 0 ? _a : true;
|
|
1092
|
+
this.requireClosedOnClose = (_b = options.requireClosed) !== null && _b !== void 0 ? _b : false;
|
|
894
1093
|
this.closed = true;
|
|
1094
|
+
this.closePromise = this.finishClose();
|
|
1095
|
+
return this.closePromise;
|
|
1096
|
+
});
|
|
1097
|
+
}
|
|
1098
|
+
finishClose() {
|
|
1099
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
1100
|
+
var _a, _b;
|
|
895
1101
|
if (this.maintenanceTimer) {
|
|
896
1102
|
clearInterval(this.maintenanceTimer);
|
|
897
1103
|
this.maintenanceTimer = null;
|
|
898
1104
|
}
|
|
1105
|
+
if (this.exclusiveMutationPromise)
|
|
1106
|
+
yield this.exclusiveMutationPromise;
|
|
1107
|
+
yield this.leaseTransitionTail;
|
|
1108
|
+
yield Promise.all([this.drainWrites(), this.drainCompactions()]);
|
|
1109
|
+
this.leaseAbort.abort();
|
|
899
1110
|
// Drain in-flight maintenance before tearing down the connection — otherwise
|
|
900
1111
|
// optimize/createIndex will hit a null db and log "VectorDB connection is closed".
|
|
901
1112
|
if (this.maintenancePromise) {
|
|
@@ -907,15 +1118,49 @@ class VectorDB {
|
|
|
907
1118
|
}
|
|
908
1119
|
(_a = this.unregisterCleanup) === null || _a === void 0 ? void 0 : _a.call(this);
|
|
909
1120
|
this.unregisterCleanup = undefined;
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
1121
|
+
let closeError;
|
|
1122
|
+
if ((_b = this.db) === null || _b === void 0 ? void 0 : _b.close) {
|
|
1123
|
+
try {
|
|
1124
|
+
const closing = this.db.close();
|
|
1125
|
+
if (this.requireClosedOnClose)
|
|
1126
|
+
yield closing;
|
|
1127
|
+
else {
|
|
1128
|
+
yield Promise.race([
|
|
1129
|
+
closing,
|
|
1130
|
+
new Promise((resolve) => setTimeout(resolve, 5000)),
|
|
1131
|
+
]);
|
|
1132
|
+
}
|
|
1133
|
+
}
|
|
1134
|
+
catch (error) {
|
|
1135
|
+
closeError = error;
|
|
916
1136
|
}
|
|
917
1137
|
}
|
|
918
1138
|
this.db = null;
|
|
1139
|
+
if (this.leasePromise) {
|
|
1140
|
+
let lease = null;
|
|
1141
|
+
try {
|
|
1142
|
+
lease = yield this.leasePromise;
|
|
1143
|
+
}
|
|
1144
|
+
catch (_c) {
|
|
1145
|
+
this.leasePromise = null;
|
|
1146
|
+
}
|
|
1147
|
+
if (lease) {
|
|
1148
|
+
try {
|
|
1149
|
+
if (this.releaseLeaseOnClose)
|
|
1150
|
+
yield lease.release();
|
|
1151
|
+
else
|
|
1152
|
+
lease.relinquish(this.leaseHolder);
|
|
1153
|
+
this.leasePromise = null;
|
|
1154
|
+
}
|
|
1155
|
+
catch (error) {
|
|
1156
|
+
closeError = closeError
|
|
1157
|
+
? new Error(`VectorDB close and store lease release failed: ${String(closeError)}; ${String(error)}`)
|
|
1158
|
+
: error;
|
|
1159
|
+
}
|
|
1160
|
+
}
|
|
1161
|
+
}
|
|
1162
|
+
if (closeError)
|
|
1163
|
+
throw closeError;
|
|
919
1164
|
});
|
|
920
1165
|
}
|
|
921
1166
|
}
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.BLOCKED_ROOTS_DESCRIPTION = void 0;
|
|
37
|
+
exports.getBlockedProjectRoots = getBlockedProjectRoots;
|
|
38
|
+
exports.isBlockedProjectRoot = isBlockedProjectRoot;
|
|
39
|
+
const os = __importStar(require("node:os"));
|
|
40
|
+
const path = __importStar(require("node:path"));
|
|
41
|
+
exports.BLOCKED_ROOTS_DESCRIPTION = "home, /, /Users, /tmp, /private, /var, /usr, /opt, /etc, /System, /Library, /Applications";
|
|
42
|
+
function getBlockedProjectRoots() {
|
|
43
|
+
const home = os.homedir();
|
|
44
|
+
return new Set([
|
|
45
|
+
home,
|
|
46
|
+
path.dirname(home),
|
|
47
|
+
"/",
|
|
48
|
+
"/tmp",
|
|
49
|
+
"/private",
|
|
50
|
+
"/private/tmp",
|
|
51
|
+
"/private/var",
|
|
52
|
+
"/var",
|
|
53
|
+
"/usr",
|
|
54
|
+
"/opt",
|
|
55
|
+
"/etc",
|
|
56
|
+
"/System",
|
|
57
|
+
"/Library",
|
|
58
|
+
"/Applications",
|
|
59
|
+
].map((candidate) => path.resolve(candidate)));
|
|
60
|
+
}
|
|
61
|
+
function isBlockedProjectRoot(root) {
|
|
62
|
+
return getBlockedProjectRoots().has(path.resolve(root));
|
|
63
|
+
}
|