grepmax 0.10.2 → 0.10.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -81,11 +81,10 @@ class ProjectBatchProcessor {
|
|
|
81
81
|
if (this.closed || this.processing)
|
|
82
82
|
return;
|
|
83
83
|
try {
|
|
84
|
-
yield this.vectorDb.
|
|
85
|
-
yield this.vectorDb.optimize();
|
|
84
|
+
yield this.vectorDb.runMaintenance();
|
|
86
85
|
}
|
|
87
86
|
catch (err) {
|
|
88
|
-
console.error(`[${this.wtag}]
|
|
87
|
+
console.error(`[${this.wtag}] Maintenance failed:`, err);
|
|
89
88
|
}
|
|
90
89
|
}), FTS_REBUILD_INTERVAL_MS);
|
|
91
90
|
this.ftsInterval.unref();
|
package/dist/lib/index/syncer.js
CHANGED
|
@@ -56,6 +56,7 @@ class VectorDB {
|
|
|
56
56
|
this.lancedbDir = lancedbDir;
|
|
57
57
|
this.db = null;
|
|
58
58
|
this.closed = false;
|
|
59
|
+
this.maintenanceRunning = false;
|
|
59
60
|
this.vectorDim = vectorDim !== null && vectorDim !== void 0 ? vectorDim : config_1.CONFIG.VECTOR_DIM;
|
|
60
61
|
this.unregisterCleanup = (0, cleanup_1.registerCleanup)(() => this.close());
|
|
61
62
|
}
|
|
@@ -275,21 +276,69 @@ class VectorDB {
|
|
|
275
276
|
});
|
|
276
277
|
}
|
|
277
278
|
optimize() {
|
|
278
|
-
return __awaiter(this,
|
|
279
|
+
return __awaiter(this, arguments, void 0, function* (retries = 3) {
|
|
279
280
|
const table = yield this.ensureTable();
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
281
|
+
const cutoff = new Date(Date.now() - 24 * 60 * 60 * 1000);
|
|
282
|
+
for (let attempt = 1; attempt <= retries; attempt++) {
|
|
283
|
+
try {
|
|
284
|
+
const done = (0, logger_1.timer)("vectordb", "optimize");
|
|
285
|
+
const stats = yield table.optimize({
|
|
286
|
+
cleanupOlderThan: cutoff,
|
|
287
|
+
deleteUnverified: true,
|
|
288
|
+
});
|
|
289
|
+
done();
|
|
290
|
+
const { compaction, prune } = stats;
|
|
291
|
+
if (compaction.fragmentsRemoved > 0 ||
|
|
292
|
+
prune.oldVersionsRemoved > 0 ||
|
|
293
|
+
prune.bytesRemoved > 0) {
|
|
294
|
+
(0, logger_1.log)("vectordb", `Compacted: ${compaction.fragmentsRemoved} frags → ${compaction.fragmentsAdded}, ` +
|
|
295
|
+
`pruned ${prune.oldVersionsRemoved} versions, ` +
|
|
296
|
+
`freed ${(prune.bytesRemoved / 1024 / 1024).toFixed(1)}MB`);
|
|
297
|
+
}
|
|
298
|
+
else {
|
|
299
|
+
(0, logger_1.debug)("vectordb", "Optimize: nothing to compact or prune");
|
|
300
|
+
}
|
|
301
|
+
return;
|
|
302
|
+
}
|
|
303
|
+
catch (e) {
|
|
304
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
305
|
+
if (msg.includes("Nothing to do")) {
|
|
306
|
+
(0, logger_1.debug)("vectordb", "Optimize: nothing to do");
|
|
307
|
+
return;
|
|
308
|
+
}
|
|
309
|
+
if (attempt < retries &&
|
|
310
|
+
(msg.includes("conflict") || msg.includes("Retryable"))) {
|
|
311
|
+
const delay = 1000 * Math.pow(2, (attempt - 1));
|
|
312
|
+
(0, logger_1.log)("vectordb", `Optimize conflict (attempt ${attempt}/${retries}), retrying in ${delay}ms`);
|
|
313
|
+
yield new Promise((r) => setTimeout(r, delay));
|
|
314
|
+
continue;
|
|
315
|
+
}
|
|
288
316
|
(0, logger_1.log)("vectordb", `Optimize failed: ${msg}`);
|
|
317
|
+
return;
|
|
289
318
|
}
|
|
290
319
|
}
|
|
291
320
|
});
|
|
292
321
|
}
|
|
322
|
+
/**
|
|
323
|
+
* Run FTS rebuild + optimize as a single serialized operation.
|
|
324
|
+
* Safe to call from multiple project processors — only one runs at a time.
|
|
325
|
+
*/
|
|
326
|
+
runMaintenance() {
|
|
327
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
328
|
+
if (this.maintenanceRunning) {
|
|
329
|
+
(0, logger_1.debug)("vectordb", "Maintenance already running, skipping");
|
|
330
|
+
return;
|
|
331
|
+
}
|
|
332
|
+
this.maintenanceRunning = true;
|
|
333
|
+
try {
|
|
334
|
+
yield this.createFTSIndex();
|
|
335
|
+
yield this.optimize();
|
|
336
|
+
}
|
|
337
|
+
finally {
|
|
338
|
+
this.maintenanceRunning = false;
|
|
339
|
+
}
|
|
340
|
+
});
|
|
341
|
+
}
|
|
293
342
|
hasAnyRows() {
|
|
294
343
|
return __awaiter(this, void 0, void 0, function* () {
|
|
295
344
|
const table = yield this.ensureTable();
|
package/package.json
CHANGED