grepmax 0.26.42 → 0.26.43
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +1 -1
- package/README.md +4 -0
- package/dist/commands/doctor.js +21 -32
- package/dist/commands/mcp.js +2 -0
- package/dist/commands/status.js +4 -2
- package/dist/lib/daemon/daemon.js +22 -7
- package/dist/lib/daemon/ipc-handler.js +11 -10
- package/dist/lib/store/compaction-result.js +35 -0
- package/dist/lib/store/vector-db.js +61 -15
- package/dist/lib/utils/doctor-optimize.js +81 -0
- package/package.json +1 -1
- package/plugins/grepmax/.claude-plugin/plugin.json +1 -1
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
"name": "grepmax",
|
|
11
11
|
"source": "./plugins/grepmax",
|
|
12
12
|
"description": "Semantic code search for Claude Code. Automatically indexes your project and provides intelligent search capabilities.",
|
|
13
|
-
"version": "0.26.
|
|
13
|
+
"version": "0.26.43",
|
|
14
14
|
"author": {
|
|
15
15
|
"name": "Robert Owens",
|
|
16
16
|
"email": "robowens@me.com"
|
package/README.md
CHANGED
|
@@ -452,6 +452,10 @@ gmax watch restart # Restart daemon
|
|
|
452
452
|
|
|
453
453
|
Compaction pauses index writes, waits for pending writes to commit, and opens a fresh table snapshot for each attempt. Each optimize call makes at most two attempts. Before each attempt it checks current free space against twice the table's logical size plus the critical-space reserve (5 GB by default). If there is insufficient headroom, it skips the rewrite and logs the required space; search remains available. A successful prune can reclaim fragment copies left by failed attempts.
|
|
454
454
|
|
|
455
|
+
`~/.gmax/logs/daemon.log` records compaction attempts and outcomes, including duration, logical size, disk size before/after, free space, and bytes reclaimed. `gmax status --json` exposes the last outcome under `daemon.compaction`; its `at` timestamp identifies when it was recorded. Five-minute resource snapshots track footprint, RSS, heap, external/ArrayBuffer memory, Lance cache, workers and pending work for memory investigations. Reading status uses retained snapshots and does not scan the index directory. Logs rotate to `daemon.log.prev`.
|
|
456
|
+
|
|
457
|
+
`gmax doctor --fix` reports completed, skipped, failed, or unverified optimization. It exits nonzero when a requested optimization did not complete or an older daemon could not verify its outcome. A busy or failed daemon is never retried as a second writer in the CLI process.
|
|
458
|
+
|
|
455
459
|
### Known issues
|
|
456
460
|
|
|
457
461
|
Some log output looks alarming but is expected and self-healing. Before filing a bug, check
|
package/dist/commands/doctor.js
CHANGED
|
@@ -51,6 +51,7 @@ const commander_1 = require("commander");
|
|
|
51
51
|
const config_1 = require("../config");
|
|
52
52
|
const embedding_status_1 = require("../lib/index/embedding-status");
|
|
53
53
|
const index_config_1 = require("../lib/index/index-config");
|
|
54
|
+
const doctor_optimize_1 = require("../lib/utils/doctor-optimize");
|
|
54
55
|
const doctor_status_1 = require("../lib/utils/doctor-status");
|
|
55
56
|
const exit_1 = require("../lib/utils/exit");
|
|
56
57
|
const lock_1 = require("../lib/utils/lock");
|
|
@@ -343,7 +344,7 @@ exports.doctor = new commander_1.Command("doctor")
|
|
|
343
344
|
.option("--agent", "Compact output for AI agents", false)
|
|
344
345
|
.action((opts) => __awaiter(void 0, void 0, void 0, function* () {
|
|
345
346
|
var _a, _b;
|
|
346
|
-
var _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o
|
|
347
|
+
var _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o;
|
|
347
348
|
if (!opts.agent)
|
|
348
349
|
console.log("gmax Doctor\n");
|
|
349
350
|
const root = config_1.PATHS.globalRoot;
|
|
@@ -572,7 +573,7 @@ exports.doctor = new commander_1.Command("doctor")
|
|
|
572
573
|
? "LOW"
|
|
573
574
|
: "ok";
|
|
574
575
|
}
|
|
575
|
-
catch (
|
|
576
|
+
catch (_p) { }
|
|
576
577
|
const staleChunkerProjects = projects.filter((p) => { var _a; return p.status === "indexed" &&
|
|
577
578
|
((_a = p.chunkerVersion) !== null && _a !== void 0 ? _a : 1) < config_1.CONFIG.CHUNKER_VERSION; });
|
|
578
579
|
const staleEmbeddingProjects = projects.filter((p) => p.status === "indexed" &&
|
|
@@ -787,7 +788,7 @@ exports.doctor = new commander_1.Command("doctor")
|
|
|
787
788
|
yield mc.close();
|
|
788
789
|
}
|
|
789
790
|
}
|
|
790
|
-
catch (
|
|
791
|
+
catch (_q) { }
|
|
791
792
|
}
|
|
792
793
|
}
|
|
793
794
|
// --fix auto-remediation
|
|
@@ -820,34 +821,22 @@ exports.doctor = new commander_1.Command("doctor")
|
|
|
820
821
|
// daemon up it owns compaction; without one we are the sole process
|
|
821
822
|
// and can do it in-process safely.
|
|
822
823
|
const { sendDaemonCommand } = yield Promise.resolve().then(() => __importStar(require("../lib/utils/daemon-client")));
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
const reason = String((_l = resp.error) !== null && _l !== void 0 ? _l : "");
|
|
830
|
-
// Fall back ONLY when nothing is listening. A daemon that is merely
|
|
831
|
-
// too old ("unknown command") or busy ("timeout") is still a live
|
|
832
|
-
// writer on the store, so optimizing here would recreate the
|
|
833
|
-
// two-writer case this routing exists to prevent.
|
|
834
|
-
const daemonAbsent = reason.includes("ECONNREFUSED") || reason.includes("ENOENT");
|
|
835
|
-
if (daemonAbsent) {
|
|
836
|
-
yield db.optimize(3, 0);
|
|
837
|
-
via = "in-process (no daemon)";
|
|
838
|
-
}
|
|
839
|
-
else if (reason.includes("unknown command")) {
|
|
840
|
-
via = "";
|
|
841
|
-
console.log("WARN Optimize skipped: the running daemon predates the optimize command. Restart it (gmax watch --daemon -b) and re-run.");
|
|
824
|
+
// Release the diagnostic reader before the daemon prunes old files.
|
|
825
|
+
yield db.close();
|
|
826
|
+
const report = yield (0, doctor_optimize_1.runDoctorOptimize)(() => sendDaemonCommand({ cmd: "optimize" }, { timeoutMs: 10 * 60 * 1000 }), () => __awaiter(void 0, void 0, void 0, function* () {
|
|
827
|
+
const repairDb = new VectorDB(config_1.PATHS.lancedbDir);
|
|
828
|
+
try {
|
|
829
|
+
return yield repairDb.optimize();
|
|
842
830
|
}
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
console.log(`WARN Optimize failed via daemon: ${reason}`);
|
|
831
|
+
finally {
|
|
832
|
+
yield repairDb.close();
|
|
846
833
|
}
|
|
847
|
-
}
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
834
|
+
}));
|
|
835
|
+
console.log((0, doctor_optimize_1.formatDoctorOptimize)(report, opts.agent));
|
|
836
|
+
if (report.status === "completed")
|
|
837
|
+
fixed++;
|
|
838
|
+
else
|
|
839
|
+
process.exitCode = 1;
|
|
851
840
|
}
|
|
852
841
|
// Only roots whose volume is mounted and whose directory is still gone
|
|
853
842
|
// reach this list — projects on an unplugged drive are classified
|
|
@@ -877,7 +866,7 @@ exports.doctor = new commander_1.Command("doctor")
|
|
|
877
866
|
const done = yield sendStreamingCommand({ cmd: "index", root: p.root, reset: true }, () => { }).catch((e) => ({ ok: false, error: String(e) }));
|
|
878
867
|
if (done.ok &&
|
|
879
868
|
(!("degraded" in done) || done.degraded !== true)) {
|
|
880
|
-
const chunks = (
|
|
869
|
+
const chunks = (_l = done.indexed) !== null && _l !== void 0 ? _l : 0;
|
|
881
870
|
if (opts.agent) {
|
|
882
871
|
console.log(`stale_chunker_reindexed\tname=${name}\tchunks=${chunks}`);
|
|
883
872
|
}
|
|
@@ -889,14 +878,14 @@ exports.doctor = new commander_1.Command("doctor")
|
|
|
889
878
|
else if (!opts.agent) {
|
|
890
879
|
const degraded = "degraded" in done && done.degraded === true;
|
|
891
880
|
const reason = degraded
|
|
892
|
-
? `${(
|
|
881
|
+
? `${(_m = ("scanErrors" in done ? done.scanErrors : 0)) !== null && _m !== void 0 ? _m : 0} scan errors, ${(_o = ("failedFiles" in done ? done.failedFiles : 0)) !== null && _o !== void 0 ? _o : 0} file failures`
|
|
893
882
|
: done.error;
|
|
894
883
|
console.log(`FAIL ${name}: reindex failed (${reason})`);
|
|
895
884
|
}
|
|
896
885
|
}
|
|
897
886
|
}
|
|
898
887
|
}
|
|
899
|
-
if (fixed === 0) {
|
|
888
|
+
if (fixed === 0 && !needsOptimize) {
|
|
900
889
|
if (!opts.agent)
|
|
901
890
|
console.log("ok Nothing to fix");
|
|
902
891
|
}
|
package/dist/commands/mcp.js
CHANGED
|
@@ -88,6 +88,7 @@ const searcher_1 = require("../lib/search/searcher");
|
|
|
88
88
|
const annotator_1 = require("../lib/skeleton/annotator");
|
|
89
89
|
const skeletonizer_1 = require("../lib/skeleton/skeletonizer");
|
|
90
90
|
const symbol_extractor_1 = require("../lib/skeleton/symbol-extractor");
|
|
91
|
+
const compaction_result_1 = require("../lib/store/compaction-result");
|
|
91
92
|
const meta_cache_1 = require("../lib/store/meta-cache");
|
|
92
93
|
const cross_project_1 = require("../lib/utils/cross-project");
|
|
93
94
|
const daemon_client_1 = require("../lib/utils/daemon-client");
|
|
@@ -1786,6 +1787,7 @@ exports.mcp = new commander_1.Command("mcp")
|
|
|
1786
1787
|
? `Last indexed: ${currentProject.lastIndexed}`
|
|
1787
1788
|
: "",
|
|
1788
1789
|
watcherLine,
|
|
1790
|
+
(0, compaction_result_1.formatCompactionStatus)(daemonStatus.compaction),
|
|
1789
1791
|
`Projects: ${projects.length} indexed (call list_projects for names + per-project chunk counts)`,
|
|
1790
1792
|
].filter(Boolean);
|
|
1791
1793
|
return ok(lines.join("\n"));
|
package/dist/commands/status.js
CHANGED
|
@@ -96,7 +96,9 @@ function loadStatusView(projects) {
|
|
|
96
96
|
chunkCounts.set(project.root, stats.chunks);
|
|
97
97
|
}
|
|
98
98
|
}
|
|
99
|
-
const daemon = Object.assign(Object.assign({}, (resp.
|
|
99
|
+
const daemon = Object.assign(Object.assign(Object.assign({}, (resp.compaction !== undefined
|
|
100
|
+
? { compaction: resp.compaction }
|
|
101
|
+
: {})), (resp.resources !== undefined
|
|
100
102
|
? { resources: resp.resources }
|
|
101
103
|
: {})), { pid: typeof resp.pid === "number" ? resp.pid : null, uptimeSec: typeof resp.uptime === "number" ? resp.uptime : null,
|
|
102
104
|
// Older daemons do not report these.
|
|
@@ -200,7 +202,7 @@ function buildStatusJson(input) {
|
|
|
200
202
|
: null);
|
|
201
203
|
}
|
|
202
204
|
return {
|
|
203
|
-
daemon: Object.assign({ running: d !== undefined, pid: (_d = d === null || d === void 0 ? void 0 : d.pid) !== null && _d !== void 0 ? _d : null, since: (d === null || d === void 0 ? void 0 : d.uptimeSec) != null ? now - d.uptimeSec * 1000 : null, workerThreads: (_e = d === null || d === void 0 ? void 0 : d.workerThreads) !== null && _e !== void 0 ? _e : null }, ((d === null || d === void 0 ? void 0 : d.resources) !== undefined ? { resources: d.resources } : {})),
|
|
205
|
+
daemon: Object.assign(Object.assign({ running: d !== undefined, pid: (_d = d === null || d === void 0 ? void 0 : d.pid) !== null && _d !== void 0 ? _d : null, since: (d === null || d === void 0 ? void 0 : d.uptimeSec) != null ? now - d.uptimeSec * 1000 : null, workerThreads: (_e = d === null || d === void 0 ? void 0 : d.workerThreads) !== null && _e !== void 0 ? _e : null }, ((d === null || d === void 0 ? void 0 : d.resources) !== undefined ? { resources: d.resources } : {})), ((d === null || d === void 0 ? void 0 : d.compaction) !== undefined ? { compaction: d.compaction } : {})),
|
|
204
206
|
settings: {
|
|
205
207
|
embedMode: globalConfig.embedMode,
|
|
206
208
|
modelTier: globalConfig.modelTier,
|
|
@@ -609,13 +609,23 @@ class Daemon {
|
|
|
609
609
|
* fall back to in-process only when no daemon answers, and are then the sole
|
|
610
610
|
* writer themselves.
|
|
611
611
|
*/
|
|
612
|
+
compactionStatus() {
|
|
613
|
+
var _a;
|
|
614
|
+
var _b;
|
|
615
|
+
return (_b = (_a = this.vectorDb) === null || _a === void 0 ? void 0 : _a.compactionStatus()) !== null && _b !== void 0 ? _b : null;
|
|
616
|
+
}
|
|
612
617
|
runOptimize() {
|
|
613
618
|
return __awaiter(this, void 0, void 0, function* () {
|
|
619
|
+
var _a;
|
|
614
620
|
const db = this.vectorDb;
|
|
615
621
|
if (!db)
|
|
616
622
|
throw new Error("daemon resources not ready");
|
|
617
|
-
yield this.runSharedOperation("store-maintenance", undefined, () => db.runMaintenance({ force: true }));
|
|
618
|
-
|
|
623
|
+
const compaction = yield this.runSharedOperation("store-maintenance", undefined, () => db.runMaintenance({ force: true }));
|
|
624
|
+
if (!compaction)
|
|
625
|
+
return { ok: false, error: "maintenance produced no compaction outcome" };
|
|
626
|
+
return Object.assign({ ok: compaction.status === "completed", compaction }, (compaction.status === "completed"
|
|
627
|
+
? {}
|
|
628
|
+
: { error: (_a = compaction.reason) !== null && _a !== void 0 ? _a : compaction.status }));
|
|
619
629
|
});
|
|
620
630
|
}
|
|
621
631
|
operationStatus() {
|
|
@@ -1867,13 +1877,18 @@ class Daemon {
|
|
|
1867
1877
|
if (this.recycleDueSinceMs === null &&
|
|
1868
1878
|
!ageExceeded &&
|
|
1869
1879
|
sampleFootprint &&
|
|
1870
|
-
RSS_WATERMARK_MB > 0
|
|
1871
|
-
// A fresh daemon above the watermark would recycle in a loop; give the
|
|
1872
|
-
// caches a chance to settle before judging the footprint.
|
|
1873
|
-
ageMs > FOOTPRINT_RECYCLE_MIN_AGE_MS) {
|
|
1880
|
+
RSS_WATERMARK_MB > 0) {
|
|
1874
1881
|
footprintMb = (0, process_footprint_1.readFootprintMb)();
|
|
1875
1882
|
}
|
|
1876
|
-
|
|
1883
|
+
// The existing five-minute probe also records a baseline for memory soaks;
|
|
1884
|
+
// the intervening heartbeat ticks add no footprint probe or snapshot.
|
|
1885
|
+
if (sampleFootprint)
|
|
1886
|
+
this.logResourceSnapshot("periodic", footprintMb);
|
|
1887
|
+
const footprintExceeded = footprintMb !== null &&
|
|
1888
|
+
footprintMb > RSS_WATERMARK_MB &&
|
|
1889
|
+
// Sampling young daemons is useful; recycling them would loop before
|
|
1890
|
+
// their caches have had a chance to settle.
|
|
1891
|
+
ageMs > FOOTPRINT_RECYCLE_MIN_AGE_MS;
|
|
1877
1892
|
if (this.recycleDueSinceMs === null) {
|
|
1878
1893
|
if (!ageExceeded && !footprintExceeded)
|
|
1879
1894
|
return;
|
|
@@ -122,8 +122,8 @@ function startHeartbeat(conn, intervalMs = HEARTBEAT_INTERVAL_MS) {
|
|
|
122
122
|
*/
|
|
123
123
|
function handleCommand(daemon, cmd, conn) {
|
|
124
124
|
return __awaiter(this, void 0, void 0, function* () {
|
|
125
|
-
var _a;
|
|
126
|
-
var
|
|
125
|
+
var _a, _b;
|
|
126
|
+
var _c, _d, _e, _f, _g, _h, _j, _k, _l;
|
|
127
127
|
try {
|
|
128
128
|
(0, logger_1.debug)("daemon", `ipc cmd=${cmd.cmd}${cmd.root ? ` root=${cmd.root}` : ""}`);
|
|
129
129
|
if (cmd.cmd !== "ping" &&
|
|
@@ -186,7 +186,8 @@ function handleCommand(daemon, cmd, conn) {
|
|
|
186
186
|
diskPressure: daemon.getDiskPressure(),
|
|
187
187
|
mlx: daemon.getMlxStatus(),
|
|
188
188
|
workers: daemon.workerCount(),
|
|
189
|
-
resources: (
|
|
189
|
+
resources: (_c = (_a = daemon.resourceSnapshot) === null || _a === void 0 ? void 0 : _a.call(daemon)) !== null && _c !== void 0 ? _c : null,
|
|
190
|
+
compaction: (_d = (_b = daemon.compactionStatus) === null || _b === void 0 ? void 0 : _b.call(daemon)) !== null && _d !== void 0 ? _d : null,
|
|
190
191
|
workerThreads: config_1.WORKER_THREADS_SETTING,
|
|
191
192
|
};
|
|
192
193
|
case "optimize":
|
|
@@ -202,14 +203,14 @@ function handleCommand(daemon, cmd, conn) {
|
|
|
202
203
|
return Object.assign({ ok: true }, (yield daemon.projectStats(root)));
|
|
203
204
|
}
|
|
204
205
|
case "shutdown": {
|
|
205
|
-
const reason = String((
|
|
206
|
-
const fromPid = (
|
|
207
|
-
const fromPpid = (
|
|
208
|
-
const fromVer = (
|
|
206
|
+
const reason = String((_e = cmd.reason) !== null && _e !== void 0 ? _e : "unknown");
|
|
207
|
+
const fromPid = (_f = cmd.from_pid) !== null && _f !== void 0 ? _f : "?";
|
|
208
|
+
const fromPpid = (_g = cmd.from_ppid) !== null && _g !== void 0 ? _g : "?";
|
|
209
|
+
const fromVer = (_h = cmd.from_version) !== null && _h !== void 0 ? _h : "?";
|
|
209
210
|
const fromArgv = Array.isArray(cmd.from_argv)
|
|
210
211
|
? cmd.from_argv.join(" ")
|
|
211
212
|
: "?";
|
|
212
|
-
const fromParentCmd = (
|
|
213
|
+
const fromParentCmd = (_j = cmd.from_parent_cmd) !== null && _j !== void 0 ? _j : "?";
|
|
213
214
|
console.log(`[daemon] shutdown command received via IPC: reason=${reason} from_pid=${fromPid} from_ppid=${fromPpid} from_version=${fromVer} from_argv=[${fromArgv}] from_parent_cmd=[${fromParentCmd}]`);
|
|
214
215
|
setImmediate(() => daemon.shutdown());
|
|
215
216
|
return { ok: true };
|
|
@@ -274,8 +275,8 @@ function handleCommand(daemon, cmd, conn) {
|
|
|
274
275
|
return { ok: false, error: "invalid project_roots" };
|
|
275
276
|
}
|
|
276
277
|
for (const prefix of [
|
|
277
|
-
...((
|
|
278
|
-
...((
|
|
278
|
+
...((_k = filters === null || filters === void 0 ? void 0 : filters.inPrefixes) !== null && _k !== void 0 ? _k : []),
|
|
279
|
+
...((_l = filters === null || filters === void 0 ? void 0 : filters.excludePrefixes) !== null && _l !== void 0 ? _l : []),
|
|
279
280
|
]) {
|
|
280
281
|
(0, path_containment_1.resolveContainedPath)(projectRoot, prefix);
|
|
281
282
|
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.skippedCompaction = skippedCompaction;
|
|
4
|
+
exports.parseCompactionResult = parseCompactionResult;
|
|
5
|
+
exports.formatCompactionStatus = formatCompactionStatus;
|
|
6
|
+
function skippedCompaction(reason) {
|
|
7
|
+
return {
|
|
8
|
+
status: "skipped",
|
|
9
|
+
reason,
|
|
10
|
+
at: Date.now(),
|
|
11
|
+
attempts: 0,
|
|
12
|
+
elapsedMs: 0,
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
function parseCompactionResult(value) {
|
|
16
|
+
if (!value || typeof value !== "object")
|
|
17
|
+
return undefined;
|
|
18
|
+
const r = value;
|
|
19
|
+
if (!["completed", "skipped", "failed"].includes(r.status) ||
|
|
20
|
+
!Number.isFinite(r.at) ||
|
|
21
|
+
Number.isNaN(new Date(r.at).getTime()) ||
|
|
22
|
+
!Number.isFinite(r.attempts) ||
|
|
23
|
+
!Number.isFinite(r.elapsedMs) ||
|
|
24
|
+
(r.reason !== undefined && typeof r.reason !== "string"))
|
|
25
|
+
return undefined;
|
|
26
|
+
return r;
|
|
27
|
+
}
|
|
28
|
+
function formatCompactionStatus(value) {
|
|
29
|
+
var _a;
|
|
30
|
+
const result = parseCompactionResult(value);
|
|
31
|
+
if (!result)
|
|
32
|
+
return undefined;
|
|
33
|
+
const reason = (_a = result.reason) === null || _a === void 0 ? void 0 : _a.replace(/[\r\n\t]/g, " ").slice(0, 512);
|
|
34
|
+
return `Last compaction: ${result.status} at ${new Date(result.at).toISOString()} (${result.attempts} attempts, ${result.elapsedMs}ms)${reason ? ` — ${reason}` : ""}`;
|
|
35
|
+
}
|
|
@@ -57,6 +57,7 @@ const cleanup_1 = require("../utils/cleanup");
|
|
|
57
57
|
const filter_builder_1 = require("../utils/filter-builder");
|
|
58
58
|
const logger_1 = require("../utils/logger");
|
|
59
59
|
const ann_config_1 = require("./ann-config");
|
|
60
|
+
const compaction_result_1 = require("./compaction-result");
|
|
60
61
|
const store_lease_1 = require("./store-lease");
|
|
61
62
|
/**
|
|
62
63
|
* Find abandoned Lance scratch files under `<lancedbDir>/ *.lance/data`.
|
|
@@ -230,6 +231,7 @@ class VectorDB {
|
|
|
230
231
|
/** Only an index owner may create/rebuild shared indexes — see markIndexOwner. */
|
|
231
232
|
this.indexOwner = false;
|
|
232
233
|
this.lastOptimizeDidWork = false;
|
|
234
|
+
this.lastCompactionResult = null;
|
|
233
235
|
/** Compaction rate limiter — see COMPACTION_MIN_INTERVAL_MS. */
|
|
234
236
|
this.lastCompactionMs = 0;
|
|
235
237
|
this.compactionIntervalMs = COMPACTION_MIN_INTERVAL_MS;
|
|
@@ -1085,8 +1087,18 @@ class VectorDB {
|
|
|
1085
1087
|
}
|
|
1086
1088
|
});
|
|
1087
1089
|
}
|
|
1090
|
+
compactionStatus() {
|
|
1091
|
+
return this.lastCompactionResult ? Object.assign({}, this.lastCompactionResult) : null;
|
|
1092
|
+
}
|
|
1093
|
+
recordSkippedCompaction(reason) {
|
|
1094
|
+
const result = (0, compaction_result_1.skippedCompaction)(reason);
|
|
1095
|
+
this.lastCompactionResult = result;
|
|
1096
|
+
(0, logger_1.log)("vectordb", `Compaction result: ${JSON.stringify(result)}`);
|
|
1097
|
+
return Object.assign({}, result);
|
|
1098
|
+
}
|
|
1088
1099
|
optimize() {
|
|
1089
1100
|
return __awaiter(this, arguments, void 0, function* (retries = COMPACTION_MAX_ATTEMPTS, retentionMs = 0, bypassExclusiveMutation = false) {
|
|
1101
|
+
var _a;
|
|
1090
1102
|
if (!bypassExclusiveMutation) {
|
|
1091
1103
|
while (this.exclusiveMutationPromise)
|
|
1092
1104
|
yield this.exclusiveMutationPromise;
|
|
@@ -1096,7 +1108,7 @@ class VectorDB {
|
|
|
1096
1108
|
if (this.compactingPromise) {
|
|
1097
1109
|
(0, logger_1.debug)("vectordb", "Optimize already in progress, skipping");
|
|
1098
1110
|
yield this.compactingPromise;
|
|
1099
|
-
return;
|
|
1111
|
+
return ((_a = this.compactionStatus()) !== null && _a !== void 0 ? _a : (0, compaction_result_1.skippedCompaction)("compaction unavailable"));
|
|
1100
1112
|
}
|
|
1101
1113
|
this.lastOptimizeDidWork = false;
|
|
1102
1114
|
let resolveCompacting;
|
|
@@ -1104,6 +1116,29 @@ class VectorDB {
|
|
|
1104
1116
|
resolveCompacting = resolve;
|
|
1105
1117
|
});
|
|
1106
1118
|
this.activeCompactions++;
|
|
1119
|
+
const startedAt = Date.now();
|
|
1120
|
+
let attempts = 0;
|
|
1121
|
+
let logicalBytes;
|
|
1122
|
+
let diskBytesBefore;
|
|
1123
|
+
let freeBytesBefore;
|
|
1124
|
+
const finish = (status, reason, bytesReclaimed) => {
|
|
1125
|
+
const result = {
|
|
1126
|
+
status,
|
|
1127
|
+
at: Date.now(),
|
|
1128
|
+
attempts,
|
|
1129
|
+
elapsedMs: Date.now() - startedAt,
|
|
1130
|
+
reason: reason === null || reason === void 0 ? void 0 : reason.replace(/[\r\n\t]/g, " ").slice(0, 512),
|
|
1131
|
+
logicalBytes,
|
|
1132
|
+
diskBytesBefore,
|
|
1133
|
+
freeBytesBefore,
|
|
1134
|
+
diskBytesAfter: this.getDirectorySize(this.lancedbDir),
|
|
1135
|
+
freeBytesAfter: this.getAvailableBytes(),
|
|
1136
|
+
bytesReclaimed,
|
|
1137
|
+
};
|
|
1138
|
+
this.lastCompactionResult = result;
|
|
1139
|
+
(0, logger_1.log)("vectordb", `Compaction result: ${JSON.stringify(result)}`);
|
|
1140
|
+
return Object.assign({}, result);
|
|
1141
|
+
};
|
|
1107
1142
|
try {
|
|
1108
1143
|
const maxAttempts = Math.min(retries, COMPACTION_MAX_ATTEMPTS);
|
|
1109
1144
|
let rebuiltFts = false;
|
|
@@ -1123,10 +1158,17 @@ class VectorDB {
|
|
|
1123
1158
|
// for rewrite/index overhead, leaving the critical reserve untouched.
|
|
1124
1159
|
const availableBytes = this.getAvailableBytes();
|
|
1125
1160
|
const requiredBytes = totalBytes * 2 + config_1.DISK_CRITICAL_BYTES;
|
|
1161
|
+
logicalBytes = totalBytes;
|
|
1162
|
+
const diskBytes = this.getDirectorySize(this.lancedbDir);
|
|
1163
|
+
diskBytesBefore !== null && diskBytesBefore !== void 0 ? diskBytesBefore : (diskBytesBefore = diskBytes);
|
|
1164
|
+
freeBytesBefore !== null && freeBytesBefore !== void 0 ? freeBytesBefore : (freeBytesBefore = availableBytes);
|
|
1126
1165
|
if (availableBytes < requiredBytes) {
|
|
1127
|
-
|
|
1128
|
-
|
|
1166
|
+
const reason = `insufficient rewrite headroom (${(availableBytes / Math.pow(1024, 3)).toFixed(1)}GB free, ${(requiredBytes / Math.pow(1024, 3)).toFixed(1)}GB required)`;
|
|
1167
|
+
(0, logger_1.log)("vectordb", `Optimize skipped: ${reason}`);
|
|
1168
|
+
return finish("skipped", reason);
|
|
1129
1169
|
}
|
|
1170
|
+
attempts++;
|
|
1171
|
+
(0, logger_1.log)("vectordb", `Compaction attempt: ${JSON.stringify({ attempt, maxAttempts, logicalBytes, diskBytes, freeBytes: availableBytes, requiredBytes })}`);
|
|
1130
1172
|
const done = (0, logger_1.timer)("vectordb", "optimize");
|
|
1131
1173
|
// deleteUnverified deletes files Lance cannot prove are unreferenced.
|
|
1132
1174
|
// LanceDB's own docs: only safe "if you can guarantee that no other
|
|
@@ -1156,19 +1198,20 @@ class VectorDB {
|
|
|
1156
1198
|
else {
|
|
1157
1199
|
(0, logger_1.debug)("vectordb", "Optimize: nothing to compact or prune");
|
|
1158
1200
|
}
|
|
1159
|
-
return;
|
|
1201
|
+
return finish("completed", undefined, prune.bytesRemoved);
|
|
1160
1202
|
}
|
|
1161
1203
|
catch (e) {
|
|
1162
1204
|
const msg = e instanceof Error ? e.message : String(e);
|
|
1163
1205
|
if (msg.includes("Nothing to do")) {
|
|
1164
1206
|
(0, logger_1.debug)("vectordb", "Optimize: nothing to do");
|
|
1165
|
-
return;
|
|
1207
|
+
return finish("completed", "nothing to compact or prune", 0);
|
|
1166
1208
|
}
|
|
1209
|
+
(0, logger_1.log)("vectordb", `Compaction attempt failed: ${JSON.stringify({ attempt, elapsedMs: Date.now() - startedAt, diskBytes: this.getDirectorySize(this.lancedbDir), freeBytes: this.getAvailableBytes(), reason: msg.slice(0, 512) })}`);
|
|
1167
1210
|
// ENOSPC: return immediately — retrying will only make things worse
|
|
1168
1211
|
if (msg.includes("No space left on device") ||
|
|
1169
1212
|
msg.includes("os error 28")) {
|
|
1170
1213
|
(0, logger_1.log)("vectordb", `Optimize failed (ENOSPC): disk full — skipping retries`);
|
|
1171
|
-
return;
|
|
1214
|
+
return finish("failed", "disk full — skipping retries");
|
|
1172
1215
|
}
|
|
1173
1216
|
if (attempt < maxAttempts &&
|
|
1174
1217
|
(msg.includes("conflict") || msg.includes("Retryable"))) {
|
|
@@ -1198,7 +1241,7 @@ class VectorDB {
|
|
|
1198
1241
|
catch (rebuildErr) {
|
|
1199
1242
|
this.ftsPanicRecoveryExhausted = true;
|
|
1200
1243
|
(0, logger_1.log)("vectordb", `FTS rebuild failed: ${rebuildErr}`);
|
|
1201
|
-
return;
|
|
1244
|
+
return finish("failed", `FTS rebuild failed: ${rebuildErr}`);
|
|
1202
1245
|
}
|
|
1203
1246
|
}
|
|
1204
1247
|
if (rebuiltFts) {
|
|
@@ -1207,9 +1250,10 @@ class VectorDB {
|
|
|
1207
1250
|
}
|
|
1208
1251
|
}
|
|
1209
1252
|
(0, logger_1.log)("vectordb", `Optimize failed: ${msg}`);
|
|
1210
|
-
return;
|
|
1253
|
+
return finish("failed", msg);
|
|
1211
1254
|
}
|
|
1212
1255
|
}
|
|
1256
|
+
return finish("skipped", "no compaction attempts requested");
|
|
1213
1257
|
}
|
|
1214
1258
|
finally {
|
|
1215
1259
|
this.compactingPromise = null;
|
|
@@ -1232,16 +1276,17 @@ class VectorDB {
|
|
|
1232
1276
|
return __awaiter(this, arguments, void 0, function* (options = {}) {
|
|
1233
1277
|
if (this.maintenanceRunning) {
|
|
1234
1278
|
(0, logger_1.debug)("vectordb", "Maintenance already running, skipping");
|
|
1235
|
-
return;
|
|
1279
|
+
return (0, compaction_result_1.skippedCompaction)("maintenance already running");
|
|
1236
1280
|
}
|
|
1237
1281
|
this.maintenanceRunning = true;
|
|
1238
1282
|
const epochSnapshot = this.writeEpoch;
|
|
1283
|
+
let result;
|
|
1239
1284
|
try {
|
|
1240
1285
|
const pressure = this.checkDiskPressure();
|
|
1241
1286
|
if (pressure === "critical") {
|
|
1242
1287
|
const freeGb = (this.getAvailableBytes() / 1024 / 1024 / 1024).toFixed(1);
|
|
1243
1288
|
(0, logger_1.log)("vectordb", `Maintenance skipped: disk critically low (${freeGb}GB free)`);
|
|
1244
|
-
return;
|
|
1289
|
+
return this.recordSkippedCompaction(`disk critically low (${freeGb}GB free)`);
|
|
1245
1290
|
}
|
|
1246
1291
|
if (!options.force &&
|
|
1247
1292
|
epochSnapshot === this.maintainedEpoch &&
|
|
@@ -1270,12 +1315,12 @@ class VectorDB {
|
|
|
1270
1315
|
}
|
|
1271
1316
|
else if (pressure === "low") {
|
|
1272
1317
|
(0, logger_1.log)("vectordb", `Low disk — single-pass optimize (no bloat retry)`);
|
|
1273
|
-
yield this.optimize(1, 0, true);
|
|
1318
|
+
result = yield this.optimize(1, 0, true);
|
|
1274
1319
|
this.noteCompaction(this.lastOptimizeDidWork);
|
|
1275
1320
|
}
|
|
1276
1321
|
else {
|
|
1277
1322
|
// Normal maintenance: full optimize + bloat check
|
|
1278
|
-
yield this.optimize(5, 0, true);
|
|
1323
|
+
result = yield this.optimize(5, 0, true);
|
|
1279
1324
|
// Track across both passes: `lastOptimizeDidWork` is reset per optimize()
|
|
1280
1325
|
// call, so a productive first pass followed by a barren bloat retry would
|
|
1281
1326
|
// otherwise read as unproductive and trigger a spurious backoff.
|
|
@@ -1300,7 +1345,7 @@ class VectorDB {
|
|
|
1300
1345
|
if (bloatRatio > 2.0 && this.checkDiskPressure() === "ok") {
|
|
1301
1346
|
(0, logger_1.log)("vectordb", `Bloat detected after optimize: ${(diskSize / 1024 / 1024).toFixed(0)}MB disk vs ${(logicalSize / 1024 / 1024).toFixed(0)}MB logical (${bloatRatio.toFixed(1)}x) — retrying`);
|
|
1302
1347
|
yield new Promise((r) => setTimeout(r, 2000));
|
|
1303
|
-
yield this.optimize(5, 0, true);
|
|
1348
|
+
result = yield this.optimize(5, 0, true);
|
|
1304
1349
|
didWork = didWork || this.lastOptimizeDidWork;
|
|
1305
1350
|
}
|
|
1306
1351
|
}
|
|
@@ -1312,6 +1357,7 @@ class VectorDB {
|
|
|
1312
1357
|
yield this.readTableVersion(maintainedTable);
|
|
1313
1358
|
this.maintainedEpoch = epochSnapshot;
|
|
1314
1359
|
this.lastMaintenanceMs = Date.now();
|
|
1360
|
+
return result;
|
|
1315
1361
|
}
|
|
1316
1362
|
finally {
|
|
1317
1363
|
this.maintenanceRunning = false;
|
|
@@ -1376,9 +1422,9 @@ class VectorDB {
|
|
|
1376
1422
|
return false;
|
|
1377
1423
|
}
|
|
1378
1424
|
(0, logger_1.log)("vectordb", `Fragment threshold exceeded (${stats.fragmentStats.numSmallFragments} > ${threshold}) — compacting`);
|
|
1379
|
-
yield this.optimize(2, 0, true);
|
|
1425
|
+
const result = yield this.optimize(2, 0, true);
|
|
1380
1426
|
this.noteCompaction(this.lastOptimizeDidWork);
|
|
1381
|
-
return
|
|
1427
|
+
return result.status === "completed";
|
|
1382
1428
|
}
|
|
1383
1429
|
}
|
|
1384
1430
|
catch (err) {
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.runDoctorOptimize = runDoctorOptimize;
|
|
13
|
+
exports.formatDoctorOptimize = formatDoctorOptimize;
|
|
14
|
+
const compaction_result_1 = require("../store/compaction-result");
|
|
15
|
+
/** A live daemon's failed repair must never become a competing local rewrite. */
|
|
16
|
+
function runDoctorOptimize(send, optimizeInProcess) {
|
|
17
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
18
|
+
var _a, _b;
|
|
19
|
+
const response = yield send();
|
|
20
|
+
const result = (0, compaction_result_1.parseCompactionResult)(response.compaction);
|
|
21
|
+
if (result) {
|
|
22
|
+
return {
|
|
23
|
+
status: !response.ok && result.status === "completed"
|
|
24
|
+
? "failed"
|
|
25
|
+
: result.status,
|
|
26
|
+
via: "daemon",
|
|
27
|
+
reason: (_a = result.reason) !== null && _a !== void 0 ? _a : (response.ok ? undefined : String(response.error)),
|
|
28
|
+
result,
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
if (response.ok) {
|
|
32
|
+
return {
|
|
33
|
+
status: "unverified",
|
|
34
|
+
via: "daemon",
|
|
35
|
+
reason: "daemon did not report a compaction outcome; restart it with the current gmax version",
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
const reason = String((_b = response.error) !== null && _b !== void 0 ? _b : "unknown failure");
|
|
39
|
+
// Socket errors are returned as exact errno codes by sendDaemonCommand.
|
|
40
|
+
// An ENOENT inside a daemon's failure message is not proof it is absent.
|
|
41
|
+
if (reason === "ECONNREFUSED" || reason === "ENOENT") {
|
|
42
|
+
try {
|
|
43
|
+
const local = yield optimizeInProcess();
|
|
44
|
+
return {
|
|
45
|
+
status: local.status,
|
|
46
|
+
via: "in-process",
|
|
47
|
+
reason: local.reason,
|
|
48
|
+
result: local,
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
catch (error) {
|
|
52
|
+
return { status: "failed", via: "in-process", reason: String(error) };
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return {
|
|
56
|
+
status: reason.includes("unknown command") ? "skipped" : "failed",
|
|
57
|
+
via: "daemon",
|
|
58
|
+
reason,
|
|
59
|
+
};
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
function formatDoctorOptimize(report, agent) {
|
|
63
|
+
var _a;
|
|
64
|
+
const reason = (_a = report.reason) === null || _a === void 0 ? void 0 : _a.replace(/[\r\n\t]/g, " ").slice(0, 512);
|
|
65
|
+
if (agent) {
|
|
66
|
+
const r = report.result;
|
|
67
|
+
return [
|
|
68
|
+
`optimize`,
|
|
69
|
+
`status=${report.status}`,
|
|
70
|
+
`via=${report.via}`,
|
|
71
|
+
...(r ? [`attempts=${r.attempts}`, `elapsed_ms=${r.elapsedMs}`] : []),
|
|
72
|
+
...(typeof (r === null || r === void 0 ? void 0 : r.bytesReclaimed) === "number"
|
|
73
|
+
? [`bytes_reclaimed=${r.bytesReclaimed}`]
|
|
74
|
+
: []),
|
|
75
|
+
...(reason ? [`reason=${reason}`] : []),
|
|
76
|
+
].join("\t");
|
|
77
|
+
}
|
|
78
|
+
if (report.status === "completed")
|
|
79
|
+
return `ok Optimize complete (${report.via})`;
|
|
80
|
+
return `${report.status === "failed" ? "FAIL" : "WARN"} Optimize ${report.status} (${report.via}): ${reason !== null && reason !== void 0 ? reason : "no outcome reported"}`;
|
|
81
|
+
}
|
package/package.json
CHANGED