grepmax 0.26.11 → 0.26.13

package/.claude-plugin/marketplace.json

@@ -10,7 +10,7 @@
       "name": "grepmax",
       "source": "./plugins/grepmax",
       "description": "Semantic code search for Claude Code. Automatically indexes your project and provides intelligent search capabilities.",
-      "version": "0.26.11",
+      "version": "0.26.13",
       "author": {
         "name": "Robert Owens",
         "email": "robowens@me.com"

package/dist/commands/doctor.js

@@ -88,7 +88,7 @@ exports.doctor = new commander_1.Command("doctor")
     .option("--fix", "Auto-fix detected issues (compact, prune, remove stale locks)", false)
     .option("--agent", "Compact output for AI agents", false)
     .action((opts) => __awaiter(void 0, void 0, void 0, function* () {
-    var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m;
+    var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o;
     if (!opts.agent)
         console.log("gmax Doctor\n");
     const root = config_1.PATHS.globalRoot;
@@ -198,7 +198,7 @@ exports.doctor = new commander_1.Command("doctor")
     let staleLock = false;
     const orphanedProjects = [];
     try {
-        const { VectorDB } = yield Promise.resolve().then(() => __importStar(require("../lib/store/vector-db")));
+        const { VectorDB, findStaleLanceTempFiles, sweepStaleLanceTempFiles } = yield Promise.resolve().then(() => __importStar(require("../lib/store/vector-db")));
         const db = new VectorDB(config_1.PATHS.lancedbDir);
         const table = yield db.ensureTable();
         const totalChunks = yield table.countRows();
@@ -258,7 +258,16 @@ exports.doctor = new commander_1.Command("doctor")
             }
         }
         // Compute warning flags
-        const bloatRatio = logicalSize > 0 ? diskSize / logicalSize : 0;
+        //
+        // Abandoned `.tmp*` scratch files are disk that optimize provably cannot
+        // reclaim (no manifest entry, so `deleteUnverified` will not touch them).
+        // They get their own remediation — a sweep — and are excluded from the
+        // ratio that decides whether to run a whole-table optimize, because
+        // counting them there prescribes a slow rewrite for bytes it cannot free.
+        const staleTempFiles = findStaleLanceTempFiles(config_1.PATHS.lancedbDir);
+        const staleTempBytes = staleTempFiles.reduce((n, f) => n + f.size, 0);
+        const reclaimableDiskSize = Math.max(0, diskSize - staleTempBytes);
+        const bloatRatio = logicalSize > 0 ? reclaimableDiskSize / logicalSize : 0;
         if (bloatRatio > 2.0)
             needsOptimize = true;
         // Track the compactor's own threshold. Warning (and worse, --fix forcing a
@@ -281,7 +290,7 @@ exports.doctor = new commander_1.Command("doctor")
                         ? "LOW"
                         : "ok";
         }
-        catch (_o) { }
+        catch (_p) { }
         const staleChunkerProjects = projects.filter((p) => {
             var _a;
             return p.status === "indexed" &&
@@ -304,6 +313,8 @@ exports.doctor = new commander_1.Command("doctor")
                 `lock=${lockStatus.split(" ")[0]}`,
                 `daemon=${daemonUp ? "running" : "stopped"}`,
                 `orphaned=${orphanedProjects.length}`,
+                `stale_temp=${staleTempFiles.length}`,
+                `stale_temp_bytes=${staleTempBytes}`,
                 `stale_chunker=${staleChunkerProjects.length}`,
                 `stale_embedding=${staleEmbeddingProjects.length}`,
                 `legacy_embedding=${legacyEmbeddingCount}`,
@@ -380,11 +391,15 @@ exports.doctor = new commander_1.Command("doctor")
                 console.log(`ok  Disk: ${formatSize(availBytes)} available`);
             }
             // Storage
+            const storageLine = `${totalChunks.toLocaleString()} rows, ${formatSize(logicalSize)} logical, ${formatSize(diskSize)} disk`;
             if (bloatRatio > 2.0) {
-                console.log(`WARN  Storage: ${totalChunks.toLocaleString()} rows, ${formatSize(logicalSize)} logical, ${formatSize(diskSize)} disk (${bloatRatio.toFixed(1)}x — orphaned files)`);
+                console.log(`WARN  Storage: ${storageLine} (${bloatRatio.toFixed(1)}x — orphaned files)`);
             }
             else {
-                console.log(`ok  Storage: ${totalChunks.toLocaleString()} rows, ${formatSize(logicalSize)} logical, ${formatSize(diskSize)} disk`);
+                console.log(`ok  Storage: ${storageLine}`);
+            }
+            if (staleTempFiles.length > 0) {
+                console.log(`WARN  Abandoned temp files: ${staleTempFiles.length} (${formatSize(staleTempBytes)}) — leftover from interrupted compaction, run with --fix`);
             }
             // Fragments
             if (numSmallFragments > config_1.FRAGMENT_COMPACT_THRESHOLD) {
@@ -486,7 +501,7 @@ exports.doctor = new commander_1.Command("doctor")
                         yield mc.close();
                     }
                 }
-                catch (_p) { }
+                catch (_q) { }
             }
         }
         // --fix auto-remediation
@@ -500,12 +515,52 @@ exports.doctor = new commander_1.Command("doctor")
                     console.log("ok  Removed stale lock");
                 fixed++;
             }
+            // Sweep before optimize: these bytes are unreachable by optimize, and
+            // clearing them first means any bloat still visible afterwards is real.
+            // Unlike optimize this needs no single-writer routing — the age gate in
+            // findStaleLanceTempFiles already excludes anything a writer could hold.
+            if (staleTempFiles.length > 0) {
+                const swept = sweepStaleLanceTempFiles(config_1.PATHS.lancedbDir);
+                if (!opts.agent)
+                    console.log(`ok  Removed ${swept.filesRemoved} abandoned temp file(s), freed ${formatSize(swept.bytesFreed)}`);
+                fixed++;
+            }
             if (needsOptimize) {
                 if (!opts.agent)
                     console.log("...  Running optimize (compact + prune)...");
-                yield db.optimize(3, 0);
-                if (!opts.agent)
-                    console.log("ok  Optimize complete");
+                // Prefer the daemon. Lance's optimize deletes unverified files, which
+                // is only safe with a single writer on the store — optimizing here
+                // while the daemon compacts made this command a second writer. With a
+                // daemon up it owns compaction; without one we are the sole process
+                // and can do it in-process safely.
+                const { sendDaemonCommand } = yield Promise.resolve().then(() => __importStar(require("../lib/utils/daemon-client")));
+                const resp = yield sendDaemonCommand({ cmd: "optimize" }, { timeoutMs: 10 * 60 * 1000 });
+                let via = "daemon";
+                if (!resp.ok) {
+                    // Only fall back when the daemon could not take it. A daemon that
+                    // answered with a real failure should surface, not be retried here
+                    // as a competing writer.
+                    const reason = String((_k = resp.error) !== null && _k !== void 0 ? _k : "");
+                    // Fall back ONLY when nothing is listening. A daemon that is merely
+                    // too old ("unknown command") or busy ("timeout") is still a live
+                    // writer on the store, so optimizing here would recreate the
+                    // two-writer case this routing exists to prevent.
+                    const daemonAbsent = reason.includes("ECONNREFUSED") || reason.includes("ENOENT");
+                    if (daemonAbsent) {
+                        yield db.optimize(3, 0);
+                        via = "in-process (no daemon)";
+                    }
+                    else if (reason.includes("unknown command")) {
+                        via = "";
+                        console.log("WARN  Optimize skipped: the running daemon predates the optimize command. Restart it (gmax watch --daemon -b) and re-run.");
+                    }
+                    else {
+                        via = "";
+                        console.log(`WARN  Optimize failed via daemon: ${reason}`);
+                    }
+                }
+                if (via && !opts.agent)
+                    console.log(`ok  Optimize complete (${via})`);
                 fixed++;
             }
             if (orphanedProjects.length > 0) {
@@ -533,7 +588,7 @@ exports.doctor = new commander_1.Command("doctor")
                         const done = yield sendStreamingCommand({ cmd: "index", root: p.root, reset: true }, () => { }).catch((e) => ({ ok: false, error: String(e) }));
                         if (done.ok &&
                             (!("degraded" in done) || done.degraded !== true)) {
-                            const chunks = (_k = done.indexed) !== null && _k !== void 0 ? _k : 0;
+                            const chunks = (_l = done.indexed) !== null && _l !== void 0 ? _l : 0;
                             if (opts.agent) {
                                 console.log(`stale_chunker_reindexed\tname=${name}\tchunks=${chunks}`);
                             }
@@ -545,7 +600,7 @@ exports.doctor = new commander_1.Command("doctor")
                         else if (!opts.agent) {
                             const degraded = "degraded" in done && done.degraded === true;
                             const reason = degraded
-                                ? `${(_l = ("scanErrors" in done ? done.scanErrors : 0)) !== null && _l !== void 0 ? _l : 0} scan errors, ${(_m = ("failedFiles" in done ? done.failedFiles : 0)) !== null && _m !== void 0 ? _m : 0} file failures`
+                                ? `${(_m = ("scanErrors" in done ? done.scanErrors : 0)) !== null && _m !== void 0 ? _m : 0} scan errors, ${(_o = ("failedFiles" in done ? done.failedFiles : 0)) !== null && _o !== void 0 ? _o : 0} file failures`
                                 : done.error;
                             console.log(`FAIL  ${name}: reindex failed (${reason})`);
                         }

package/dist/config.js

@@ -33,7 +33,7 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.INDEXABLE_EXTENSIONS = exports.VERSION_PRUNE_THRESHOLD = exports.FRAGMENT_COMPACT_THRESHOLD = exports.DISK_LOW_BYTES = exports.DISK_CRITICAL_BYTES = exports.MAX_FILE_SIZE_BYTES = exports.PATHS = exports.MAX_WORKER_MEMORY_MB = exports.WORKER_BOOT_TIMEOUT_MS = exports.WORKER_TIMEOUT_MS = exports.REBUILD_COMMAND = exports.CHUNKER_VERSION_HISTORY = exports.CONFIG = exports.MODEL_IDS = exports.DEFAULT_MODEL_TIER = exports.MODEL_TIERS = void 0;
+exports.INDEXABLE_EXTENSIONS = exports.STALE_TEMP_FILE_AGE_MS = exports.VERSION_PRUNE_THRESHOLD = exports.FRAGMENT_COMPACT_THRESHOLD = exports.DISK_LOW_BYTES = exports.DISK_CRITICAL_BYTES = exports.MAX_FILE_SIZE_BYTES = exports.PATHS = exports.MAX_WORKER_MEMORY_MB = exports.WORKER_BOOT_TIMEOUT_MS = exports.WORKER_TIMEOUT_MS = exports.REBUILD_COMMAND = exports.CHUNKER_VERSION_HISTORY = exports.CONFIG = exports.MODEL_IDS = exports.DEFAULT_MODEL_TIER = exports.MODEL_TIERS = void 0;
 exports.describeChunkerGap = describeChunkerGap;
 exports.describeEmbeddingGap = describeEmbeddingGap;
 exports.describeSchemaDimGap = describeSchemaDimGap;
@@ -281,6 +281,19 @@ exports.FRAGMENT_COMPACT_THRESHOLD = 400;
 // fire on every healthy store — and make `doctor --fix` force the whole-table
 // rewrite that the fragment threshold exists to avoid.
 exports.VERSION_PRUNE_THRESHOLD = 2000;
+// Age past which a Lance `.tmp*` scratch file in a `*.lance/data` directory is
+// considered abandoned rather than in flight. Lance writes a new fragment to a
+// dot-prefixed temp file and renames it on success, so a killed optimize leaves
+// the temp behind forever: it is in no manifest, so `optimize({deleteUnverified})`
+// never collects it, and the dot prefix hides it from a plain `ls`. Three such
+// files (7.3 GB, up to 2.5 months old) were what kept this store pinned at a 2.1x
+// disk/logical ratio that repeated optimize passes could not move.
+//
+// The age gate is the whole safety argument: an in-flight write keeps bumping its
+// temp file's mtime, so anything this stale has no writer, and deleting it is safe
+// without knowing how many processes are on the store. Observed optimize runs on a
+// 325k-row store peak at ~1.4 min, so an hour is ~40x the worst case seen.
+exports.STALE_TEMP_FILE_AGE_MS = 60 * 60 * 1000;
 // Extensions we consider for indexing to avoid binary noise and improve relevance.
 exports.INDEXABLE_EXTENSIONS = new Set([
     ".ts",

package/dist/lib/daemon/daemon.js

@@ -486,6 +486,28 @@ class Daemon {
     runSharedOperation(name, signal, fn) {
         return this.operations.runShared(name, signal, fn);
     }
+    /**
+     * Compact + prune on behalf of a CLI caller.
+     *
+     * Lance's optimize runs with `deleteUnverified: true`, which its own docs say
+     * is only safe when no other process is working on the dataset. That held by
+     * accident, not by design: the daemon serializes its own maintenance, but
+     * `gmax doctor --fix` optimized in-process, so a user running it during a
+     * compaction was a second writer on a store that assumes one. Routing the
+     * request here makes it structural — with a daemon up, the daemon is the only
+     * optimizer, sharing the same serialization as the maintenance loop. Callers
+     * fall back to in-process only when no daemon answers, and are then the sole
+     * writer themselves.
+     */
+    runOptimize() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const db = this.vectorDb;
+            if (!db)
+                throw new Error("daemon resources not ready");
+            yield this.runSharedOperation("store-maintenance", undefined, () => db.runMaintenance({ force: true }));
+            return { ok: true };
+        });
+    }
     operationStatus() {
         return this.operations.status;
     }

package/dist/lib/daemon/ipc-handler.js

@@ -173,6 +173,9 @@ function handleCommand(daemon, cmd, conn) {
                         diskPressure: daemon.getDiskPressure(),
                         mlx: daemon.getMlxStatus(),
                     };
+                case "optimize":
+                    // Keeps compaction single-writer: see Daemon.runOptimize.
+                    return yield daemon.runOptimize();
                 case "project-stats": {
                     const root = String(cmd.root || "");
                     if (!root)

package/dist/lib/store/vector-db.js

@@ -43,6 +43,8 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.VectorDB = exports.DiskPressureError = void 0;
+exports.findStaleLanceTempFiles = findStaleLanceTempFiles;
+exports.sweepStaleLanceTempFiles = sweepStaleLanceTempFiles;
 exports.isLanceCorruptionError = isLanceCorruptionError;
 exports.isMissingTableError = isMissingTableError;
 const fs = __importStar(require("node:fs"));
@@ -55,6 +57,73 @@ const filter_builder_1 = require("../utils/filter-builder");
 const logger_1 = require("../utils/logger");
 const ann_config_1 = require("./ann-config");
 const store_lease_1 = require("./store-lease");
+/**
+ * Find abandoned Lance scratch files under `<lancedbDir>/ *.lance/data`.
+ *
+ * Lance stages a new fragment in a dot-prefixed `.tmp*` file and renames it into
+ * place on success. A killed or panicking optimize leaves the temp behind, and
+ * nothing ever collects it: it appears in no manifest, so `deleteUnverified`
+ * cannot prove it is garbage, and the leading dot hides it from `ls`. The result
+ * is permanent disk bloat that no amount of optimizing reclaims.
+ *
+ * Only files older than `minAgeMs` are reported. An in-flight write keeps
+ * bumping its temp file's mtime, so the age gate — not a writer count — is what
+ * makes deleting the result safe from any process.
+ */
+function findStaleLanceTempFiles(lancedbDir, minAgeMs = config_1.STALE_TEMP_FILE_AGE_MS) {
+    const found = [];
+    const cutoff = Date.now() - minAgeMs;
+    let tables;
+    try {
+        tables = fs.readdirSync(lancedbDir);
+    }
+    catch (_a) {
+        return found;
+    }
+    for (const tableDir of tables) {
+        if (!tableDir.endsWith(".lance"))
+            continue;
+        const dataDir = `${lancedbDir}/${tableDir}/data`;
+        let entries;
+        try {
+            entries = fs.readdirSync(dataDir);
+        }
+        catch (_b) {
+            continue;
+        }
+        for (const entry of entries) {
+            if (!entry.startsWith(".tmp"))
+                continue;
+            const full = `${dataDir}/${entry}`;
+            try {
+                const s = fs.statSync(full);
+                if (!s.isFile() || s.mtimeMs > cutoff)
+                    continue;
+                found.push({ path: full, size: s.size, mtimeMs: s.mtimeMs });
+            }
+            catch (_c) { }
+        }
+    }
+    return found;
+}
+/**
+ * Delete abandoned Lance scratch files. Returns bytes actually reclaimed —
+ * a file that vanishes between the scan and the unlink is not counted, so the
+ * number reported is never larger than what was freed.
+ */
+function sweepStaleLanceTempFiles(lancedbDir, minAgeMs = config_1.STALE_TEMP_FILE_AGE_MS) {
+    let filesRemoved = 0;
+    let bytesFreed = 0;
+    for (const f of findStaleLanceTempFiles(lancedbDir, minAgeMs)) {
+        try {
+            fs.unlinkSync(f.path);
+            filesRemoved++;
+            bytesFreed += f.size;
+        }
+        catch (_a) { }
+    }
+    return { filesRemoved, bytesFreed };
+}
 class DiskPressureError extends Error {
     constructor(message = "Disk critically low — writes suspended") {
         super(message);
@@ -956,6 +1025,15 @@ class VectorDB {
                     yield this.drainWrites();
                     try {
                         const done = (0, logger_1.timer)("vectordb", "optimize");
+                        // deleteUnverified deletes files Lance cannot prove are unreferenced.
+                        // LanceDB's own docs: only safe "if you can guarantee that no other
+                        // process is currently working on this dataset. Otherwise the dataset
+                        // could be put into a corrupted state." gmax is multi-process, so that
+                        // guarantee comes from routing compaction to a single writer — the
+                        // daemon serializes it (Daemon.runOptimize + the maintenance loop),
+                        // and CLI callers go through it rather than optimizing alongside it.
+                        // Anything new that calls optimize() must preserve that: either be the
+                        // daemon, or be the only process on the store.
                         const stats = yield table.optimize({
                             cleanupOlderThan: cutoff,
                             deleteUnverified: true,
@@ -1084,6 +1162,15 @@ class VectorDB {
                     if (this.lastOptimizeDidWork) {
                         const table = yield this.openExistingTableUnsafe();
                         if (table) {
+                            // Collect abandoned scratch files before measuring. They are pure
+                            // disk with no manifest entry, so they inflate the ratio while being
+                            // invisible to optimize — measuring first would blame the compactor
+                            // for bytes it has no way to reach and send us into a second
+                            // full-table pass that reclaims nothing.
+                            const swept = sweepStaleLanceTempFiles(this.lancedbDir);
+                            if (swept.filesRemoved > 0) {
+                                (0, logger_1.log)("vectordb", `Swept ${swept.filesRemoved} abandoned temp file(s), freed ${(swept.bytesFreed / 1024 / 1024).toFixed(1)}MB`);
+                            }
                             const stats = yield table.stats();
                             const diskSize = this.getDirectorySize(this.lancedbDir);
                             const logicalSize = stats.totalBytes;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "grepmax",
-  "version": "0.26.11",
+  "version": "0.26.13",
   "author": "Robert Owens <78518764+reowens@users.noreply.github.com>",
   "homepage": "https://github.com/reowens/grepmax",
   "bugs": {

package/plugins/grepmax/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "grepmax",
-  "version": "0.26.11",
+  "version": "0.26.13",
   "description": "Semantic code search for Claude Code. Automatically indexes your project and provides intelligent search capabilities.",
   "author": {
     "name": "Robert Owens",