grepmax 0.26.12 → 0.26.14

package/.claude-plugin/marketplace.json

@@ -10,7 +10,7 @@
       "name": "grepmax",
       "source": "./plugins/grepmax",
       "description": "Semantic code search for Claude Code. Automatically indexes your project and provides intelligent search capabilities.",
-      "version": "0.26.12",
+      "version": "0.26.14",
       "author": {
         "name": "Robert Owens",
         "email": "robowens@me.com"

package/dist/commands/doctor.js

@@ -56,6 +56,7 @@ const lock_1 = require("../lib/utils/lock");
 const mlx_hf_cache_1 = require("../lib/utils/mlx-hf-cache");
 const project_registry_1 = require("../lib/utils/project-registry");
 const project_root_1 = require("../lib/utils/project-root");
+const root_availability_1 = require("../lib/utils/root-availability");
 function formatSize(bytes) {
     if (bytes < 1024)
         return `${bytes} B`;
@@ -197,8 +198,9 @@ exports.doctor = new commander_1.Command("doctor")
     let needsOptimize = false;
     let staleLock = false;
     const orphanedProjects = [];
+    const unavailableProjects = [];
     try {
-        const { VectorDB } = yield Promise.resolve().then(() => __importStar(require("../lib/store/vector-db")));
+        const { VectorDB, findStaleLanceTempFiles, sweepStaleLanceTempFiles } = yield Promise.resolve().then(() => __importStar(require("../lib/store/vector-db")));
         const db = new VectorDB(config_1.PATHS.lancedbDir);
         const table = yield db.ensureTable();
         const totalChunks = yield table.countRows();
@@ -250,15 +252,33 @@ exports.doctor = new commander_1.Command("doctor")
         // Daemon status
         const { isDaemonRunning } = yield Promise.resolve().then(() => __importStar(require("../lib/utils/daemon-client")));
         const daemonUp = yield isDaemonRunning();
-        // Project registry health
+        // Project registry health.
+        //
+        // A root can be unreadable for two very different reasons, and only one of
+        // them is the project's fault. An unplugged external drive or a downed
+        // network share leaves the project perfectly intact — deregistering it
+        // would strand its vectors in the shared table with no owning project and
+        // would not come back when the volume returns. Only a root whose volume is
+        // mounted, and whose directory is still gone, is actually orphaned.
         const projects = (0, project_registry_1.listProjects)();
         for (const p of projects) {
-            if (!fs.existsSync(p.root)) {
+            const availability = (0, root_availability_1.classifyRoot)(p.root);
+            if (availability === "missing")
                 orphanedProjects.push(p.root);
-            }
+            else if (availability === "unavailable")
+                unavailableProjects.push(p.root);
         }
         // Compute warning flags
-        const bloatRatio = logicalSize > 0 ? diskSize / logicalSize : 0;
+        //
+        // Abandoned `.tmp*` scratch files are disk that optimize provably cannot
+        // reclaim (no manifest entry, so `deleteUnverified` will not touch them).
+        // They get their own remediation — a sweep — and are excluded from the
+        // ratio that decides whether to run a whole-table optimize, because
+        // counting them there prescribes a slow rewrite for bytes it cannot free.
+        const staleTempFiles = findStaleLanceTempFiles(config_1.PATHS.lancedbDir);
+        const staleTempBytes = staleTempFiles.reduce((n, f) => n + f.size, 0);
+        const reclaimableDiskSize = Math.max(0, diskSize - staleTempBytes);
+        const bloatRatio = logicalSize > 0 ? reclaimableDiskSize / logicalSize : 0;
         if (bloatRatio > 2.0)
             needsOptimize = true;
         // Track the compactor's own threshold. Warning (and worse, --fix forcing a
@@ -304,6 +324,9 @@ exports.doctor = new commander_1.Command("doctor")
                 `lock=${lockStatus.split(" ")[0]}`,
                 `daemon=${daemonUp ? "running" : "stopped"}`,
                 `orphaned=${orphanedProjects.length}`,
+                `offline=${unavailableProjects.length}`,
+                `stale_temp=${staleTempFiles.length}`,
+                `stale_temp_bytes=${staleTempBytes}`,
                 `stale_chunker=${staleChunkerProjects.length}`,
                 `stale_embedding=${staleEmbeddingProjects.length}`,
                 `legacy_embedding=${legacyEmbeddingCount}`,
@@ -380,11 +403,15 @@ exports.doctor = new commander_1.Command("doctor")
                 console.log(`ok  Disk: ${formatSize(availBytes)} available`);
             }
             // Storage
+            const storageLine = `${totalChunks.toLocaleString()} rows, ${formatSize(logicalSize)} logical, ${formatSize(diskSize)} disk`;
             if (bloatRatio > 2.0) {
-                console.log(`WARN  Storage: ${totalChunks.toLocaleString()} rows, ${formatSize(logicalSize)} logical, ${formatSize(diskSize)} disk (${bloatRatio.toFixed(1)}x — orphaned files)`);
+                console.log(`WARN  Storage: ${storageLine} (${bloatRatio.toFixed(1)}x — orphaned files)`);
             }
             else {
-                console.log(`ok  Storage: ${totalChunks.toLocaleString()} rows, ${formatSize(logicalSize)} logical, ${formatSize(diskSize)} disk`);
+                console.log(`ok  Storage: ${storageLine}`);
+            }
+            if (staleTempFiles.length > 0) {
+                console.log(`WARN  Abandoned temp files: ${staleTempFiles.length} (${formatSize(staleTempBytes)}) — leftover from interrupted compaction, run with --fix`);
             }
             // Fragments
             if (numSmallFragments > config_1.FRAGMENT_COMPACT_THRESHOLD) {
@@ -440,13 +467,19 @@ exports.doctor = new commander_1.Command("doctor")
             if (legacyNotice)
                 console.log(legacyNotice);
             // Projects
+            if (unavailableProjects.length > 0) {
+                console.log(`INFO  Offline projects: ${unavailableProjects.length} (volume not mounted — kept, reconnect to resume)`);
+                for (const up of unavailableProjects) {
+                    console.log(`       - ${up}`);
+                }
+            }
             if (orphanedProjects.length > 0) {
                 console.log(`WARN  Orphaned projects: ${orphanedProjects.length} (directories no longer exist)`);
                 for (const op of orphanedProjects) {
                     console.log(`       - ${op}`);
                 }
             }
-            else if (projects.length > 0) {
+            else if (projects.length > 0 && unavailableProjects.length === 0) {
                 console.log(`ok  Projects: ${projects.length} registered, all directories exist`);
             }
             // Cache Coherence
@@ -500,6 +533,16 @@ exports.doctor = new commander_1.Command("doctor")
                     console.log("ok  Removed stale lock");
                 fixed++;
             }
+            // Sweep before optimize: these bytes are unreachable by optimize, and
+            // clearing them first means any bloat still visible afterwards is real.
+            // Unlike optimize this needs no single-writer routing — the age gate in
+            // findStaleLanceTempFiles already excludes anything a writer could hold.
+            if (staleTempFiles.length > 0) {
+                const swept = sweepStaleLanceTempFiles(config_1.PATHS.lancedbDir);
+                if (!opts.agent)
+                    console.log(`ok  Removed ${swept.filesRemoved} abandoned temp file(s), freed ${formatSize(swept.bytesFreed)}`);
+                fixed++;
+            }
             if (needsOptimize) {
                 if (!opts.agent)
                     console.log("...  Running optimize (compact + prune)...");
@@ -538,6 +581,9 @@ exports.doctor = new commander_1.Command("doctor")
                     console.log(`ok  Optimize complete (${via})`);
                 fixed++;
             }
+            // Only roots whose volume is mounted and whose directory is still gone
+            // reach this list — projects on an unplugged drive are classified
+            // `unavailable` and never enter it, so --fix cannot deregister them.
             if (orphanedProjects.length > 0) {
                 for (const op of orphanedProjects) {
                     (0, project_registry_1.removeProject)(op);

package/dist/config.js

@@ -33,7 +33,7 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.INDEXABLE_EXTENSIONS = exports.VERSION_PRUNE_THRESHOLD = exports.FRAGMENT_COMPACT_THRESHOLD = exports.DISK_LOW_BYTES = exports.DISK_CRITICAL_BYTES = exports.MAX_FILE_SIZE_BYTES = exports.PATHS = exports.MAX_WORKER_MEMORY_MB = exports.WORKER_BOOT_TIMEOUT_MS = exports.WORKER_TIMEOUT_MS = exports.REBUILD_COMMAND = exports.CHUNKER_VERSION_HISTORY = exports.CONFIG = exports.MODEL_IDS = exports.DEFAULT_MODEL_TIER = exports.MODEL_TIERS = void 0;
+exports.INDEXABLE_EXTENSIONS = exports.STALE_TEMP_FILE_AGE_MS = exports.VERSION_PRUNE_THRESHOLD = exports.FRAGMENT_COMPACT_THRESHOLD = exports.DISK_LOW_BYTES = exports.DISK_CRITICAL_BYTES = exports.MAX_FILE_SIZE_BYTES = exports.PATHS = exports.MAX_WORKER_MEMORY_MB = exports.WORKER_BOOT_TIMEOUT_MS = exports.WORKER_TIMEOUT_MS = exports.REBUILD_COMMAND = exports.CHUNKER_VERSION_HISTORY = exports.CONFIG = exports.MODEL_IDS = exports.DEFAULT_MODEL_TIER = exports.MODEL_TIERS = void 0;
 exports.describeChunkerGap = describeChunkerGap;
 exports.describeEmbeddingGap = describeEmbeddingGap;
 exports.describeSchemaDimGap = describeSchemaDimGap;
@@ -281,6 +281,19 @@ exports.FRAGMENT_COMPACT_THRESHOLD = 400;
 // fire on every healthy store — and make `doctor --fix` force the whole-table
 // rewrite that the fragment threshold exists to avoid.
 exports.VERSION_PRUNE_THRESHOLD = 2000;
+// Age past which a Lance `.tmp*` scratch file in a `*.lance/data` directory is
+// considered abandoned rather than in flight. Lance writes a new fragment to a
+// dot-prefixed temp file and renames it on success, so a killed optimize leaves
+// the temp behind forever: it is in no manifest, so `optimize({deleteUnverified})`
+// never collects it, and the dot prefix hides it from a plain `ls`. Three such
+// files (7.3 GB, up to 2.5 months old) were what kept this store pinned at a 2.1x
+// disk/logical ratio that repeated optimize passes could not move.
+//
+// The age gate is the whole safety argument: an in-flight write keeps bumping its
+// temp file's mtime, so anything this stale has no writer, and deleting it is safe
+// without knowing how many processes are on the store. Observed optimize runs on a
+// 325k-row store peak at ~1.4 min, so an hour is ~40x the worst case seen.
+exports.STALE_TEMP_FILE_AGE_MS = 60 * 60 * 1000;
 // Extensions we consider for indexing to avoid binary noise and improve relevance.
 exports.INDEXABLE_EXTENSIONS = new Set([
     ".ts",

package/dist/lib/daemon/daemon.js

@@ -67,6 +67,7 @@ const logger_1 = require("../utils/logger");
 const operation_coordinator_1 = require("../utils/operation-coordinator");
 const process_1 = require("../utils/process");
 const project_registry_1 = require("../utils/project-registry");
+const root_availability_1 = require("../utils/root-availability");
 const watcher_store_1 = require("../utils/watcher-store");
 const pool_1 = require("../workers/pool");
 const ipc_handler_1 = require("./ipc-handler");
@@ -352,7 +353,10 @@ class Daemon {
             const indexed = allProjects.filter((p) => p.status === "indexed");
             for (const p of indexed) {
                 if (!fs.existsSync(p.root)) {
-                    console.log(`[daemon] Skipping ${path.basename(p.root)} — directory not found`);
+                    // Skipping is already the safe behavior — the project keeps its registry
+                    // entry and its vectors either way. Naming the reason keeps an unplugged
+                    // drive from reading like data loss in the log.
+                    console.log(`[daemon] Skipping ${path.basename(p.root)} — ${(0, root_availability_1.describeRoot)(p.root)}`);
                     continue;
                 }
                 try {

package/dist/lib/store/vector-db.js

@@ -43,6 +43,8 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.VectorDB = exports.DiskPressureError = void 0;
+exports.findStaleLanceTempFiles = findStaleLanceTempFiles;
+exports.sweepStaleLanceTempFiles = sweepStaleLanceTempFiles;
 exports.isLanceCorruptionError = isLanceCorruptionError;
 exports.isMissingTableError = isMissingTableError;
 const fs = __importStar(require("node:fs"));
@@ -55,6 +57,73 @@ const filter_builder_1 = require("../utils/filter-builder");
 const logger_1 = require("../utils/logger");
 const ann_config_1 = require("./ann-config");
 const store_lease_1 = require("./store-lease");
+/**
+ * Find abandoned Lance scratch files under `<lancedbDir>/ *.lance/data`.
+ *
+ * Lance stages a new fragment in a dot-prefixed `.tmp*` file and renames it into
+ * place on success. A killed or panicking optimize leaves the temp behind, and
+ * nothing ever collects it: it appears in no manifest, so `deleteUnverified`
+ * cannot prove it is garbage, and the leading dot hides it from `ls`. The result
+ * is permanent disk bloat that no amount of optimizing reclaims.
+ *
+ * Only files older than `minAgeMs` are reported. An in-flight write keeps
+ * bumping its temp file's mtime, so the age gate — not a writer count — is what
+ * makes deleting the result safe from any process.
+ */
+function findStaleLanceTempFiles(lancedbDir, minAgeMs = config_1.STALE_TEMP_FILE_AGE_MS) {
+    const found = [];
+    const cutoff = Date.now() - minAgeMs;
+    let tables;
+    try {
+        tables = fs.readdirSync(lancedbDir);
+    }
+    catch (_a) {
+        return found;
+    }
+    for (const tableDir of tables) {
+        if (!tableDir.endsWith(".lance"))
+            continue;
+        const dataDir = `${lancedbDir}/${tableDir}/data`;
+        let entries;
+        try {
+            entries = fs.readdirSync(dataDir);
+        }
+        catch (_b) {
+            continue;
+        }
+        for (const entry of entries) {
+            if (!entry.startsWith(".tmp"))
+                continue;
+            const full = `${dataDir}/${entry}`;
+            try {
+                const s = fs.statSync(full);
+                if (!s.isFile() || s.mtimeMs > cutoff)
+                    continue;
+                found.push({ path: full, size: s.size, mtimeMs: s.mtimeMs });
+            }
+            catch (_c) { }
+        }
+    }
+    return found;
+}
+/**
+ * Delete abandoned Lance scratch files. Returns bytes actually reclaimed —
+ * a file that vanishes between the scan and the unlink is not counted, so the
+ * number reported is never larger than what was freed.
+ */
+function sweepStaleLanceTempFiles(lancedbDir, minAgeMs = config_1.STALE_TEMP_FILE_AGE_MS) {
+    let filesRemoved = 0;
+    let bytesFreed = 0;
+    for (const f of findStaleLanceTempFiles(lancedbDir, minAgeMs)) {
+        try {
+            fs.unlinkSync(f.path);
+            filesRemoved++;
+            bytesFreed += f.size;
+        }
+        catch (_a) { }
+    }
+    return { filesRemoved, bytesFreed };
+}
 class DiskPressureError extends Error {
     constructor(message = "Disk critically low — writes suspended") {
         super(message);
@@ -1093,6 +1162,15 @@ class VectorDB {
                     if (this.lastOptimizeDidWork) {
                         const table = yield this.openExistingTableUnsafe();
                         if (table) {
+                            // Collect abandoned scratch files before measuring. They are pure
+                            // disk with no manifest entry, so they inflate the ratio while being
+                            // invisible to optimize — measuring first would blame the compactor
+                            // for bytes it has no way to reach and send us into a second
+                            // full-table pass that reclaims nothing.
+                            const swept = sweepStaleLanceTempFiles(this.lancedbDir);
+                            if (swept.filesRemoved > 0) {
+                                (0, logger_1.log)("vectordb", `Swept ${swept.filesRemoved} abandoned temp file(s), freed ${(swept.bytesFreed / 1024 / 1024).toFixed(1)}MB`);
+                            }
                             const stats = yield table.stats();
                             const diskSize = this.getDirectorySize(this.lancedbDir);
                             const logicalSize = stats.totalBytes;

package/dist/lib/utils/root-availability.js

@@ -0,0 +1,105 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.classifyRoot = classifyRoot;
+exports.isRootUnavailable = isRootUnavailable;
+exports.describeRoot = describeRoot;
+const fs = __importStar(require("node:fs"));
+/**
+ * Directories that hold mount points rather than user data, with how many path
+ * segments below them the mount point itself sits.
+ *
+ * `mountDepth` errs high where a platform is ambiguous (Linux uses both
+ * `/media/<volume>` and `/media/<user>/<volume>`). Overshooting biases the
+ * classifier toward `unavailable`, which is the safe direction: a wrong
+ * `unavailable` only means we decline to auto-remove a genuinely dead project,
+ * while a wrong `missing` silently deregisters a live one.
+ */
+const MOUNT_CONTAINERS = [
+    { prefix: "/Volumes", mountDepth: 1 }, // macOS
+    { prefix: "/run/media", mountDepth: 2 }, // Linux, udisks2
+    { prefix: "/media", mountDepth: 2 },
+    { prefix: "/mnt", mountDepth: 1 },
+    { prefix: "/net", mountDepth: 1 }, // autofs network mounts
+];
+function exists(p) {
+    try {
+        fs.statSync(p);
+        return true;
+    }
+    catch (_a) {
+        // A disconnected network mount can throw EIO/ETIMEDOUT rather than ENOENT.
+        // Either way the path is not readable, and the caller's container check
+        // decides whether that means unavailable or missing.
+        return false;
+    }
+}
+/**
+ * Classify a project root that may live on removable or network storage.
+ *
+ * Longest matching container wins, so `/run/media` is not shadowed by `/media`.
+ */
+function classifyRoot(root) {
+    if (exists(root))
+        return "present";
+    const container = MOUNT_CONTAINERS.filter((c) => root === c.prefix || root.startsWith(`${c.prefix}/`)).sort((a, b) => b.prefix.length - a.prefix.length)[0];
+    if (!container)
+        return "missing";
+    // The mount point is the container plus `mountDepth` segments. If it is
+    // absent, nothing is mounted there and the project is merely out of reach.
+    const rest = root.slice(container.prefix.length).split("/").filter(Boolean);
+    if (rest.length < container.mountDepth)
+        return "missing";
+    const mountPoint = `${container.prefix}/${rest.slice(0, container.mountDepth).join("/")}`;
+    return exists(mountPoint) ? "missing" : "unavailable";
+}
+/**
+ * True when the root is absent only because its volume is not mounted.
+ * Callers that delete or rewrite persistent state must check this first.
+ */
+function isRootUnavailable(root) {
+    return classifyRoot(root) === "unavailable";
+}
+/** Human-readable reason for a non-present root, for logs and doctor output. */
+function describeRoot(root) {
+    switch (classifyRoot(root)) {
+        case "present":
+            return "available";
+        case "unavailable":
+            return "volume not mounted";
+        case "missing":
+            return "directory not found";
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "grepmax",
-  "version": "0.26.12",
+  "version": "0.26.14",
   "author": "Robert Owens <78518764+reowens@users.noreply.github.com>",
   "homepage": "https://github.com/reowens/grepmax",
   "bugs": {

package/plugins/grepmax/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "grepmax",
-  "version": "0.26.12",
+  "version": "0.26.14",
   "description": "Semantic code search for Claude Code. Automatically indexes your project and provides intelligent search capabilities.",
   "author": {
     "name": "Robert Owens",