grepmax 0.26.12 → 0.26.13

package/.claude-plugin/marketplace.json

@@ -10,7 +10,7 @@
       "name": "grepmax",
       "source": "./plugins/grepmax",
       "description": "Semantic code search for Claude Code. Automatically indexes your project and provides intelligent search capabilities.",
-      "version": "0.26.12",
+      "version": "0.26.13",
       "author": {
         "name": "Robert Owens",
         "email": "robowens@me.com"

package/dist/commands/doctor.js

@@ -198,7 +198,7 @@ exports.doctor = new commander_1.Command("doctor")
     let staleLock = false;
     const orphanedProjects = [];
     try {
-        const { VectorDB } = yield Promise.resolve().then(() => __importStar(require("../lib/store/vector-db")));
+        const { VectorDB, findStaleLanceTempFiles, sweepStaleLanceTempFiles } = yield Promise.resolve().then(() => __importStar(require("../lib/store/vector-db")));
         const db = new VectorDB(config_1.PATHS.lancedbDir);
         const table = yield db.ensureTable();
         const totalChunks = yield table.countRows();
@@ -258,7 +258,16 @@ exports.doctor = new commander_1.Command("doctor")
             }
         }
         // Compute warning flags
-        const bloatRatio = logicalSize > 0 ? diskSize / logicalSize : 0;
+        //
+        // Abandoned `.tmp*` scratch files are disk that optimize provably cannot
+        // reclaim (no manifest entry, so `deleteUnverified` will not touch them).
+        // They get their own remediation — a sweep — and are excluded from the
+        // ratio that decides whether to run a whole-table optimize, because
+        // counting them there prescribes a slow rewrite for bytes it cannot free.
+        const staleTempFiles = findStaleLanceTempFiles(config_1.PATHS.lancedbDir);
+        const staleTempBytes = staleTempFiles.reduce((n, f) => n + f.size, 0);
+        const reclaimableDiskSize = Math.max(0, diskSize - staleTempBytes);
+        const bloatRatio = logicalSize > 0 ? reclaimableDiskSize / logicalSize : 0;
         if (bloatRatio > 2.0)
             needsOptimize = true;
         // Track the compactor's own threshold. Warning (and worse, --fix forcing a
@@ -304,6 +313,8 @@ exports.doctor = new commander_1.Command("doctor")
                 `lock=${lockStatus.split(" ")[0]}`,
                 `daemon=${daemonUp ? "running" : "stopped"}`,
                 `orphaned=${orphanedProjects.length}`,
+                `stale_temp=${staleTempFiles.length}`,
+                `stale_temp_bytes=${staleTempBytes}`,
                 `stale_chunker=${staleChunkerProjects.length}`,
                 `stale_embedding=${staleEmbeddingProjects.length}`,
                 `legacy_embedding=${legacyEmbeddingCount}`,
@@ -380,11 +391,15 @@ exports.doctor = new commander_1.Command("doctor")
                 console.log(`ok  Disk: ${formatSize(availBytes)} available`);
             }
             // Storage
+            const storageLine = `${totalChunks.toLocaleString()} rows, ${formatSize(logicalSize)} logical, ${formatSize(diskSize)} disk`;
             if (bloatRatio > 2.0) {
-                console.log(`WARN  Storage: ${totalChunks.toLocaleString()} rows, ${formatSize(logicalSize)} logical, ${formatSize(diskSize)} disk (${bloatRatio.toFixed(1)}x — orphaned files)`);
+                console.log(`WARN  Storage: ${storageLine} (${bloatRatio.toFixed(1)}x — orphaned files)`);
             }
             else {
-                console.log(`ok  Storage: ${totalChunks.toLocaleString()} rows, ${formatSize(logicalSize)} logical, ${formatSize(diskSize)} disk`);
+                console.log(`ok  Storage: ${storageLine}`);
+            }
+            if (staleTempFiles.length > 0) {
+                console.log(`WARN  Abandoned temp files: ${staleTempFiles.length} (${formatSize(staleTempBytes)}) — leftover from interrupted compaction, run with --fix`);
             }
             // Fragments
             if (numSmallFragments > config_1.FRAGMENT_COMPACT_THRESHOLD) {
@@ -500,6 +515,16 @@ exports.doctor = new commander_1.Command("doctor")
                     console.log("ok  Removed stale lock");
                 fixed++;
             }
+            // Sweep before optimize: these bytes are unreachable by optimize, and
+            // clearing them first means any bloat still visible afterwards is real.
+            // Unlike optimize this needs no single-writer routing — the age gate in
+            // findStaleLanceTempFiles already excludes anything a writer could hold.
+            if (staleTempFiles.length > 0) {
+                const swept = sweepStaleLanceTempFiles(config_1.PATHS.lancedbDir);
+                if (!opts.agent)
+                    console.log(`ok  Removed ${swept.filesRemoved} abandoned temp file(s), freed ${formatSize(swept.bytesFreed)}`);
+                fixed++;
+            }
             if (needsOptimize) {
                 if (!opts.agent)
                     console.log("...  Running optimize (compact + prune)...");

package/dist/config.js

@@ -33,7 +33,7 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.INDEXABLE_EXTENSIONS = exports.VERSION_PRUNE_THRESHOLD = exports.FRAGMENT_COMPACT_THRESHOLD = exports.DISK_LOW_BYTES = exports.DISK_CRITICAL_BYTES = exports.MAX_FILE_SIZE_BYTES = exports.PATHS = exports.MAX_WORKER_MEMORY_MB = exports.WORKER_BOOT_TIMEOUT_MS = exports.WORKER_TIMEOUT_MS = exports.REBUILD_COMMAND = exports.CHUNKER_VERSION_HISTORY = exports.CONFIG = exports.MODEL_IDS = exports.DEFAULT_MODEL_TIER = exports.MODEL_TIERS = void 0;
+exports.INDEXABLE_EXTENSIONS = exports.STALE_TEMP_FILE_AGE_MS = exports.VERSION_PRUNE_THRESHOLD = exports.FRAGMENT_COMPACT_THRESHOLD = exports.DISK_LOW_BYTES = exports.DISK_CRITICAL_BYTES = exports.MAX_FILE_SIZE_BYTES = exports.PATHS = exports.MAX_WORKER_MEMORY_MB = exports.WORKER_BOOT_TIMEOUT_MS = exports.WORKER_TIMEOUT_MS = exports.REBUILD_COMMAND = exports.CHUNKER_VERSION_HISTORY = exports.CONFIG = exports.MODEL_IDS = exports.DEFAULT_MODEL_TIER = exports.MODEL_TIERS = void 0;
 exports.describeChunkerGap = describeChunkerGap;
 exports.describeEmbeddingGap = describeEmbeddingGap;
 exports.describeSchemaDimGap = describeSchemaDimGap;
@@ -281,6 +281,19 @@ exports.FRAGMENT_COMPACT_THRESHOLD = 400;
 // fire on every healthy store — and make `doctor --fix` force the whole-table
 // rewrite that the fragment threshold exists to avoid.
 exports.VERSION_PRUNE_THRESHOLD = 2000;
+// Age past which a Lance `.tmp*` scratch file in a `*.lance/data` directory is
+// considered abandoned rather than in flight. Lance writes a new fragment to a
+// dot-prefixed temp file and renames it on success, so a killed optimize leaves
+// the temp behind forever: it is in no manifest, so `optimize({deleteUnverified})`
+// never collects it, and the dot prefix hides it from a plain `ls`. Three such
+// files (7.3 GB, up to 2.5 months old) were what kept this store pinned at a 2.1x
+// disk/logical ratio that repeated optimize passes could not move.
+//
+// The age gate is the whole safety argument: an in-flight write keeps bumping its
+// temp file's mtime, so anything this stale has no writer, and deleting it is safe
+// without knowing how many processes are on the store. Observed optimize runs on a
+// 325k-row store peak at ~1.4 min, so an hour is ~40x the worst case seen.
+exports.STALE_TEMP_FILE_AGE_MS = 60 * 60 * 1000;
 // Extensions we consider for indexing to avoid binary noise and improve relevance.
 exports.INDEXABLE_EXTENSIONS = new Set([
     ".ts",

package/dist/lib/store/vector-db.js

@@ -43,6 +43,8 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.VectorDB = exports.DiskPressureError = void 0;
+exports.findStaleLanceTempFiles = findStaleLanceTempFiles;
+exports.sweepStaleLanceTempFiles = sweepStaleLanceTempFiles;
 exports.isLanceCorruptionError = isLanceCorruptionError;
 exports.isMissingTableError = isMissingTableError;
 const fs = __importStar(require("node:fs"));
@@ -55,6 +57,73 @@ const filter_builder_1 = require("../utils/filter-builder");
 const logger_1 = require("../utils/logger");
 const ann_config_1 = require("./ann-config");
 const store_lease_1 = require("./store-lease");
+/**
+ * Find abandoned Lance scratch files under `<lancedbDir>/ *.lance/data`.
+ *
+ * Lance stages a new fragment in a dot-prefixed `.tmp*` file and renames it into
+ * place on success. A killed or panicking optimize leaves the temp behind, and
+ * nothing ever collects it: it appears in no manifest, so `deleteUnverified`
+ * cannot prove it is garbage, and the leading dot hides it from `ls`. The result
+ * is permanent disk bloat that no amount of optimizing reclaims.
+ *
+ * Only files older than `minAgeMs` are reported. An in-flight write keeps
+ * bumping its temp file's mtime, so the age gate — not a writer count — is what
+ * makes deleting the result safe from any process.
+ */
+function findStaleLanceTempFiles(lancedbDir, minAgeMs = config_1.STALE_TEMP_FILE_AGE_MS) {
+    const found = [];
+    const cutoff = Date.now() - minAgeMs;
+    let tables;
+    try {
+        tables = fs.readdirSync(lancedbDir);
+    }
+    catch (_a) {
+        return found;
+    }
+    for (const tableDir of tables) {
+        if (!tableDir.endsWith(".lance"))
+            continue;
+        const dataDir = `${lancedbDir}/${tableDir}/data`;
+        let entries;
+        try {
+            entries = fs.readdirSync(dataDir);
+        }
+        catch (_b) {
+            continue;
+        }
+        for (const entry of entries) {
+            if (!entry.startsWith(".tmp"))
+                continue;
+            const full = `${dataDir}/${entry}`;
+            try {
+                const s = fs.statSync(full);
+                if (!s.isFile() || s.mtimeMs > cutoff)
+                    continue;
+                found.push({ path: full, size: s.size, mtimeMs: s.mtimeMs });
+            }
+            catch (_c) { }
+        }
+    }
+    return found;
+}
+/**
+ * Delete abandoned Lance scratch files. Returns bytes actually reclaimed —
+ * a file that vanishes between the scan and the unlink is not counted, so the
+ * number reported is never larger than what was freed.
+ */
+function sweepStaleLanceTempFiles(lancedbDir, minAgeMs = config_1.STALE_TEMP_FILE_AGE_MS) {
+    let filesRemoved = 0;
+    let bytesFreed = 0;
+    for (const f of findStaleLanceTempFiles(lancedbDir, minAgeMs)) {
+        try {
+            fs.unlinkSync(f.path);
+            filesRemoved++;
+            bytesFreed += f.size;
+        }
+        catch (_a) { }
+    }
+    return { filesRemoved, bytesFreed };
+}
 class DiskPressureError extends Error {
     constructor(message = "Disk critically low — writes suspended") {
         super(message);
@@ -1093,6 +1162,15 @@ class VectorDB {
                     if (this.lastOptimizeDidWork) {
                         const table = yield this.openExistingTableUnsafe();
                         if (table) {
+                            // Collect abandoned scratch files before measuring. They are pure
+                            // disk with no manifest entry, so they inflate the ratio while being
+                            // invisible to optimize — measuring first would blame the compactor
+                            // for bytes it has no way to reach and send us into a second
+                            // full-table pass that reclaims nothing.
+                            const swept = sweepStaleLanceTempFiles(this.lancedbDir);
+                            if (swept.filesRemoved > 0) {
+                                (0, logger_1.log)("vectordb", `Swept ${swept.filesRemoved} abandoned temp file(s), freed ${(swept.bytesFreed / 1024 / 1024).toFixed(1)}MB`);
+                            }
                             const stats = yield table.stats();
                             const diskSize = this.getDirectorySize(this.lancedbDir);
                             const logicalSize = stats.totalBytes;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "grepmax",
-  "version": "0.26.12",
+  "version": "0.26.13",
   "author": "Robert Owens <78518764+reowens@users.noreply.github.com>",
   "homepage": "https://github.com/reowens/grepmax",
   "bugs": {

package/plugins/grepmax/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "grepmax",
-  "version": "0.26.12",
+  "version": "0.26.13",
   "description": "Semantic code search for Claude Code. Automatically indexes your project and provides intelligent search capabilities.",
   "author": {
     "name": "Robert Owens",