grepmax 0.14.9 → 0.15.0

package/dist/commands/doctor.js

@@ -228,12 +228,23 @@ exports.doctor = new commander_1.Command("doctor")
             needsOptimize = true;
         if (versions.length > 50)
             needsOptimize = true;
+        // Disk space check
+        let availBytes = 0;
+        let diskLevel = "ok";
+        try {
+            const diskStats = fs.statfsSync(config_1.PATHS.lancedbDir);
+            availBytes = diskStats.bavail * diskStats.bsize;
+            diskLevel = availBytes < config_1.DISK_CRITICAL_BYTES ? "CRITICAL" : availBytes < config_1.DISK_LOW_BYTES ? "LOW" : "ok";
+        }
+        catch (_e) { }
         if (opts.agent) {
             const fields = [
                 "index_health",
                 `rows=${totalChunks}`,
                 `logical=${formatSize(logicalSize)}`,
                 `disk=${formatSize(diskSize)}`,
+                `free=${formatSize(availBytes)}`,
+                `disk_pressure=${diskLevel}`,
                 `fragments=${numFragments}`,
                 `small=${numSmallFragments}`,
                 `versions=${versions.length}`,
@@ -245,6 +256,13 @@ exports.doctor = new commander_1.Command("doctor")
         }
         else {
             console.log("\nIndex Health\n");
+            // Disk space
+            if (diskLevel !== "ok") {
+                console.log(`WARN  Disk: ${formatSize(availBytes)} available (${diskLevel})`);
+            }
+            else {
+                console.log(`ok  Disk: ${formatSize(availBytes)} available`);
+            }
             // Storage
             if (bloatRatio > 2.0) {
                 console.log(`WARN  Storage: ${totalChunks.toLocaleString()} rows, ${formatSize(logicalSize)} logical, ${formatSize(diskSize)} disk (${bloatRatio.toFixed(1)}x — orphaned files)`);
@@ -306,7 +324,7 @@ exports.doctor = new commander_1.Command("doctor")
                     }
                     yield mc.close();
                 }
-                catch (_e) { }
+                catch (_f) { }
             }
         }
         // --fix auto-remediation
@@ -343,7 +361,7 @@ exports.doctor = new commander_1.Command("doctor")
         }
         yield db.close();
     }
-    catch (_f) {
+    catch (_g) {
         if (opts.agent) {
             console.log("index_health\terror=could_not_check");
         }

package/dist/config.js

@@ -33,7 +33,7 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.INDEXABLE_EXTENSIONS = exports.MAX_FILE_SIZE_BYTES = exports.PATHS = exports.MAX_WORKER_MEMORY_MB = exports.WORKER_BOOT_TIMEOUT_MS = exports.WORKER_TIMEOUT_MS = exports.CONFIG = exports.MODEL_IDS = exports.DEFAULT_MODEL_TIER = exports.MODEL_TIERS = void 0;
+exports.INDEXABLE_EXTENSIONS = exports.FRAGMENT_COMPACT_THRESHOLD = exports.DISK_LOW_BYTES = exports.DISK_CRITICAL_BYTES = exports.MAX_FILE_SIZE_BYTES = exports.PATHS = exports.MAX_WORKER_MEMORY_MB = exports.WORKER_BOOT_TIMEOUT_MS = exports.WORKER_TIMEOUT_MS = exports.CONFIG = exports.MODEL_IDS = exports.DEFAULT_MODEL_TIER = exports.MODEL_TIERS = void 0;
 const os = __importStar(require("node:os"));
 const path = __importStar(require("node:path"));
 exports.MODEL_TIERS = {
@@ -108,6 +108,19 @@ exports.PATHS = {
     llmLogFile: path.join(GLOBAL_ROOT, "logs", "llm-server.log"),
 };
 exports.MAX_FILE_SIZE_BYTES = 1024 * 1024 * 2; // 2MB limit for indexing
+// Disk pressure thresholds — writes are suspended below critical, compaction limited below low
+exports.DISK_CRITICAL_BYTES = (() => {
+    var _a;
+    const gb = Number.parseFloat((_a = process.env.GMAX_DISK_CRITICAL_GB) !== null && _a !== void 0 ? _a : "5");
+    return (Number.isFinite(gb) && gb > 0 ? gb : 5) * 1024 * 1024 * 1024;
+})();
+exports.DISK_LOW_BYTES = (() => {
+    var _a;
+    const gb = Number.parseFloat((_a = process.env.GMAX_DISK_LOW_GB) !== null && _a !== void 0 ? _a : "20");
+    return (Number.isFinite(gb) && gb > 0 ? gb : 20) * 1024 * 1024 * 1024;
+})();
+// Trigger compaction when small (uncompacted) fragment count exceeds this
+exports.FRAGMENT_COMPACT_THRESHOLD = 50;
 // Extensions we consider for indexing to avoid binary noise and improve relevance.
 exports.INDEXABLE_EXTENSIONS = new Set([
     ".ts",

package/dist/lib/daemon/daemon.js

@@ -94,6 +94,8 @@ class Daemon {
         this.pendingOps = new Set();
         this.watcherFailCount = new Map();
         this.pollIntervals = new Map();
+        this.lastOverflowMs = new Map();
+        this.lastCatchupEndMs = new Map();
         this.projectLocks = new Map();
         this.llmServer = null;
         this.mlxChild = null;
@@ -256,8 +258,8 @@ class Daemon {
                     console.error(`[daemon] Failed to watch ${path.basename(p.root)}:`, err);
                 }
             }
-            // 8b. Index pending projects in the background
-            const pending = allProjects.filter((p) => p.status === "pending" && fs.existsSync(p.root));
+            // 8b. Index pending/error projects in the background
+            const pending = allProjects.filter((p) => (p.status === "pending" || p.status === "error") && fs.existsSync(p.root));
             for (const p of pending) {
                 this.indexPendingProject(p.root).catch((err) => {
                     console.error(`[daemon] Failed to index pending ${path.basename(p.root)}:`, err);
@@ -271,6 +273,7 @@ class Daemon {
                     fs.utimesSync(config_1.PATHS.daemonLockFile, now, now);
                 }
                 catch (_a) { }
+                (0, log_rotate_1.rotateLogFds)(path.join(config_1.PATHS.logsDir, "daemon.log"));
             }, HEARTBEAT_INTERVAL_MS);
             // 10. Idle timeout
             this.idleInterval = setInterval(() => {
@@ -360,13 +363,17 @@ class Daemon {
                 this.subscriptions.delete(root);
             }
             const sub = yield watcher.subscribe(root, (err, events) => {
+                var _a;
                 if (err) {
                     console.error(`[daemon:${name}] Watcher error:`, err);
                     this.recoverWatcher(root, processor);
                     return;
                 }
-                // Watcher is healthy — reset fail counter
-                this.watcherFailCount.delete(root);
+                // Only reset fail counter after sustained health (5min since last overflow)
+                const lastOverflow = (_a = this.lastOverflowMs.get(root)) !== null && _a !== void 0 ? _a : 0;
+                if (Date.now() - lastOverflow > 5 * 60 * 1000) {
+                    this.watcherFailCount.delete(root);
+                }
                 for (const event of events) {
                     processor.handleFileEvent(event.type === "delete" ? "unlink" : "change", event.path);
                 }
@@ -387,6 +394,7 @@ class Daemon {
         this.pendingOps.add(recoveryKey);
         const fails = ((_a = this.watcherFailCount.get(root)) !== null && _a !== void 0 ? _a : 0) + 1;
         this.watcherFailCount.set(root, fails);
+        this.lastOverflowMs.set(root, Date.now());
         const MAX_WATCHER_RETRIES = 3;
         const POLL_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes
         if (fails > MAX_WATCHER_RETRIES) {
@@ -432,9 +440,17 @@ class Daemon {
                 return;
             }
             (() => __awaiter(this, void 0, void 0, function* () {
+                var _a;
                 try {
                     yield this.subscribeWatcher(root, processor);
-                    yield this.catchupScan(root, processor);
+                    const lastCatchup = (_a = this.lastCatchupEndMs.get(root)) !== null && _a !== void 0 ? _a : 0;
+                    const CATCHUP_COOLDOWN_MS = 60000;
+                    if (Date.now() - lastCatchup < CATCHUP_COOLDOWN_MS) {
+                        console.log(`[daemon:${name}] Skipping catchup scan (last completed ${Math.round((Date.now() - lastCatchup) / 1000)}s ago)`);
+                    }
+                    else {
+                        yield this.catchupScan(root, processor);
+                    }
                     console.log(`[daemon:${name}] Watcher recovered`);
                 }
                 catch (err) {
@@ -498,6 +514,12 @@ class Daemon {
                             }
                             processor.handleFileEvent("change", absPath);
                             queued++;
+                            // Throttle: pause periodically during large catchup scans to let the
+                            // batch processor drain and compaction run between bursts.
+                            if (queued % 500 === 0) {
+                                (0, logger_1.debug)("catchup", `${path.basename(root)}: throttle pause at ${queued} queued`);
+                                yield new Promise(r => setTimeout(r, 5000));
+                            }
                         }
                         else {
                             skipped++;
@@ -530,6 +552,7 @@ class Daemon {
                     parts.push(`${purged} deleted`);
                 console.log(`[daemon:${path.basename(root)}] Catchup: ${parts.join(", ")} file(s) while offline`);
             }
+            this.lastCatchupEndMs.set(root, Date.now());
         });
     }
     indexPendingProject(root) {
@@ -559,6 +582,10 @@ class Daemon {
                 catch (err) {
                     const msg = err instanceof Error ? err.message : String(err);
                     console.error(`[daemon] indexPendingProject failed for ${name} after ${Date.now() - start}ms: ${msg}`);
+                    const proj = (0, project_registry_1.getProject)(root);
+                    if (proj) {
+                        (0, project_registry_1.registerProject)(Object.assign(Object.assign({}, proj), { status: "error" }));
+                    }
                 }
                 finally {
                     (_a = this.vectorDb) === null || _a === void 0 ? void 0 : _a.resumeMaintenanceLoop();
@@ -578,6 +605,8 @@ class Daemon {
                 this.subscriptions.delete(root);
             }
             this.processors.delete(root);
+            this.lastOverflowMs.delete(root);
+            this.lastCatchupEndMs.delete(root);
             (0, watcher_store_1.unregisterWatcherByRoot)(root);
             console.log(`[daemon] Unwatched ${root}`);
         });
@@ -591,6 +620,10 @@ class Daemon {
     uptime() {
         return Math.floor((Date.now() - this.startTime) / 1000);
     }
+    getDiskPressure() {
+        var _a, _b;
+        return (_b = (_a = this.vectorDb) === null || _a === void 0 ? void 0 : _a.diskPressure) !== null && _b !== void 0 ? _b : "unknown";
+    }
     /** Reset idle timer — call during long-running operations. */
     resetActivity() {
         this.lastActivity = Date.now();
@@ -865,12 +898,31 @@ class Daemon {
             });
         });
     }
+    getPortPid(port) {
+        try {
+            const out = (0, node_child_process_1.execSync)(`lsof -ti :${port}`, { timeout: 5000 }).toString().trim();
+            const pid = parseInt(out.split("\n")[0], 10);
+            return Number.isFinite(pid) ? pid : null;
+        }
+        catch (_a) {
+            return null;
+        }
+    }
     ensureMlxServer(mlxModel) {
         return __awaiter(this, void 0, void 0, function* () {
             if (yield this.isMlxServerUp()) {
                 console.log("[daemon] MLX embed server already running");
                 return;
             }
+            // Kill stale process holding the port (orphaned from a previous daemon)
+            const port = parseInt(process.env.MLX_EMBED_PORT || "8100", 10);
+            const stalePid = this.getPortPid(port);
+            if (stalePid) {
+                console.log(`[daemon] Killing stale MLX process on port ${port} (PID: ${stalePid})`);
+                yield (0, process_1.killProcess)(stalePid);
+                // Brief pause for OS to release the port
+                yield new Promise((r) => setTimeout(r, 500));
+            }
             // Find mlx-embed-server/server.py relative to the grepmax package
             const candidates = [
                 path.resolve(__dirname, "../../../mlx-embed-server"),

package/dist/lib/daemon/ipc-handler.js

@@ -63,6 +63,7 @@ function handleCommand(daemon, cmd, conn) {
                         pid: process.pid,
                         uptime: daemon.uptime(),
                         projects: daemon.listProjects(),
+                        diskPressure: daemon.getDiskPressure(),
                     };
                 case "shutdown":
                     // Respond before shutting down so the client gets the response

package/dist/lib/index/batch-processor.js

@@ -49,6 +49,7 @@ const config_1 = require("../../config");
 const cache_check_1 = require("../utils/cache-check");
 const file_utils_1 = require("../utils/file-utils");
 const logger_1 = require("../utils/logger");
+const vector_db_1 = require("../store/vector-db");
 const pool_1 = require("../workers/pool");
 const watcher_batch_1 = require("./watcher-batch");
 // Fast path-segment check to reject events that leak through FSEvents overflow.
@@ -113,6 +114,14 @@ class ProjectBatchProcessor {
             var _a;
             if (this.closed || this.processing || this.pending.size === 0)
                 return;
+            // Circuit breaker: don't attempt writes when disk is critically low
+            if (this.vectorDb.diskPressure === "critical") {
+                (0, logger_1.log)(this.wtag, "Disk critically low — deferring batch processing");
+                if (this.debounceTimer)
+                    clearTimeout(this.debounceTimer);
+                this.debounceTimer = setTimeout(() => this.processBatch(), 60000);
+                return;
+            }
             this.processing = true;
             const batchAc = new AbortController();
             this.currentBatchAc = batchAc;
@@ -136,6 +145,7 @@ class ProjectBatchProcessor {
             const start = Date.now();
             let reindexed = 0;
             let processed = 0;
+            let backoffOverrideMs = 0;
             try {
                 // No lock needed — daemon is the single writer to LanceDB/MetaCache
                 const pool = (0, pool_1.getWorkerPool)();
@@ -167,6 +177,16 @@ class ProjectBatchProcessor {
                         if ((0, cache_check_1.isFileCached)(cached, stats)) {
                             continue;
                         }
+                        // Fast path: if only mtime changed but size matches and we have a hash,
+                        // verify in-process instead of dispatching to a worker (~220ms saved).
+                        if (cached && cached.hash && cached.size === stats.size) {
+                            const buf = yield fs.promises.readFile(absPath);
+                            const hash = (0, file_utils_1.computeBufferHash)(buf);
+                            if (hash === cached.hash) {
+                                metaUpdates.set(absPath, Object.assign(Object.assign({}, cached), { mtimeMs: stats.mtimeMs }));
+                                continue;
+                            }
+                        }
                         const result = yield pool.processFile({
                             path: absPath,
                             absolutePath: absPath,
@@ -246,22 +266,41 @@ class ProjectBatchProcessor {
                 for (const absPath of batch.keys()) {
                     this.retryCount.delete(absPath);
                 }
+                // Trigger compaction if fragments are accumulating
+                if (reindexed > 0) {
+                    try {
+                        yield this.vectorDb.compactIfNeeded();
+                    }
+                    catch (e) {
+                        (0, logger_1.log)(this.wtag, `Post-batch compaction failed: ${e}`);
+                    }
+                }
             }
             catch (err) {
-                console.error(`[${this.wtag}] Batch processing failed:`, err);
-                const { requeued, dropped, backoffMs } = (0, watcher_batch_1.computeRetryAction)(batch, this.retryCount, MAX_RETRIES, false, 0, DEBOUNCE_MS);
-                for (const [absPath, event] of requeued) {
-                    if (!this.pending.has(absPath)) {
-                        this.pending.set(absPath, event);
+                // Disk pressure: requeue without counting as retries (not the file's fault)
+                if (err instanceof vector_db_1.DiskPressureError) {
+                    for (const [absPath, event] of batch) {
+                        if (!this.pending.has(absPath)) {
+                            this.pending.set(absPath, event);
+                        }
                     }
+                    (0, logger_1.log)(this.wtag, "Disk pressure — requeued batch, will retry in 60s");
+                    // Use batchTimeoutMs slot to signal finally not to reschedule at 2s
+                    backoffOverrideMs = 60000;
                 }
-                if (dropped > 0) {
-                    console.warn(`[${this.wtag}] Dropped ${dropped} file(s) after ${MAX_RETRIES} failed retries`);
-                }
-                if (this.pending.size > 0) {
-                    if (this.debounceTimer)
-                        clearTimeout(this.debounceTimer);
-                    this.debounceTimer = setTimeout(() => this.processBatch(), backoffMs);
+                else {
+                    console.error(`[${this.wtag}] Batch processing failed:`, err);
+                    const { requeued, dropped, backoffMs } = (0, watcher_batch_1.computeRetryAction)(batch, this.retryCount, MAX_RETRIES, false, 0, DEBOUNCE_MS);
+                    for (const [absPath, event] of requeued) {
+                        if (!this.pending.has(absPath)) {
+                            this.pending.set(absPath, event);
+                        }
+                    }
+                    if (dropped > 0) {
+                        const droppedPaths = [...batch.keys()].filter(p => !requeued.has(p));
+                        (0, logger_1.log)(this.wtag, `Dropped ${dropped} file(s) after ${MAX_RETRIES} retries: ${droppedPaths.map(p => path.basename(p)).join(", ")}`);
+                    }
+                    backoffOverrideMs = this.pending.size > 0 ? backoffMs : 0;
                 }
             }
             finally {
@@ -269,7 +308,14 @@ class ProjectBatchProcessor {
                 this.currentBatchAc = null;
                 this.processing = false;
                 if (this.pending.size > 0) {
-                    this.scheduleBatch();
+                    if (backoffOverrideMs > 0) {
+                        if (this.debounceTimer)
+                            clearTimeout(this.debounceTimer);
+                        this.debounceTimer = setTimeout(() => this.processBatch(), backoffOverrideMs);
+                    }
+                    else {
+                        this.scheduleBatch();
+                    }
                 }
             }
         });

package/dist/lib/index/syncer.js

@@ -217,7 +217,9 @@ function initialSync(options) {
             const batch = [];
             const pendingMeta = new Map();
             const pendingDeletes = new Set();
-            const batchLimit = Math.max(1, config_1.CONFIG.EMBED_BATCH_SIZE);
+            // Use a large flush batch to reduce LanceDB fragment count during sync.
+            // 24 vectors/flush creates ~834 fragments for 10K chunks; 2000 creates ~5.
+            const batchLimit = 2000;
             const maxConcurrency = Math.max(1, config_1.CONFIG.WORKER_THREADS);
             const activeTasks = [];
             let processed = 0;
@@ -230,6 +232,7 @@ function initialSync(options) {
             let flushError;
             let flushPromise = null;
             let flushLock = Promise.resolve();
+            let flushCount = 0;
             const markProgress = (filePath) => {
                 onProgress === null || onProgress === void 0 ? void 0 : onProgress({ processed, indexed, total, filePath });
             };
@@ -253,6 +256,11 @@ function initialSync(options) {
                     try {
                         yield currentFlush;
                         (0, logger_1.debug)("index", `flush done: ${Date.now() - flushStart}ms`);
+                        flushCount++;
+                        // Periodically compact during sync to prevent fragment accumulation
+                        if (flushCount % 10 === 0) {
+                            yield vectorDb.compactIfNeeded(30);
+                        }
                     }
                     catch (err) {
                         (0, logger_1.debug)("index", `flush error: ${err}`);

package/dist/lib/llm/server.js

@@ -256,6 +256,12 @@ class LlmServer {
         const timeoutMs = this.config.idleTimeoutMin * 60 * 1000;
         const checkInterval = Math.min(DEFAULT_IDLE_CHECK_INTERVAL_MS, timeoutMs);
         this.idleTimer = setInterval(() => __awaiter(this, void 0, void 0, function* () {
+            const pid = this.readPid();
+            if (pid && !this.isAlive(pid)) {
+                console.error("[llm] Server crashed (stale PID) — cleaning up");
+                this.cleanupPidFile();
+                return;
+            }
             if (this.lastRequestTime === 0)
                 return;
             if (Date.now() - this.lastRequestTime > timeoutMs) {

package/dist/lib/store/vector-db.js

@@ -42,7 +42,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.VectorDB = void 0;
+exports.VectorDB = exports.DiskPressureError = void 0;
 const fs = __importStar(require("node:fs"));
 const lancedb = __importStar(require("@lancedb/lancedb"));
 const apache_arrow_1 = require("apache-arrow");
@@ -50,6 +50,13 @@ const config_1 = require("../../config");
 const filter_builder_1 = require("../utils/filter-builder");
 const logger_1 = require("../utils/logger");
 const cleanup_1 = require("../utils/cleanup");
+class DiskPressureError extends Error {
+    constructor(message = "Disk critically low — writes suspended") {
+        super(message);
+        this.name = "DiskPressureError";
+    }
+}
+exports.DiskPressureError = DiskPressureError;
 const TABLE_NAME = "chunks";
 const MAINTENANCE_INTERVAL_MS = 5 * 60 * 1000;
 class VectorDB {
@@ -59,6 +66,14 @@ class VectorDB {
         this.closed = false;
         this.maintenanceRunning = false;
         this.maintenanceTimer = null;
+        this.diskPressure = "ok";
+        this.lastDiskCheckMs = 0;
+        this.lastLoggedPressure = "ok";
+        // Write gate: async read-write lock where writes are "readers" (shared)
+        // and compaction is the "writer" (exclusive).
+        this.activeWrites = 0;
+        this.writeDrainResolve = null;
+        this.compactingPromise = null;
         this.vectorDim = vectorDim !== null && vectorDim !== void 0 ? vectorDim : config_1.CONFIG.VECTOR_DIM;
         this.unregisterCleanup = (0, cleanup_1.registerCleanup)(() => this.close());
     }
@@ -104,6 +119,85 @@ class VectorDB {
             return this.db;
         });
     }
+    getAvailableBytes() {
+        try {
+            const stats = fs.statfsSync(this.lancedbDir);
+            return stats.bavail * stats.bsize;
+        }
+        catch (_a) {
+            return Number.MAX_SAFE_INTEGER; // fail-open
+        }
+    }
+    checkDiskPressure() {
+        const now = Date.now();
+        if (now - this.lastDiskCheckMs < VectorDB.DISK_CHECK_INTERVAL_MS) {
+            return this.diskPressure;
+        }
+        this.lastDiskCheckMs = now;
+        const avail = this.getAvailableBytes();
+        let level;
+        if (avail < config_1.DISK_CRITICAL_BYTES) {
+            level = "critical";
+        }
+        else if (avail < config_1.DISK_LOW_BYTES) {
+            level = "low";
+        }
+        else {
+            level = "ok";
+        }
+        if (level !== this.lastLoggedPressure) {
+            const freeStr = `${(avail / 1024 / 1024 / 1024).toFixed(1)}GB`;
+            if (level === "critical") {
+                (0, logger_1.log)("vectordb", `CRITICAL: disk space critically low (${freeStr} free) — writes suspended`);
+            }
+            else if (level === "low") {
+                (0, logger_1.log)("vectordb", `WARNING: disk space low (${freeStr} free) — compaction limited`);
+            }
+            else if (this.lastLoggedPressure !== "ok") {
+                (0, logger_1.log)("vectordb", `Disk pressure resolved (${freeStr} free) — writes resuming`);
+            }
+            this.lastLoggedPressure = level;
+        }
+        this.diskPressure = level;
+        return level;
+    }
+    ensureDiskOk() {
+        if (this.checkDiskPressure() === "critical") {
+            throw new DiskPressureError();
+        }
+    }
+    /**
+     * Wrap a write operation so it coordinates with compaction.
+     * Multiple writes can proceed concurrently (shared access),
+     * but all writes pause when compaction wants exclusive access.
+     */
+    withWriteGate(fn) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.compactingPromise) {
+                yield this.compactingPromise;
+            }
+            this.activeWrites++;
+            try {
+                return yield fn();
+            }
+            finally {
+                this.activeWrites--;
+                if (this.activeWrites === 0 && this.writeDrainResolve) {
+                    this.writeDrainResolve();
+                    this.writeDrainResolve = null;
+                }
+            }
+        });
+    }
+    /** Wait for all in-flight writes to complete before compaction. */
+    drainWrites() {
+        if (this.activeWrites === 0)
+            return Promise.resolve();
+        (0, logger_1.debug)("vectordb", `Draining ${this.activeWrites} in-flight write(s) before compaction`);
+        return new Promise((resolve) => {
+            this.writeDrainResolve = resolve;
+        });
+    }
     seedRow() {
         return {
             id: "seed",
@@ -201,6 +295,7 @@ class VectorDB {
             var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m, _o, _p;
             if (!records.length)
                 return;
+            this.ensureDiskOk();
             const table = yield this.ensureTable();
             const toBuffer = (val) => {
                 var _a, _b;
@@ -278,7 +373,7 @@ class VectorDB {
                 rec.summary = (_p = rec.summary) !== null && _p !== void 0 ? _p : null;
             }
             try {
-                yield table.add(records);
+                yield this.withWriteGate(() => table.add(records));
             }
             catch (err) {
                 const msg = err instanceof Error ? err.message : String(err);
@@ -328,46 +423,64 @@ class VectorDB {
     }
     optimize() {
         return __awaiter(this, arguments, void 0, function* (retries = 5, retentionMs = 0) {
+            if (this.compactingPromise) {
+                (0, logger_1.debug)("vectordb", "Optimize already in progress, skipping");
+                return;
+            }
             const table = yield this.ensureTable();
             const cutoff = new Date(Date.now() - retentionMs);
-            for (let attempt = 1; attempt <= retries; attempt++) {
-                try {
-                    const done = (0, logger_1.timer)("vectordb", "optimize");
-                    const stats = yield table.optimize({
-                        cleanupOlderThan: cutoff,
-                        deleteUnverified: true,
-                    });
-                    done();
-                    const { compaction, prune } = stats;
-                    if (compaction.fragmentsRemoved > 0 ||
-                        prune.oldVersionsRemoved > 0 ||
-                        prune.bytesRemoved > 0) {
-                        (0, logger_1.log)("vectordb", `Compacted: ${compaction.fragmentsRemoved} frags → ${compaction.fragmentsAdded}, ` +
-                            `pruned ${prune.oldVersionsRemoved} versions, ` +
-                            `freed ${(prune.bytesRemoved / 1024 / 1024).toFixed(1)}MB`);
-                    }
-                    else {
-                        (0, logger_1.debug)("vectordb", "Optimize: nothing to compact or prune");
-                    }
-                    return;
-                }
-                catch (e) {
-                    const msg = e instanceof Error ? e.message : String(e);
-                    if (msg.includes("Nothing to do")) {
-                        (0, logger_1.debug)("vectordb", "Optimize: nothing to do");
+            let resolveCompacting;
+            this.compactingPromise = new Promise((r) => { resolveCompacting = r; });
+            try {
+                for (let attempt = 1; attempt <= retries; attempt++) {
+                    yield this.drainWrites();
+                    try {
+                        const done = (0, logger_1.timer)("vectordb", "optimize");
+                        const stats = yield table.optimize({
+                            cleanupOlderThan: cutoff,
+                            deleteUnverified: true,
+                        });
+                        done();
+                        const { compaction, prune } = stats;
+                        if (compaction.fragmentsRemoved > 0 ||
+                            prune.oldVersionsRemoved > 0 ||
+                            prune.bytesRemoved > 0) {
+                            (0, logger_1.log)("vectordb", `Compacted: ${compaction.fragmentsRemoved} frags → ${compaction.fragmentsAdded}, ` +
+                                `pruned ${prune.oldVersionsRemoved} versions, ` +
+                                `freed ${(prune.bytesRemoved / 1024 / 1024).toFixed(1)}MB`);
+                        }
+                        else {
+                            (0, logger_1.debug)("vectordb", "Optimize: nothing to compact or prune");
+                        }
                         return;
                     }
-                    if (attempt < retries &&
-                        (msg.includes("conflict") || msg.includes("Retryable"))) {
-                        const delay = 1000 * Math.pow(2, (attempt - 1));
-                        (0, logger_1.log)("vectordb", `Optimize conflict (attempt ${attempt}/${retries}), retrying in ${delay}ms`);
-                        yield new Promise((r) => setTimeout(r, delay));
-                        continue;
+                    catch (e) {
+                        const msg = e instanceof Error ? e.message : String(e);
+                        if (msg.includes("Nothing to do")) {
+                            (0, logger_1.debug)("vectordb", "Optimize: nothing to do");
+                            return;
+                        }
+                        // ENOSPC: return immediately — retrying will only make things worse
+                        if (msg.includes("No space left on device") || msg.includes("os error 28")) {
+                            (0, logger_1.log)("vectordb", `Optimize failed (ENOSPC): disk full — skipping retries`);
+                            return;
+                        }
+                        if (attempt < retries &&
+                            (msg.includes("conflict") || msg.includes("Retryable"))) {
+                            const delay = 1000 * Math.pow(2, (attempt - 1));
+                            (0, logger_1.log)("vectordb", `Optimize conflict (attempt ${attempt}/${retries}), retrying in ${delay}ms`);
+                            yield new Promise((r) => setTimeout(r, delay));
+                            continue;
+                        }
+                        (0, logger_1.log)("vectordb", `Optimize failed: ${msg}`);
+                        return;
                     }
-                    (0, logger_1.log)("vectordb", `Optimize failed: ${msg}`);
-                    return;
                 }
             }
+            finally {
+                this.compactingPromise = null;
+                resolveCompacting();
+            }
         });
     }
     /**
@@ -383,17 +496,27 @@ class VectorDB {
             }
             this.maintenanceRunning = true;
             try {
+                const pressure = this.checkDiskPressure();
+                if (pressure === "critical") {
+                    const freeGb = (this.getAvailableBytes() / 1024 / 1024 / 1024).toFixed(1);
+                    (0, logger_1.log)("vectordb", `Maintenance skipped: disk critically low (${freeGb}GB free)`);
+                    return;
+                }
                 yield this.createFTSIndex();
+                if (pressure === "low") {
+                    (0, logger_1.log)("vectordb", `Low disk — single-pass optimize (no bloat retry)`);
+                    yield this.optimize(1);
+                    return;
+                }
+                // Normal maintenance: full optimize + bloat check
                 yield this.optimize();
-                // Check for bloat after first optimize pass — if fragments were locked
-                // by concurrent readers, optimize succeeds but skips them. A second pass
-                // after a brief pause catches what the first couldn't.
                 const table = yield this.ensureTable();
                 const stats = yield table.stats();
                 const diskSize = this.getDirectorySize(this.lancedbDir);
                 const logicalSize = stats.totalBytes;
                 const bloatRatio = logicalSize > 0 ? diskSize / logicalSize : 0;
-                if (bloatRatio > 2.0) {
+                // Only retry if disk is still ok after optimize (don't spiral)
+                if (bloatRatio > 2.0 && this.checkDiskPressure() === "ok") {
                     (0, logger_1.log)("vectordb", `Bloat detected after optimize: ${(diskSize / 1024 / 1024).toFixed(0)}MB disk vs ${(logicalSize / 1024 / 1024).toFixed(0)}MB logical (${bloatRatio.toFixed(1)}x) — retrying`);
                     yield new Promise((r) => setTimeout(r, 2000));
                     yield this.optimize();
@@ -404,6 +527,33 @@ class VectorDB {
             }
         });
     }
+    compactIfNeeded() {
+        return __awaiter(this, arguments, void 0, function* (threshold = config_1.FRAGMENT_COMPACT_THRESHOLD) {
+            if (this.maintenanceRunning)
+                return false;
+            if (this.checkDiskPressure() !== "ok")
+                return false;
+            try {
+                const table = yield this.ensureTable();
+                const stats = yield table.stats();
+                if (stats.fragmentStats.numSmallFragments > threshold) {
+                    (0, logger_1.log)("vectordb", `Fragment threshold exceeded (${stats.fragmentStats.numSmallFragments} > ${threshold}) — compacting`);
+                    this.maintenanceRunning = true;
+                    try {
+                        yield this.optimize(2);
+                    }
+                    finally {
+                        this.maintenanceRunning = false;
+                    }
+                    return true;
+                }
+            }
+            catch (err) {
+                (0, logger_1.debug)("vectordb", `compactIfNeeded check failed: ${err}`);
+            }
+            return false;
+        });
+    }
     getDirectorySize(dirPath) {
         let totalSize = 0;
         try {
@@ -491,73 +641,82 @@ class VectorDB {
         return __awaiter(this, void 0, void 0, function* () {
             if (!paths.length)
                 return;
+            this.ensureDiskOk();
             const table = yield this.ensureTable();
             const unique = Array.from(new Set(paths));
             const batchSize = 500;
-            for (let i = 0; i < unique.length; i += batchSize) {
-                const slice = unique.slice(i, i + batchSize);
-                const values = slice.map((p) => `'${(0, filter_builder_1.escapeSqlString)(p)}'`).join(",");
-                const where = `path IN (${values})`;
-                // Skip no-op deletes to avoid creating empty LanceDB versions
-                const existing = yield table
-                    .query()
-                    .select(["id"])
-                    .where(where)
-                    .limit(1)
-                    .toArray();
-                if (existing.length > 0) {
-                    yield table.delete(where);
+            yield this.withWriteGate(() => __awaiter(this, void 0, void 0, function* () {
+                for (let i = 0; i < unique.length; i += batchSize) {
+                    const slice = unique.slice(i, i + batchSize);
+                    const values = slice.map((p) => `'${(0, filter_builder_1.escapeSqlString)(p)}'`).join(",");
+                    const where = `path IN (${values})`;
+                    // Skip no-op deletes to avoid creating empty LanceDB versions
+                    const existing = yield table
+                        .query()
+                        .select(["id"])
+                        .where(where)
+                        .limit(1)
+                        .toArray();
+                    if (existing.length > 0) {
+                        yield table.delete(where);
+                    }
                 }
-            }
+            }));
         });
     }
     updateRows(ids, field, values) {
         return __awaiter(this, void 0, void 0, function* () {
-            var _a;
             if (!ids.length)
                 return;
             const table = yield this.ensureTable();
-            for (let i = 0; i < ids.length; i++) {
-                const escaped = (0, filter_builder_1.escapeSqlString)(ids[i]);
-                yield table.update({
-                    where: `id = '${escaped}'`,
-                    values: { [field]: (_a = values[i]) !== null && _a !== void 0 ? _a : "" },
-                });
-            }
+            yield this.withWriteGate(() => __awaiter(this, void 0, void 0, function* () {
+                var _a;
+                for (let i = 0; i < ids.length; i++) {
+                    const escaped = (0, filter_builder_1.escapeSqlString)(ids[i]);
+                    yield table.update({
+                        where: `id = '${escaped}'`,
+                        values: { [field]: (_a = values[i]) !== null && _a !== void 0 ? _a : "" },
+                    });
+                }
+            }));
         });
     }
     deletePathsExcludingIds(paths, excludeIds) {
         return __awaiter(this, void 0, void 0, function* () {
             if (!paths.length)
                 return;
+            this.ensureDiskOk();
             const table = yield this.ensureTable();
             const unique = Array.from(new Set(paths));
             const batchSize = 500;
             const idExclusion = excludeIds.length > 0
                 ? ` AND id NOT IN (${excludeIds.map((id) => `'${(0, filter_builder_1.escapeSqlString)(id)}'`).join(",")})`
                 : "";
-            for (let i = 0; i < unique.length; i += batchSize) {
-                const slice = unique.slice(i, i + batchSize);
-                const values = slice
-                    .map((p) => `'${(0, filter_builder_1.escapeSqlString)(p)}'`)
-                    .join(",");
-                const where = `path IN (${values})${idExclusion}`;
-                const existing = yield table
-                    .query()
-                    .select(["id"])
-                    .where(where)
-                    .limit(1)
-                    .toArray();
-                if (existing.length > 0) {
-                    yield table.delete(where);
+            yield this.withWriteGate(() => __awaiter(this, void 0, void 0, function* () {
+                for (let i = 0; i < unique.length; i += batchSize) {
+                    const slice = unique.slice(i, i + batchSize);
+                    const values = slice
+                        .map((p) => `'${(0, filter_builder_1.escapeSqlString)(p)}'`)
+                        .join(",");
+                    const where = `path IN (${values})${idExclusion}`;
+                    const existing = yield table
+                        .query()
+                        .select(["id"])
+                        .where(where)
+                        .limit(1)
+                        .toArray();
+                    if (existing.length > 0) {
+                        yield table.delete(where);
+                    }
                 }
-            }
+            }));
         });
     }
     deletePathsWithPrefix(prefix) {
         return __awaiter(this, void 0, void 0, function* () {
+            this.ensureDiskOk();
             const table = yield this.ensureTable();
-            yield table.delete(`path LIKE '${(0, filter_builder_1.escapeSqlString)(prefix)}%'`);
+            yield this.withWriteGate(() => table.delete(`path LIKE '${(0, filter_builder_1.escapeSqlString)(prefix)}%'`));
         });
     }
     drop() {
@@ -596,3 +755,4 @@ class VectorDB {
     }
 }
 exports.VectorDB = VectorDB;
+VectorDB.DISK_CHECK_INTERVAL_MS = 30000;

package/dist/lib/utils/log-rotate.js

@@ -34,6 +34,7 @@ var __importStar = (this && this.__importStar) || (function () {
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.openRotatedLog = openRotatedLog;
+exports.rotateLogFds = rotateLogFds;
 const fs = __importStar(require("node:fs"));
 const path = __importStar(require("node:path"));
 const MAX_LOG_BYTES = 5 * 1024 * 1024; // 5 MB
@@ -53,3 +54,29 @@ function openRotatedLog(logPath, maxBytes = MAX_LOG_BYTES) {
     catch (_a) { }
     return fs.openSync(logPath, "a");
 }
+/**
+ * Mid-session log rotation for daemon processes.
+ * Renames the log to .prev and reopens stdout/stderr (fd 1, 2) to a fresh file.
+ * Safe on Unix: synchronous close/open guarantees fd 1 and 2 are reassigned.
+ */
+function rotateLogFds(logPath, maxBytes = MAX_LOG_BYTES) {
+    try {
+        const stat = fs.statSync(logPath);
+        if (stat.size <= maxBytes)
+            return false;
+    }
+    catch (_a) {
+        return false;
+    }
+    try {
+        fs.renameSync(logPath, `${logPath}.prev`);
+        fs.closeSync(1);
+        fs.closeSync(2);
+        fs.openSync(logPath, "a"); // gets fd 1 (stdout)
+        fs.openSync(logPath, "a"); // gets fd 2 (stderr)
+        return true;
+    }
+    catch (_b) {
+        return false;
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "grepmax",
-  "version": "0.14.9",
+  "version": "0.15.0",
   "author": "Robert Owens <78518764+reowens@users.noreply.github.com>",
   "homepage": "https://github.com/reowens/grepmax",
   "bugs": {

package/plugins/grepmax/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "grepmax",
-  "version": "0.14.9",
+  "version": "0.15.0",
   "description": "Semantic code search for Claude Code. Automatically indexes your project and provides intelligent search capabilities.",
   "author": {
     "name": "Robert Owens",