grepmax 0.14.3 → 0.14.5

package/dist/commands/add.js

@@ -92,7 +92,27 @@ Examples:
         if (children.length > 0) {
             const names = children.map((c) => c.name).join(", ");
             console.log(`Absorbing ${children.length} sub-project(s): ${names}`);
+            const { ensureDaemonRunning: checkDaemon, sendStreamingCommand: sendCmd } = yield Promise.resolve().then(() => __importStar(require("../lib/utils/daemon-client")));
+            const daemonUp = yield checkDaemon();
             for (const child of children) {
+                if (daemonUp) {
+                    // Daemon handles unwatch + vector delete + MetaCache cleanup
+                    yield sendCmd({ cmd: "remove", root: child.root }, () => { });
+                }
+                else {
+                    // Direct mode: delete vectors and MetaCache entries
+                    const childPaths = (0, project_root_1.ensureProjectPaths)(child.root);
+                    const db = new vector_db_1.VectorDB(childPaths.lancedbDir);
+                    const childPrefix = child.root.endsWith("/") ? child.root : `${child.root}/`;
+                    yield db.deletePathsWithPrefix(childPrefix);
+                    const { MetaCache } = yield Promise.resolve().then(() => __importStar(require("../lib/store/meta-cache")));
+                    const mc = new MetaCache(childPaths.lmdbPath);
+                    const keys = yield mc.getKeysWithPrefix(childPrefix);
+                    for (const key of keys)
+                        mc.delete(key);
+                    mc.close();
+                    yield db.close();
+                }
                 (0, project_registry_1.removeProject)(child.root);
             }
         }

package/dist/commands/watch.js

@@ -114,6 +114,13 @@ exports.watch = new commander_1.Command("watch")
         }
         process.on("SIGINT", () => daemon.shutdown().then(() => (0, exit_1.gracefulExit)()));
         process.on("SIGTERM", () => daemon.shutdown().then(() => (0, exit_1.gracefulExit)()));
+        process.on("uncaughtException", (err) => {
+            console.error("[daemon] uncaughtException:", err);
+            daemon.shutdown().then(() => process.exit(1));
+        });
+        process.on("unhandledRejection", (reason) => {
+            console.error("[daemon] unhandledRejection:", reason);
+        });
         return;
     }
     // --- Per-project mode ---

package/dist/lib/daemon/daemon.js

@@ -69,6 +69,13 @@ const project_registry_1 = require("../utils/project-registry");
 const watcher_store_1 = require("../utils/watcher-store");
 const server_1 = require("../llm/server");
 const ipc_handler_1 = require("./ipc-handler");
+const logger_1 = require("../utils/logger");
+const daemon_client_1 = require("../utils/daemon-client");
+const watcher_store_2 = require("../utils/watcher-store");
+const index_config_1 = require("../index/index-config");
+const log_rotate_1 = require("../utils/log-rotate");
+const node_child_process_1 = require("node:child_process");
+const http = __importStar(require("node:http"));
 const IDLE_TIMEOUT_MS = 30 * 60 * 1000; // 30 minutes
 const HEARTBEAT_INTERVAL_MS = 60 * 1000;
 class Daemon {
@@ -85,20 +92,49 @@ class Daemon {
         this.idleInterval = null;
         this.shuttingDown = false;
         this.pendingOps = new Set();
+        this.watcherFailCount = new Map();
+        this.pollIntervals = new Map();
         this.projectLocks = new Map();
         this.llmServer = null;
+        this.mlxChild = null;
     }
     start() {
         return __awaiter(this, void 0, void 0, function* () {
             process.title = "gmax-daemon";
+            // 0. Singleton enforcement: check PID file for existing daemon
+            try {
+                const pidStr = fs.readFileSync(config_1.PATHS.daemonPidFile, "utf-8").trim();
+                const existingPid = parseInt(pidStr, 10);
+                if (existingPid && existingPid !== process.pid && (0, watcher_store_2.isProcessRunning)(existingPid)) {
+                    (0, logger_1.log)("daemon", `found existing daemon PID:${existingPid}, checking socket...`);
+                    const responsive = yield (0, daemon_client_1.isDaemonRunning)();
+                    if (responsive) {
+                        (0, logger_1.log)("daemon", "existing daemon is responsive — exiting");
+                        process.exit(0);
+                    }
+                    // Unresponsive but alive — kill it
+                    (0, logger_1.log)("daemon", `existing daemon PID:${existingPid} unresponsive — killing`);
+                    yield (0, process_1.killProcess)(existingPid);
+                    (0, logger_1.log)("daemon", `killed stale daemon PID:${existingPid}`);
+                }
+            }
+            catch (_a) {
+                // No PID file or unreadable — proceed normally
+            }
             // 1. Acquire exclusive lock — kernel-enforced, atomic, auto-released on death
             fs.mkdirSync(path.dirname(config_1.PATHS.daemonLockFile), { recursive: true });
             fs.writeFileSync(config_1.PATHS.daemonLockFile, "", { flag: "a" }); // ensure file exists
+            (0, logger_1.debug)("daemon", "acquiring lock...");
             try {
                 this.releaseLock = yield proper_lockfile_1.default.lock(config_1.PATHS.daemonLockFile, {
                     retries: 0,
-                    stale: 30000,
+                    stale: 120000,
+                    onCompromised: () => {
+                        console.error("[daemon] Lock compromised — another daemon took over. Shutting down.");
+                        this.shutdown();
+                    },
                 });
+                (0, logger_1.debug)("daemon", "lock acquired");
             }
             catch (err) {
                 if (err.code === "ELOCKED") {
@@ -120,7 +156,7 @@ class Daemon {
             try {
                 fs.unlinkSync(config_1.PATHS.daemonSocket);
             }
-            catch (_a) { }
+            catch (_b) { }
             // 5. Open shared resources
             try {
                 fs.mkdirSync(config_1.PATHS.cacheDir, { recursive: true });
@@ -137,9 +173,15 @@ class Daemon {
             }
             // 6. LLM server manager (constructed, not started — starts on first request)
             this.llmServer = new server_1.LlmServer();
+            // 6b. MLX embed server — start if GPU mode is active
+            const globalConfig = (0, index_config_1.readGlobalConfig)();
+            const isAppleSilicon = process.arch === "arm64" && process.platform === "darwin";
+            if (isAppleSilicon && globalConfig.embedMode === "gpu") {
+                yield this.ensureMlxServer(globalConfig.mlxModel);
+            }
             // 7. Register daemon (only after resources are open)
             (0, watcher_store_1.registerDaemon)(process.pid);
-            // 7. Subscribe to all registered projects (skip missing directories)
+            // 8. Subscribe to all registered projects (skip missing directories)
             const allProjects = (0, project_registry_1.listProjects)();
             const indexed = allProjects.filter((p) => p.status === "indexed");
             for (const p of indexed) {
@@ -154,26 +196,27 @@ class Daemon {
                     console.error(`[daemon] Failed to watch ${path.basename(p.root)}:`, err);
                 }
             }
-            // 7b. Index pending projects in the background
+            // 8b. Index pending projects in the background
             const pending = allProjects.filter((p) => p.status === "pending" && fs.existsSync(p.root));
             for (const p of pending) {
                 this.indexPendingProject(p.root).catch((err) => {
                     console.error(`[daemon] Failed to index pending ${path.basename(p.root)}:`, err);
                 });
             }
-            // 8. Heartbeat
+            // 9. Heartbeat
             this.heartbeatInterval = setInterval(() => {
                 (0, watcher_store_1.heartbeat)(process.pid);
             }, HEARTBEAT_INTERVAL_MS);
-            // 9. Idle timeout
+            // 10. Idle timeout
             this.idleInterval = setInterval(() => {
                 if (Date.now() - this.lastActivity > IDLE_TIMEOUT_MS) {
                     console.log("[daemon] Idle for 30 minutes, shutting down");
                     this.shutdown();
                 }
             }, HEARTBEAT_INTERVAL_MS);
-            // 10. Socket server
+            // 11. Socket server
             this.server = net.createServer((conn) => {
+                (0, logger_1.debug)("daemon", "client connected");
                 let buf = "";
                 conn.on("data", (chunk) => {
                     buf += chunk.toString();
@@ -275,17 +318,7 @@ class Daemon {
             });
             this.processors.set(root, processor);
             // Subscribe with @parcel/watcher — native backend, no polling
-            const sub = yield watcher.subscribe(root, (err, events) => {
-                if (err) {
-                    console.error(`[daemon:${path.basename(root)}] Watcher error:`, err);
-                    return;
-                }
-                for (const event of events) {
-                    processor.handleFileEvent(event.type === "delete" ? "unlink" : "change", event.path);
-                }
-                this.lastActivity = Date.now();
-            }, { ignore: watcher_1.WATCHER_IGNORE_GLOBS });
-            this.subscriptions.set(root, sub);
+            yield this.subscribeWatcher(root, processor);
             (0, watcher_store_1.registerWatcher)({
                 pid: process.pid,
                 projectRoot: root,
@@ -301,16 +334,117 @@ class Daemon {
             console.log(`[daemon] Watching ${root}`);
         });
     }
+    subscribeWatcher(root, processor) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const name = path.basename(root);
+            // Unsubscribe existing watcher if any (e.g. during recovery)
+            const existingSub = this.subscriptions.get(root);
+            if (existingSub) {
+                try {
+                    yield existingSub.unsubscribe();
+                }
+                catch (_a) { }
+                this.subscriptions.delete(root);
+            }
+            const sub = yield watcher.subscribe(root, (err, events) => {
+                if (err) {
+                    console.error(`[daemon:${name}] Watcher error:`, err);
+                    this.recoverWatcher(root, processor);
+                    return;
+                }
+                // Watcher is healthy — reset fail counter
+                this.watcherFailCount.delete(root);
+                for (const event of events) {
+                    processor.handleFileEvent(event.type === "delete" ? "unlink" : "change", event.path);
+                }
+                this.lastActivity = Date.now();
+            }, { ignore: watcher_1.WATCHER_IGNORE_GLOBS });
+            this.subscriptions.set(root, sub);
+        });
+    }
+    recoverWatcher(root, processor) {
+        var _a;
+        const name = path.basename(root);
+        if (this.shuttingDown)
+            return;
+        // Debounce: avoid multiple overlapping recovery attempts
+        const recoveryKey = `recover:${root}`;
+        if (this.pendingOps.has(recoveryKey))
+            return;
+        this.pendingOps.add(recoveryKey);
+        const fails = ((_a = this.watcherFailCount.get(root)) !== null && _a !== void 0 ? _a : 0) + 1;
+        this.watcherFailCount.set(root, fails);
+        const MAX_WATCHER_RETRIES = 3;
+        const POLL_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes
+        if (fails > MAX_WATCHER_RETRIES) {
+            // FSEvents can't handle this project — degrade to periodic catchup scans
+            if (!this.pollIntervals.has(root)) {
+                console.error(`[daemon:${name}] FSEvents unreliable after ${fails} failures — switching to poll mode (${POLL_INTERVAL_MS / 60000}min interval)`);
+                // Unsubscribe the broken watcher
+                const sub = this.subscriptions.get(root);
+                if (sub) {
+                    sub.unsubscribe().catch(() => { });
+                    this.subscriptions.delete(root);
+                }
+                // Run an immediate catchup, then schedule periodic ones
+                this.catchupScan(root, processor).catch((err) => {
+                    console.error(`[daemon:${name}] Poll catchup failed:`, err);
+                });
+                const interval = setInterval(() => {
+                    if (this.shuttingDown)
+                        return;
+                    this.lastActivity = Date.now();
+                    this.catchupScan(root, processor).catch((err) => {
+                        console.error(`[daemon:${name}] Poll catchup failed:`, err);
+                    });
+                }, POLL_INTERVAL_MS);
+                this.pollIntervals.set(root, interval);
+                (0, watcher_store_1.registerWatcher)({
+                    pid: process.pid,
+                    projectRoot: root,
+                    startTime: Date.now(),
+                    status: "watching",
+                    lastHeartbeat: Date.now(),
+                });
+            }
+            this.pendingOps.delete(recoveryKey);
+            return;
+        }
+        // Backoff: wait before re-subscribing (3s, 6s, 12s)
+        const delayMs = 3000 * Math.pow(2, fails - 1);
+        console.error(`[daemon:${name}] Recovering watcher (attempt ${fails}/${MAX_WATCHER_RETRIES}, backoff ${delayMs}ms)...`);
+        setTimeout(() => {
+            if (this.shuttingDown) {
+                this.pendingOps.delete(recoveryKey);
+                return;
+            }
+            (() => __awaiter(this, void 0, void 0, function* () {
+                try {
+                    yield this.subscribeWatcher(root, processor);
+                    yield this.catchupScan(root, processor);
+                    console.log(`[daemon:${name}] Watcher recovered`);
+                }
+                catch (err) {
+                    console.error(`[daemon:${name}] Watcher recovery failed:`, err);
+                }
+                finally {
+                    this.pendingOps.delete(recoveryKey);
+                }
+            }))();
+        }, delayMs);
+    }
     catchupScan(root, processor) {
         return __awaiter(this, void 0, void 0, function* () {
             var _a, e_1, _b, _c;
             const { walk } = yield Promise.resolve().then(() => __importStar(require("../index/walker")));
-            const { INDEXABLE_EXTENSIONS } = yield Promise.resolve().then(() => __importStar(require("../../config")));
+            const { INDEXABLE_EXTENSIONS, MAX_FILE_SIZE_BYTES } = yield Promise.resolve().then(() => __importStar(require("../../config")));
             const { isFileCached } = yield Promise.resolve().then(() => __importStar(require("../utils/cache-check")));
             const rootPrefix = root.endsWith("/") ? root : `${root}/`;
             const cachedPaths = yield this.metaCache.getKeysWithPrefix(rootPrefix);
             const seenPaths = new Set();
             let queued = 0;
+            let skipped = 0;
+            let debugSamples = 0;
             try {
                 for (var _d = true, _e = __asyncValues(walk(root, {
                     additionalPatterns: ["**/.git/**", "**/.gmax/**"],
@@ -326,11 +460,35 @@ class Daemon {
                     seenPaths.add(absPath);
                     try {
                         const stats = yield fs.promises.stat(absPath);
+                        // Skip files that are too large or empty — they'll never be indexed
+                        if (stats.size === 0 || stats.size > MAX_FILE_SIZE_BYTES)
+                            continue;
                         const cached = this.metaCache.get(absPath);
                         if (!isFileCached(cached, stats)) {
+                            // Fast path: if only mtime changed but size is identical and we have a hash,
+                            // just verify the hash in-process instead of sending to a worker.
+                            if (cached && cached.hash && cached.size === stats.size) {
+                                const { computeBufferHash } = yield Promise.resolve().then(() => __importStar(require("../utils/file-utils")));
+                                const buf = yield fs.promises.readFile(absPath);
+                                const hash = computeBufferHash(buf);
+                                if (hash === cached.hash) {
+                                    // Content unchanged — update mtime in cache and skip worker
+                                    this.metaCache.put(absPath, Object.assign(Object.assign({}, cached), { mtimeMs: stats.mtimeMs }));
+                                    skipped++;
+                                    continue;
+                                }
+                            }
+                            // Debug: log first few misses to diagnose re-queue loops
+                            if (debugSamples < 5) {
+                                (0, logger_1.debug)("catchup", `miss ${relPath}: cached=${cached ? `mtime=${Math.trunc(cached.mtimeMs)} size=${cached.size}` : "null"} stat=mtime=${Math.trunc(stats.mtimeMs)} size=${stats.size}`);
+                                debugSamples++;
+                            }
                             processor.handleFileEvent("change", absPath);
                             queued++;
                         }
+                        else {
+                            skipped++;
+                        }
                     }
                     catch (_g) { }
                 }
@@ -342,6 +500,7 @@ class Daemon {
                 }
                 finally { if (e_1) throw e_1.error; }
             }
+            (0, logger_1.debug)("catchup", `${path.basename(root)}: ${queued} queued, ${skipped} skipped (cached ok), ${seenPaths.size} total`);
             // Purge files deleted while daemon was offline
             let purged = 0;
             for (const cachedPath of cachedPaths) {
@@ -366,7 +525,9 @@ class Daemon {
                 var _a;
                 if (!this.vectorDb || !this.metaCache)
                     return;
-                console.log(`[daemon] Indexing pending project: ${path.basename(root)}`);
+                const name = path.basename(root);
+                const start = Date.now();
+                (0, logger_1.log)("daemon", `indexPendingProject start: ${name} (${root})`);
                 this.vectorDb.pauseMaintenanceLoop();
                 try {
                     const result = yield (0, syncer_1.initialSync)({
@@ -380,11 +541,11 @@ class Daemon {
                         (0, project_registry_1.registerProject)(Object.assign(Object.assign({}, proj), { lastIndexed: new Date().toISOString(), chunkCount: result.indexed, status: "indexed" }));
                     }
                     yield this.watchProject(root);
-                    console.log(`[daemon] Indexed ${path.basename(root)} (${result.total} files, ${result.indexed} chunks)`);
+                    (0, logger_1.log)("daemon", `indexPendingProject done: ${name} — ${result.total} files, ${result.indexed} chunks, ${Date.now() - start}ms`);
                 }
                 catch (err) {
                     const msg = err instanceof Error ? err.message : String(err);
-                    console.error(`[daemon] indexPendingProject failed for ${path.basename(root)}: ${msg}`);
+                    console.error(`[daemon] indexPendingProject failed for ${name} after ${Date.now() - start}ms: ${msg}`);
                 }
                 finally {
                     (_a = this.vectorDb) === null || _a === void 0 ? void 0 : _a.resumeMaintenanceLoop();
@@ -680,6 +841,68 @@ class Daemon {
             }
         });
     }
+    // --- MLX embed server management ---
+    isMlxServerUp() {
+        return __awaiter(this, void 0, void 0, function* () {
+            const port = parseInt(process.env.MLX_EMBED_PORT || "8100", 10);
+            return new Promise((resolve) => {
+                const req = http.get({ hostname: "127.0.0.1", port, path: "/health", timeout: 2000 }, (res) => { res.resume(); resolve(res.statusCode === 200); });
+                req.on("error", () => resolve(false));
+                req.on("timeout", () => { req.destroy(); resolve(false); });
+            });
+        });
+    }
+    ensureMlxServer(mlxModel) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (yield this.isMlxServerUp()) {
+                console.log("[daemon] MLX embed server already running");
+                return;
+            }
+            // Find mlx-embed-server/server.py relative to the grepmax package
+            const candidates = [
+                path.resolve(__dirname, "../../../mlx-embed-server"),
+                path.resolve(__dirname, "../../mlx-embed-server"),
+            ];
+            const serverDir = candidates.find((d) => fs.existsSync(path.join(d, "server.py")));
+            if (!serverDir) {
+                console.warn("[daemon] MLX embed server not found — falling back to CPU embeddings");
+                return;
+            }
+            const logFd = (0, log_rotate_1.openRotatedLog)(path.join(config_1.PATHS.logsDir, "mlx-embed-server.log"));
+            const env = Object.assign({}, process.env);
+            if (mlxModel)
+                env.MLX_EMBED_MODEL = mlxModel;
+            this.mlxChild = (0, node_child_process_1.spawn)("uv", ["run", "python", "server.py"], {
+                cwd: serverDir,
+                detached: true,
+                stdio: ["ignore", logFd, logFd],
+                env,
+            });
+            this.mlxChild.unref();
+            console.log(`[daemon] Starting MLX embed server (PID: ${this.mlxChild.pid})`);
+            // Poll for readiness (up to 30s)
+            for (let i = 0; i < 30; i++) {
+                yield new Promise((r) => setTimeout(r, 1000));
+                if (yield this.isMlxServerUp()) {
+                    console.log("[daemon] MLX embed server ready");
+                    return;
+                }
+            }
+            console.error("[daemon] MLX embed server failed to start within 30s — falling back to CPU embeddings");
+            this.mlxChild = null;
+        });
+    }
+    stopMlxServer() {
+        var _a;
+        if (!((_a = this.mlxChild) === null || _a === void 0 ? void 0 : _a.pid))
+            return;
+        try {
+            process.kill(this.mlxChild.pid, "SIGTERM");
+            console.log(`[daemon] Stopped MLX embed server (PID: ${this.mlxChild.pid})`);
+        }
+        catch (_b) { }
+        this.mlxChild = null;
+    }
     shutdown() {
         return __awaiter(this, void 0, void 0, function* () {
             var _a, _b, _c, _d;
@@ -700,6 +923,13 @@ class Daemon {
                 yield ((_a = this.llmServer) === null || _a === void 0 ? void 0 : _a.stop());
             }
             catch (_e) { }
+            // Stop MLX embed server if we started it
+            this.stopMlxServer();
+            // Stop poll intervals
+            for (const interval of this.pollIntervals.values()) {
+                clearInterval(interval);
+            }
+            this.pollIntervals.clear();
             // Unsubscribe all watchers
             for (const sub of this.subscriptions.values()) {
                 try {

package/dist/lib/daemon/ipc-handler.js

@@ -12,6 +12,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.writeProgress = writeProgress;
 exports.writeDone = writeDone;
 exports.handleCommand = handleCommand;
+const logger_1 = require("../utils/logger");
 /**
  * Write a streaming progress line to the IPC connection.
  */
@@ -38,6 +39,7 @@ function writeDone(conn, data) {
 function handleCommand(daemon, cmd, conn) {
     return __awaiter(this, void 0, void 0, function* () {
         try {
+            (0, logger_1.debug)("daemon", `ipc cmd=${cmd.cmd}${cmd.root ? ` root=${cmd.root}` : ""}`);
             switch (cmd.cmd) {
                 case "ping":
                     return { ok: true, pid: process.pid, uptime: daemon.uptime() };

package/dist/lib/index/batch-processor.js

@@ -51,6 +51,9 @@ const file_utils_1 = require("../utils/file-utils");
 const logger_1 = require("../utils/logger");
 const pool_1 = require("../workers/pool");
 const watcher_batch_1 = require("./watcher-batch");
+// Fast path-segment check to reject events that leak through FSEvents overflow.
+// Matches /node_modules/, /.git/, /dist/, /build/, /.next/, etc. anywhere in path.
+const IGNORED_PATH_SEGMENTS_RE = /\/(?:node_modules|\.git|\.next|\.nuxt|__pycache__|coverage|\.gmax)\//;
 const DEBOUNCE_MS = 2000;
 const MAX_RETRIES = 5;
 const MAX_BATCH_SIZE = 50;
@@ -83,6 +86,10 @@ class ProjectBatchProcessor {
         const bn = path.basename(absPath).toLowerCase();
         if (!config_1.INDEXABLE_EXTENSIONS.has(ext) && !config_1.INDEXABLE_EXTENSIONS.has(bn))
             return;
+        // Safety net: reject paths with ignored directory segments.
+        // FSEvents can leak events during overflow before the watcher drops them.
+        if (IGNORED_PATH_SEGMENTS_RE.test(absPath))
+            return;
         this.pending.set(absPath, event);
         (_a = this.onActivity) === null || _a === void 0 ? void 0 : _a.call(this);
         this.scheduleBatch();
@@ -210,15 +217,6 @@ class ProjectBatchProcessor {
                         this.pending.set(absPath, event);
                     }
                 }
-                // Requeue files that were attempted but not successfully processed
-                // (e.g. pool became unhealthy mid-batch before vectors were flushed)
-                for (const [absPath, event] of batch) {
-                    if (attempted.has(absPath) && !metaUpdates.has(absPath) && !metaDeletes.includes(absPath)) {
-                        if (!this.pending.has(absPath)) {
-                            this.pending.set(absPath, event);
-                        }
-                    }
-                }
                 // Flush to VectorDB: insert first, then delete old (preserving new)
                 const newIds = vectors.map((v) => v.id);
                 if (vectors.length > 0) {