grepmax 0.12.13 → 0.13.1

package/dist/lib/daemon/daemon.js

@@ -53,6 +53,7 @@ const watcher = __importStar(require("@parcel/watcher"));
 const proper_lockfile_1 = __importDefault(require("proper-lockfile"));
 const config_1 = require("../../config");
 const batch_processor_1 = require("../index/batch-processor");
+const syncer_1 = require("../index/syncer");
 const watcher_1 = require("../index/watcher");
 const meta_cache_1 = require("../store/meta-cache");
 const vector_db_1 = require("../store/vector-db");
@@ -76,6 +77,7 @@ class Daemon {
         this.idleInterval = null;
         this.shuttingDown = false;
         this.pendingOps = new Set();
+        this.projectLocks = new Map();
     }
     start() {
         return __awaiter(this, void 0, void 0, function* () {
@@ -168,9 +170,12 @@ class Daemon {
                         conn.end();
                         return;
                     }
-                    (0, ipc_handler_1.handleCommand)(this, cmd).then((resp) => {
-                        conn.write(`${JSON.stringify(resp)}\n`);
-                        conn.end();
+                    (0, ipc_handler_1.handleCommand)(this, cmd, conn).then((resp) => {
+                        // null means the handler is managing the connection (streaming)
+                        if (resp !== null) {
+                            conn.write(`${JSON.stringify(resp)}\n`);
+                            conn.end();
+                        }
                     });
                 });
                 conn.on("error", () => { });
@@ -207,7 +212,6 @@ class Daemon {
                 projectRoot: root,
                 vectorDb: this.vectorDb,
                 metaCache: this.metaCache,
-                dataDir: config_1.PATHS.globalRoot,
                 onReindex: (files, ms) => {
                     console.log(`[daemon:${path.basename(root)}] Reindexed ${files} file${files !== 1 ? "s" : ""} (${(ms / 1000).toFixed(1)}s)`);
                     // Update project registry so gmax status shows fresh data
@@ -286,6 +290,212 @@ class Daemon {
     uptime() {
         return Math.floor((Date.now() - this.startTime) / 1000);
     }
+    /** Reset idle timer — call during long-running operations. */
+    resetActivity() {
+        this.lastActivity = Date.now();
+    }
+    // --- Per-project operation serialization ---
+    withProjectLock(root, fn) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            const prev = (_a = this.projectLocks.get(root)) !== null && _a !== void 0 ? _a : Promise.resolve();
+            let release;
+            const next = new Promise((r) => { release = r; });
+            this.projectLocks.set(root, next);
+            yield prev;
+            try {
+                return yield fn();
+            }
+            finally {
+                release();
+                if (this.projectLocks.get(root) === next) {
+                    this.projectLocks.delete(root);
+                }
+            }
+        });
+    }
+    // --- Streaming write operations (IPC) ---
+    addProject(root, conn) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.withProjectLock(root, () => __awaiter(this, void 0, void 0, function* () {
+                var _a;
+                if (!this.vectorDb || !this.metaCache) {
+                    (0, ipc_handler_1.writeDone)(conn, { ok: false, error: "daemon resources not ready" });
+                    return;
+                }
+                const ac = new AbortController();
+                conn.on("close", () => ac.abort());
+                this.vectorDb.pauseMaintenanceLoop();
+                let lastProgressTime = 0;
+                try {
+                    const result = yield (0, syncer_1.initialSync)({
+                        projectRoot: root,
+                        vectorDb: this.vectorDb,
+                        metaCache: this.metaCache,
+                        signal: ac.signal,
+                        onProgress: (info) => {
+                            this.resetActivity();
+                            const now = Date.now();
+                            if (now - lastProgressTime < 100)
+                                return;
+                            lastProgressTime = now;
+                            (0, ipc_handler_1.writeProgress)(conn, {
+                                processed: info.processed,
+                                indexed: info.indexed,
+                                total: info.total,
+                                filePath: info.filePath,
+                            });
+                        },
+                    });
+                    yield this.watchProject(root);
+                    (0, ipc_handler_1.writeDone)(conn, {
+                        ok: true,
+                        processed: result.processed,
+                        indexed: result.indexed,
+                        total: result.total,
+                        failedFiles: result.failedFiles,
+                    });
+                }
+                catch (err) {
+                    const msg = err instanceof Error ? err.message : String(err);
+                    console.error(`[daemon] addProject failed for ${path.basename(root)}:`, msg);
+                    (0, ipc_handler_1.writeDone)(conn, { ok: false, error: msg });
+                }
+                finally {
+                    (_a = this.vectorDb) === null || _a === void 0 ? void 0 : _a.resumeMaintenanceLoop();
+                }
+            }));
+        });
+    }
+    indexProject(root, conn, opts) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.withProjectLock(root, () => __awaiter(this, void 0, void 0, function* () {
+                var _a;
+                if (!this.vectorDb || !this.metaCache) {
+                    (0, ipc_handler_1.writeDone)(conn, { ok: false, error: "daemon resources not ready" });
+                    return;
+                }
+                // Pause the project's batch processor during full index
+                const processor = this.processors.get(root);
+                if (processor) {
+                    yield processor.close();
+                    this.processors.delete(root);
+                }
+                const sub = this.subscriptions.get(root);
+                if (sub) {
+                    yield sub.unsubscribe();
+                    this.subscriptions.delete(root);
+                }
+                const ac = new AbortController();
+                conn.on("close", () => ac.abort());
+                this.vectorDb.pauseMaintenanceLoop();
+                let lastProgressTime = 0;
+                try {
+                    const result = yield (0, syncer_1.initialSync)({
+                        projectRoot: root,
+                        reset: opts.reset,
+                        dryRun: opts.dryRun,
+                        vectorDb: this.vectorDb,
+                        metaCache: this.metaCache,
+                        signal: ac.signal,
+                        onProgress: (info) => {
+                            this.resetActivity();
+                            const now = Date.now();
+                            if (now - lastProgressTime < 100)
+                                return;
+                            lastProgressTime = now;
+                            (0, ipc_handler_1.writeProgress)(conn, {
+                                processed: info.processed,
+                                indexed: info.indexed,
+                                total: info.total,
+                                filePath: info.filePath,
+                            });
+                        },
+                    });
+                    (0, ipc_handler_1.writeDone)(conn, {
+                        ok: true,
+                        processed: result.processed,
+                        indexed: result.indexed,
+                        total: result.total,
+                        failedFiles: result.failedFiles,
+                    });
+                }
+                catch (err) {
+                    const msg = err instanceof Error ? err.message : String(err);
+                    console.error(`[daemon] indexProject failed for ${path.basename(root)}:`, msg);
+                    (0, ipc_handler_1.writeDone)(conn, { ok: false, error: msg });
+                }
+                finally {
+                    (_a = this.vectorDb) === null || _a === void 0 ? void 0 : _a.resumeMaintenanceLoop();
+                    // Re-enable watcher
+                    try {
+                        yield this.watchProject(root);
+                    }
+                    catch (err) {
+                        console.error(`[daemon] Failed to re-watch ${path.basename(root)}:`, err);
+                    }
+                }
+            }));
+        });
+    }
+    removeProject(root, conn) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.withProjectLock(root, () => __awaiter(this, void 0, void 0, function* () {
+                if (!this.vectorDb || !this.metaCache) {
+                    (0, ipc_handler_1.writeDone)(conn, { ok: false, error: "daemon resources not ready" });
+                    return;
+                }
+                try {
+                    yield this.unwatchProject(root);
+                    const rootPrefix = root.endsWith("/") ? root : `${root}/`;
+                    yield this.vectorDb.deletePathsWithPrefix(rootPrefix);
+                    const keys = yield this.metaCache.getKeysWithPrefix(rootPrefix);
+                    for (const key of keys) {
+                        this.metaCache.delete(key);
+                    }
+                    (0, ipc_handler_1.writeDone)(conn, { ok: true });
+                }
+                catch (err) {
+                    const msg = err instanceof Error ? err.message : String(err);
+                    console.error(`[daemon] removeProject failed for ${path.basename(root)}:`, msg);
+                    (0, ipc_handler_1.writeDone)(conn, { ok: false, error: msg });
+                }
+            }));
+        });
+    }
+    summarizeProject(root, conn, opts) {
+        return __awaiter(this, void 0, void 0, function* () {
+            yield this.withProjectLock(root, () => __awaiter(this, void 0, void 0, function* () {
+                var _a;
+                if (!this.vectorDb) {
+                    (0, ipc_handler_1.writeDone)(conn, { ok: false, error: "daemon resources not ready" });
+                    return;
+                }
+                const rootPrefix = (_a = opts.pathPrefix) !== null && _a !== void 0 ? _a : (root.endsWith("/") ? root : `${root}/`);
+                let lastProgressTime = 0;
+                try {
+                    const result = yield (0, syncer_1.generateSummaries)(this.vectorDb, rootPrefix, (done, total) => {
+                        this.resetActivity();
+                        const now = Date.now();
+                        if (now - lastProgressTime < 100)
+                            return;
+                        lastProgressTime = now;
+                        (0, ipc_handler_1.writeProgress)(conn, { summarized: done, total });
+                    }, opts.limit);
+                    (0, ipc_handler_1.writeDone)(conn, {
+                        ok: true,
+                        summarized: result.summarized,
+                        remaining: result.remaining,
+                    });
+                }
+                catch (err) {
+                    const msg = err instanceof Error ? err.message : String(err);
+                    console.error(`[daemon] summarizeProject failed for ${path.basename(root)}:`, msg);
+                    (0, ipc_handler_1.writeDone)(conn, { ok: false, error: msg });
+                }
+            }));
+        });
+    }
     shutdown() {
         return __awaiter(this, void 0, void 0, function* () {
             var _a, _b, _c;

package/dist/lib/daemon/ipc-handler.js

@@ -9,8 +9,33 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
+exports.writeProgress = writeProgress;
+exports.writeDone = writeDone;
 exports.handleCommand = handleCommand;
-function handleCommand(daemon, cmd) {
+/**
+ * Write a streaming progress line to the IPC connection.
+ */
+function writeProgress(conn, data) {
+    if (!conn.writable)
+        return;
+    conn.write(`${JSON.stringify(Object.assign({ type: "progress" }, data))}\n`);
+}
+/**
+ * Write the final streaming done line and end the connection.
+ */
+function writeDone(conn, data) {
+    if (!conn.writable)
+        return;
+    conn.write(`${JSON.stringify(Object.assign({ type: "done" }, data))}\n`);
+    conn.end();
+}
+/**
+ * Handle a single IPC command.
+ *
+ * Returns a DaemonResponse for simple commands (caller writes + closes).
+ * Returns null for streaming commands (handler manages connection lifecycle).
+ */
+function handleCommand(daemon, cmd, conn) {
     return __awaiter(this, void 0, void 0, function* () {
         try {
             switch (cmd.cmd) {
@@ -41,6 +66,41 @@ function handleCommand(daemon, cmd) {
                     // Respond before shutting down so the client gets the response
                     setImmediate(() => daemon.shutdown());
                     return { ok: true };
+                // --- Streaming commands (daemon manages connection) ---
+                case "add": {
+                    const root = String(cmd.root || "");
+                    if (!root)
+                        return { ok: false, error: "missing root" };
+                    daemon.addProject(root, conn);
+                    return null;
+                }
+                case "index": {
+                    const root = String(cmd.root || "");
+                    if (!root)
+                        return { ok: false, error: "missing root" };
+                    daemon.indexProject(root, conn, {
+                        reset: !!cmd.reset,
+                        dryRun: !!cmd.dryRun,
+                    });
+                    return null;
+                }
+                case "remove": {
+                    const root = String(cmd.root || "");
+                    if (!root)
+                        return { ok: false, error: "missing root" };
+                    daemon.removeProject(root, conn);
+                    return null;
+                }
+                case "summarize": {
+                    const root = String(cmd.root || "");
+                    if (!root)
+                        return { ok: false, error: "missing root" };
+                    daemon.summarizeProject(root, conn, {
+                        limit: typeof cmd.limit === "number" ? cmd.limit : undefined,
+                        pathPrefix: typeof cmd.pathPrefix === "string" ? cmd.pathPrefix : undefined,
+                    });
+                    return null;
+                }
                 default:
                     return { ok: false, error: `unknown command: ${cmd.cmd}` };
             }

package/dist/lib/index/batch-processor.js

@@ -49,7 +49,6 @@ const config_1 = require("../../config");
 const cache_check_1 = require("../utils/cache-check");
 const file_utils_1 = require("../utils/file-utils");
 const logger_1 = require("../utils/logger");
-const lock_1 = require("../utils/lock");
 const pool_1 = require("../workers/pool");
 const watcher_batch_1 = require("./watcher-batch");
 const DEBOUNCE_MS = 2000;
@@ -61,12 +60,10 @@ class ProjectBatchProcessor {
         this.debounceTimer = null;
         this.processing = false;
         this.closed = false;
-        this.consecutiveLockFailures = 0;
         this.currentBatchAc = null;
         this.projectRoot = opts.projectRoot;
         this.vectorDb = opts.vectorDb;
         this.metaCache = opts.metaCache;
-        this.dataDir = opts.dataDir;
         this.onReindex = opts.onReindex;
         this.onActivity = opts.onActivity;
         this.wtag = `watch:${path.basename(opts.projectRoot)}`;
@@ -122,127 +119,114 @@ class ProjectBatchProcessor {
             const start = Date.now();
             let reindexed = 0;
             try {
-                const lock = yield (0, lock_1.acquireWriterLockWithRetry)(this.dataDir, {
-                    maxRetries: 3,
-                    retryDelayMs: 500,
-                });
-                try {
-                    const pool = (0, pool_1.getWorkerPool)();
-                    const deletes = [];
-                    const vectors = [];
-                    const metaUpdates = new Map();
-                    const metaDeletes = [];
-                    const attempted = new Set();
-                    for (const [absPath, event] of batch) {
-                        if (batchAc.signal.aborted)
-                            break;
-                        attempted.add(absPath);
-                        if (event === "unlink") {
-                            deletes.push(absPath);
-                            metaDeletes.push(absPath);
-                            reindexed++;
+                // No lock needed — daemon is the single writer to LanceDB/MetaCache
+                const pool = (0, pool_1.getWorkerPool)();
+                const deletes = [];
+                const vectors = [];
+                const metaUpdates = new Map();
+                const metaDeletes = [];
+                const attempted = new Set();
+                for (const [absPath, event] of batch) {
+                    if (batchAc.signal.aborted)
+                        break;
+                    attempted.add(absPath);
+                    if (event === "unlink") {
+                        deletes.push(absPath);
+                        metaDeletes.push(absPath);
+                        reindexed++;
+                        continue;
+                    }
+                    // change or add
+                    try {
+                        const stats = yield fs.promises.stat(absPath);
+                        if (!(0, file_utils_1.isIndexableFile)(absPath, stats.size))
+                            continue;
+                        const cached = this.metaCache.get(absPath);
+                        if ((0, cache_check_1.isFileCached)(cached, stats)) {
                             continue;
                         }
-                        // change or add
-                        try {
-                            const stats = yield fs.promises.stat(absPath);
-                            if (!(0, file_utils_1.isIndexableFile)(absPath, stats.size))
-                                continue;
-                            const cached = this.metaCache.get(absPath);
-                            if ((0, cache_check_1.isFileCached)(cached, stats)) {
-                                continue;
-                            }
-                            const result = yield pool.processFile({
-                                path: absPath,
-                                absolutePath: absPath,
-                            }, batchAc.signal);
-                            const metaEntry = {
-                                hash: result.hash,
-                                mtimeMs: result.mtimeMs,
-                                size: result.size,
-                            };
-                            if (cached && cached.hash === result.hash) {
-                                metaUpdates.set(absPath, metaEntry);
-                                continue;
-                            }
-                            if (result.shouldDelete) {
-                                deletes.push(absPath);
-                                metaUpdates.set(absPath, metaEntry);
-                                reindexed++;
-                                continue;
-                            }
+                        const result = yield pool.processFile({
+                            path: absPath,
+                            absolutePath: absPath,
+                        }, batchAc.signal);
+                        const metaEntry = {
+                            hash: result.hash,
+                            mtimeMs: result.mtimeMs,
+                            size: result.size,
+                        };
+                        if (cached && cached.hash === result.hash) {
+                            metaUpdates.set(absPath, metaEntry);
+                            continue;
+                        }
+                        if (result.shouldDelete) {
                             deletes.push(absPath);
-                            if (result.vectors.length > 0) {
-                                vectors.push(...result.vectors);
-                            }
                             metaUpdates.set(absPath, metaEntry);
                             reindexed++;
+                            continue;
                         }
-                        catch (err) {
-                            if (batchAc.signal.aborted)
-                                break;
-                            const code = err === null || err === void 0 ? void 0 : err.code;
-                            if (code === "ENOENT") {
-                                deletes.push(absPath);
-                                metaDeletes.push(absPath);
-                                reindexed++;
-                            }
-                            else {
-                                console.error(`[${this.wtag}] Failed to process ${absPath}:`, err);
-                                if (!pool.isHealthy()) {
-                                    console.error(`[${this.wtag}] Worker pool unhealthy, aborting batch`);
-                                    break;
-                                }
-                            }
-                        }
-                    }
-                    // Requeue files that weren't attempted (aborted or pool unhealthy)
-                    for (const [absPath, event] of batch) {
-                        if (!attempted.has(absPath) && !this.pending.has(absPath)) {
-                            this.pending.set(absPath, event);
+                        deletes.push(absPath);
+                        if (result.vectors.length > 0) {
+                            vectors.push(...result.vectors);
                         }
+                        metaUpdates.set(absPath, metaEntry);
+                        reindexed++;
                     }
-                    // Flush to VectorDB: insert first, then delete old (preserving new)
-                    const newIds = vectors.map((v) => v.id);
-                    if (vectors.length > 0) {
-                        yield this.vectorDb.insertBatch(vectors);
-                    }
-                    if (deletes.length > 0) {
-                        if (newIds.length > 0) {
-                            yield this.vectorDb.deletePathsExcludingIds(deletes, newIds);
+                    catch (err) {
+                        if (batchAc.signal.aborted)
+                            break;
+                        const code = err === null || err === void 0 ? void 0 : err.code;
+                        if (code === "ENOENT") {
+                            deletes.push(absPath);
+                            metaDeletes.push(absPath);
+                            reindexed++;
                         }
                         else {
-                            yield this.vectorDb.deletePaths(deletes);
+                            console.error(`[${this.wtag}] Failed to process ${absPath}:`, err);
+                            if (!pool.isHealthy()) {
+                                console.error(`[${this.wtag}] Worker pool unhealthy, aborting batch`);
+                                break;
+                            }
                         }
                     }
-                    // Update MetaCache
-                    for (const [p, entry] of metaUpdates) {
-                        this.metaCache.put(p, entry);
+                }
+                // Requeue files that weren't attempted (aborted or pool unhealthy)
+                for (const [absPath, event] of batch) {
+                    if (!attempted.has(absPath) && !this.pending.has(absPath)) {
+                        this.pending.set(absPath, event);
+                    }
+                }
+                // Flush to VectorDB: insert first, then delete old (preserving new)
+                const newIds = vectors.map((v) => v.id);
+                if (vectors.length > 0) {
+                    yield this.vectorDb.insertBatch(vectors);
+                }
+                if (deletes.length > 0) {
+                    if (newIds.length > 0) {
+                        yield this.vectorDb.deletePathsExcludingIds(deletes, newIds);
                     }
-                    for (const p of metaDeletes) {
-                        this.metaCache.delete(p);
+                    else {
+                        yield this.vectorDb.deletePaths(deletes);
                     }
                 }
-                finally {
-                    yield lock.release();
+                // Update MetaCache
+                for (const [p, entry] of metaUpdates) {
+                    this.metaCache.put(p, entry);
+                }
+                for (const p of metaDeletes) {
+                    this.metaCache.delete(p);
                 }
                 const duration = Date.now() - start;
                 if (reindexed > 0) {
                     (_a = this.onReindex) === null || _a === void 0 ? void 0 : _a.call(this, reindexed, duration);
                 }
                 (0, logger_1.log)(this.wtag, `Batch complete: ${batch.size} files, ${reindexed} reindexed (${(duration / 1000).toFixed(1)}s)`);
-                this.consecutiveLockFailures = 0;
                 for (const absPath of batch.keys()) {
                     this.retryCount.delete(absPath);
                 }
             }
             catch (err) {
-                const isLockError = err instanceof Error && err.message.includes("lock already held");
-                if (isLockError) {
-                    this.consecutiveLockFailures++;
-                }
                 console.error(`[${this.wtag}] Batch processing failed:`, err);
-                const { requeued, dropped, backoffMs } = (0, watcher_batch_1.computeRetryAction)(batch, this.retryCount, MAX_RETRIES, isLockError, this.consecutiveLockFailures, DEBOUNCE_MS);
+                const { requeued, dropped, backoffMs } = (0, watcher_batch_1.computeRetryAction)(batch, this.retryCount, MAX_RETRIES, false, 0, DEBOUNCE_MS);
                 for (const [absPath, event] of requeued) {
                     if (!this.pending.has(absPath)) {
                         this.pending.set(absPath, event);