grepmax 0.26.67 → 0.26.69

package/.claude-plugin/marketplace.json

@@ -10,7 +10,7 @@
       "name": "grepmax",
       "source": "./plugins/grepmax",
       "description": "Semantic code search for Claude Code. Automatically indexes your project and provides intelligent search capabilities.",
-      "version": "0.26.67",
+      "version": "0.26.69",
       "author": {
         "name": "Robert Owens",
         "email": "robowens@me.com"

package/README.md

@@ -240,7 +240,7 @@ gmax status                   # See all projects + watcher status
 
 Adding or indexing a project can start the daemon automatically. MCP acquires primary-project watch leases lazily when an index-reading tool is used; discovery and catalog listing start no background work. It shuts down after 4 hours of inactivity, and hands off to a fresh daemon after 24 hours or once its memory footprint passes 2.5 GB. File-change batches are processed on up to four worker processes by default, added only when work backs up, with one kept free for searches; full-table compaction and new FTS/ANN index builds are disabled by containment. Existing indexes remain readable. Host safety stops persist across restart and block heavy operations.
 
-`gmax watch status` reports failed files, native watcher recovery or polling, dropped-event counts, and the last complete filesystem reconciliation. FSEvents dropped-event warnings keep the native stream attached and preserve the accompanying events. A reconciliation covers the gap; repeated warnings coalesce into one later scan, with a 30-second scan interval. Recovery stays visible until a complete scan covers the latest gap. Other backend errors still trigger subscription recovery; repeated terminal failures fall back to five-minute polling, where recent edits can lag. Search and MCP responses retain health warnings even after the file queue drains. A complete reconciliation means the filesystem scan completed; queued embedding work can still be pending.
+`gmax watch status` reports failed files, native watcher recovery or polling, dropped-event counts, and the last complete filesystem reconciliation. FSEvents dropped-event warnings keep the native stream attached and preserve the accompanying events. A reconciliation covers the gap; repeated warnings coalesce into one later scan, with a 30-second scan interval. Recovery stays visible until a complete scan covers the latest gap. Other backend errors still trigger subscription recovery; repeated terminal failures fall back to five-minute polling, where recent edits can lag. Search and MCP responses retain health warnings even after the file queue drains. On macOS, gmax also forwards up to eight existing policy-excluded directories to the native event stream, prioritizing dependency, build and development output. Discovery is bounded; remaining paths retain glob filtering and file-policy checks. Policy edits refresh exclusions, and a policy change during subscription setup is checked before keeping a directory exclusion. Live edit batches can run during reconciliation instead of waiting for the full filesystem walk. Whole-project reindex and removal still quiesce the watcher and drain its work before changing the project. A complete reconciliation means the filesystem scan completed; queued embedding work can still be pending.
 
 Live filesystem edits take priority over background catchup and cleanup. Background batches dispatch one file at a time, with at most four catchup files or 50 cleanup paths. They yield between files when a live edit arrives or the two-second dispatch window is reached; an active file or store commit finishes before yielding. Catchup and cleanup alternate, and one background file is admitted after three consecutive live batches so cleanup continues during sustained editing. Existing retry backoff and worker limits remain in effect.
 

package/dist/lib/daemon/daemon.js

@@ -192,7 +192,14 @@ class Daemon {
             evictSearcher: (root) => {
                 this.searchers.delete(root);
             },
-            runProjectOperation: (root, name, signal, fn) => this.withProjectLock(root, signal, () => this.runSharedOperation(name, signal, fn)),
+            runProjectOperation: (root, name, signal, fn) => {
+                // A filesystem walk can take minutes. Keep it lifecycle-tracked and
+                // admitted, but let live batches hold the project's write lock while
+                // the scan yields. Project removal/reindex quiesces the watcher first.
+                if (name === "watch-catchup")
+                    return this.runSharedOperation(name, signal, fn);
+                return this.withProjectLock(root, signal, () => this.runSharedOperation(name, signal, fn));
+            },
         });
         this.projectMutex = new keyed_mutex_1.KeyedMutex();
         // Tests construct daemons against the real HOME; keep their leases in memory.
@@ -1237,7 +1244,15 @@ class Daemon {
             this.vectorDb.pauseMaintenanceLoop();
             const stopHeartbeat = (0, ipc_handler_1.startHeartbeat)(conn);
             let lastProgressTime = 0;
+            const wasWatching = this.processors.has(root);
+            let watcherQuiesced = false;
             try {
+                // Re-adding a watched project is also a full sync. A catchup scan no
+                // longer holds the write lock, so explicitly drain it before syncing.
+                if (wasWatching) {
+                    yield this.unwatchProjectWithinOperation(root);
+                    watcherQuiesced = true;
+                }
                 const result = yield (0, syncer_1.initialSync)({
                     projectRoot: root,
                     vectorDb: this.vectorDb,
@@ -1287,6 +1302,7 @@ class Daemon {
                     });
                 }
                 yield this.watchProjectWithinOperation(root);
+                watcherQuiesced = false;
                 (0, ipc_handler_1.writeDone)(conn, {
                     ok: true,
                     processed: result.processed,
@@ -1316,6 +1332,15 @@ class Daemon {
             }
             finally {
                 stopHeartbeat();
+                if (watcherQuiesced && !this.shuttingDown) {
+                    try {
+                        this.assertHeavyOperationAdmission("watch");
+                        yield this.watchProjectWithinOperation(root);
+                    }
+                    catch (error) {
+                        console.error("[daemon] Could not restore watcher after add:", error);
+                    }
+                }
                 (_a = this.vectorDb) === null || _a === void 0 ? void 0 : _a.resumeMaintenanceLoop();
             }
         });

package/dist/lib/daemon/watcher-manager.js

@@ -51,7 +51,6 @@ var __asyncValues = (this && this.__asyncValues) || function (o) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.WatcherManager = void 0;
 const path = __importStar(require("node:path"));
-const watcher = __importStar(require("@parcel/watcher"));
 const batch_processor_1 = require("../index/batch-processor");
 const cache_coherence_1 = require("../index/cache-coherence");
 const file_policy_1 = require("../index/file-policy");
@@ -327,7 +326,7 @@ class WatcherManager {
                 catch (_c) { }
                 this.deps.subscriptions.delete(root);
             }
-            const sub = yield watcher.subscribe(root, (err, events) => {
+            const { subscription: sub } = yield (0, watcher_ignore_1.subscribeWithNativeExclusions)(root, (err, events) => {
                 var _a, _b;
                 var _c, _d;
                 if (this.deps.processors.get(root) !== processor ||
@@ -360,12 +359,7 @@ class WatcherManager {
                     processor.handleFileEvent(event.type === "delete" ? "unlink" : "change", event.path);
                 }
                 this.deps.touchActivity();
-            }, {
-                ignore: [
-                    ...watcher_1.WATCHER_IGNORE_GLOBS,
-                    ...(yield (0, watcher_ignore_1.readProjectWatcherIgnores)(root)),
-                ],
-            });
+            }, watcher_1.WATCHER_IGNORE_GLOBS);
             if (this.deps.processors.get(root) !== processor ||
                 this.watcherGenerations.get(root) !== generation) {
                 yield sub.unsubscribe();
@@ -653,6 +647,7 @@ class WatcherManager {
                             // Fast path: if only mtime changed but size is identical and we have a hash,
                             // just verify the hash in-process instead of sending to a worker.
                             if ((cached === null || cached === void 0 ? void 0 : cached.hash) && cached.size === stats.size) {
+                                const observed = Object.assign({}, cached);
                                 const snapshot = yield (0, file_utils_1.readFileSnapshot)(absPath, {
                                     projectRoot: root,
                                 });
@@ -664,11 +659,21 @@ class WatcherManager {
                                     continue;
                                 }
                                 const hash = (0, file_utils_1.computeContentHash)(snapshot.buffer, absPath);
-                                if (hash === cached.hash) {
+                                if (hash === observed.hash) {
                                     // Content unchanged — update mtime in cache and skip worker
                                     if (signal.aborted)
                                         return false;
-                                    metaCache.put(absPath, Object.assign(Object.assign({}, cached), { mtimeMs: stats.mtimeMs }));
+                                    // A live batch can finish while this read yields. Never replace
+                                    // newer metadata or resurrect a cache entry it removed.
+                                    const current = metaCache.get(absPath);
+                                    if (current &&
+                                        current.hash === observed.hash &&
+                                        current.mtimeMs === observed.mtimeMs &&
+                                        current.size === observed.size &&
+                                        current.hashVersion === observed.hashVersion &&
+                                        current.hasVectors === observed.hasVectors) {
+                                        metaCache.put(absPath, Object.assign(Object.assign({}, observed), { mtimeMs: stats.mtimeMs }));
+                                    }
                                     skipped++;
                                     continue;
                                 }

package/dist/lib/index/watcher-ignore.js

@@ -41,15 +41,27 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.nativeProjectIgnoreGlobs = nativeProjectIgnoreGlobs;
 exports.readProjectWatcherIgnores = readProjectWatcherIgnores;
+exports.readNativeWatcherIgnores = readNativeWatcherIgnores;
+exports.watcherIgnoreIdentity = watcherIgnoreIdentity;
+exports.subscribeWithNativeExclusions = subscribeWithNativeExclusions;
 const fs = __importStar(require("node:fs/promises"));
 const path = __importStar(require("node:path"));
+const watcher = __importStar(require("@parcel/watcher"));
 const ignore_1 = __importDefault(require("ignore"));
+const file_policy_1 = require("./file-policy");
 /** Only the common positive gitignore/glob subset is safe to push into Parcel.
  * Negations, escapes and unsupported syntax stay with ProjectFilePolicy, so a
  * native ignore can never hide a file that an explicit re-inclusion needs. */
@@ -102,3 +114,174 @@ function readProjectWatcherIgnores(root) {
         }
     });
 }
+const NATIVE_EXCLUSION_LIMIT = 8;
+const DISCOVERY_DIRECTORY_LIMIT = 512;
+const DISCOVERY_ENTRY_LIMIT = 16384;
+const DISCOVERY_DEPTH_LIMIT = 4;
+const DISCOVERY_TIME_MS = 1000;
+/** FSEvents only receives Parcel's literal paths, not its glob filters. Select
+ * bounded, existing directories that file policy already excludes. Never hide
+ * a source ancestor merely because some descendants are ignored. */
+function readNativeWatcherIgnores(root, baseGlobs) {
+    return __awaiter(this, void 0, void 0, function* () {
+        var _a, e_1, _b, _c;
+        const policy = new file_policy_1.ProjectFilePolicy(root);
+        const candidates = [];
+        const discoveryPriority = (directory, depth) => {
+            const parts = path.relative(root, directory).split(path.sep);
+            // Package containers often hold nested build roots. Inspect those before
+            // descending authored source trees, which can exhaust the entry budget.
+            const container = parts.some((part) => ["packages", "apps", "services", "ios", "android", "macos"].includes(part));
+            const source = parts.some((part) => ["src", "source", "Sources", "tests", "test", "Tests"].includes(part));
+            return (container ? 1000 : 0) - (source ? 2000 : 0) - depth;
+        };
+        const pending = [{ directory: root, depth: 0 }];
+        const deadline = performance.now() + DISCOVERY_TIME_MS;
+        let entries = 0;
+        let visited = 0;
+        for (let cursor = 0; cursor < pending.length; cursor++) {
+            if (visited++ >= DISCOVERY_DIRECTORY_LIMIT || performance.now() > deadline)
+                break;
+            let next = cursor;
+            for (let i = cursor + 1; i < pending.length; i++) {
+                if (discoveryPriority(pending[i].directory, pending[i].depth) >
+                    discoveryPriority(pending[next].directory, pending[next].depth))
+                    next = i;
+            }
+            [pending[cursor], pending[next]] = [pending[next], pending[cursor]];
+            const { directory, depth } = pending[cursor];
+            try {
+                const dir = yield fs.opendir(directory);
+                try {
+                    for (var _d = true, dir_1 = (e_1 = void 0, __asyncValues(dir)), dir_1_1; dir_1_1 = yield dir_1.next(), _a = dir_1_1.done, !_a; _d = true) {
+                        _c = dir_1_1.value;
+                        _d = false;
+                        const entry = _c;
+                        if (++entries > DISCOVERY_ENTRY_LIMIT || performance.now() > deadline)
+                            break;
+                        if (!entry.isDirectory() || entry.isSymbolicLink())
+                            continue;
+                        const absolute = path.join(directory, entry.name);
+                        const classified = yield policy.classifyDirectory(absolute);
+                        if (classified.status === "excluded") {
+                            if (!["default ignore policy", "project ignore policy"].includes(classified.reason))
+                                continue;
+                            const relative = path
+                                .relative(root, absolute)
+                                .split(path.sep)
+                                .join("/");
+                            // Public Parcel options distinguish literals from globs. Skip names
+                            // it could interpret as a pattern; existing glob filtering remains.
+                            if (!/^[A-Za-z0-9_. /-]+$/.test(relative))
+                                continue;
+                            const priority = entry.name === ".git" || entry.name === "node_modules"
+                                ? depth === 0
+                                    ? 1000
+                                    : 500
+                                : entry.name === ".dev"
+                                    ? depth === 0
+                                        ? 950
+                                        : 550
+                                    : entry.name === ".build" || entry.name === "DerivedData"
+                                        ? 800
+                                        : entry.name === ".next"
+                                            ? 700
+                                            : depth === 0
+                                                ? 600
+                                                : 200;
+                            candidates.push({ relative, score: priority - depth * 10 });
+                        }
+                        else if (classified.status === "traverse" &&
+                            depth < DISCOVERY_DEPTH_LIMIT &&
+                            pending.length < DISCOVERY_DIRECTORY_LIMIT) {
+                            pending.push({ directory: absolute, depth: depth + 1 });
+                        }
+                    }
+                }
+                catch (e_1_1) { e_1 = { error: e_1_1 }; }
+                finally {
+                    try {
+                        if (!_d && !_a && (_b = dir_1.return)) yield _b.call(dir_1);
+                    }
+                    finally { if (e_1) throw e_1.error; }
+                }
+            }
+            catch (_e) {
+                // Failed/partial discovery only loses an optimization. Existing glob
+                // delivery and authoritative file-policy classification remain in force.
+            }
+            if (entries >= DISCOVERY_ENTRY_LIMIT)
+                break;
+        }
+        candidates.sort((a, b) => b.score - a.score || a.relative.localeCompare(b.relative));
+        const literalPaths = candidates
+            .slice(0, NATIVE_EXCLUSION_LIMIT)
+            .map((c) => c.relative);
+        return [...literalPaths, ...(yield nativeWatcherFilters(root, baseGlobs))];
+    });
+}
+function nativeWatcherFilters(root, baseGlobs) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const projectGlobs = yield readProjectWatcherIgnores(root);
+        return [...baseGlobs, ...projectGlobs].map((pattern) => {
+            if (pattern.includes("*"))
+                return pattern;
+            // Keep file exclusions and excess directory literals out of ignorePaths:
+            // passing more than eight silently defeats macOS stream exclusion.
+            const escaped = pattern.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+            return new RegExp(`^${escaped}(?:/|$)`);
+        });
+    });
+}
+function verifyNativeWatcherIgnores(root, baseGlobs, ignores) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const policy = new file_policy_1.ProjectFilePolicy(root);
+        const literals = [];
+        // Recheck the selected paths, rather than repeating timed discovery. A
+        // different partial scan must not cause subscription churn or disable the
+        // exclusions we already proved safe. New candidates can wait for refresh.
+        for (const pattern of ignores) {
+            if (typeof pattern !== "string" || pattern.includes("*"))
+                continue;
+            const classified = yield policy.classifyDirectory(path.join(root, pattern));
+            if (classified.status === "excluded" &&
+                ["default ignore policy", "project ignore policy"].includes(classified.reason))
+                literals.push(pattern);
+        }
+        return [...literals, ...(yield nativeWatcherFilters(root, baseGlobs))];
+    });
+}
+function watcherIgnoreIdentity(ignores) {
+    return JSON.stringify(ignores.map((pattern) => String(pattern)));
+}
+/** A policy edit during subscribe has no event listener yet. Check again after
+ * attaching so an old directory exclusion cannot permanently hide newly
+ * re-included source. Sustained policy churn falls back to immutable globs. */
+function subscribeWithNativeExclusions(root, callback, baseGlobs, initialIgnores) {
+    return __awaiter(this, void 0, void 0, function* () {
+        let ignores = initialIgnores !== null && initialIgnores !== void 0 ? initialIgnores : (yield readNativeWatcherIgnores(root, baseGlobs));
+        for (let attempt = 0; attempt < 2; attempt++) {
+            const subscription = yield watcher.subscribe(root, callback, {
+                ignore: ignores,
+            });
+            let verified;
+            try {
+                verified = yield verifyNativeWatcherIgnores(root, baseGlobs, ignores);
+            }
+            catch (error) {
+                yield subscription.unsubscribe();
+                throw error;
+            }
+            const policy = watcherIgnoreIdentity(ignores);
+            if (policy === watcherIgnoreIdentity(verified))
+                return { subscription, policy };
+            yield subscription.unsubscribe();
+            ignores = verified;
+        }
+        const fallback = [...baseGlobs];
+        return {
+            subscription: yield watcher.subscribe(root, callback, { ignore: fallback }),
+            policy: watcherIgnoreIdentity(fallback),
+        };
+    });
+}

package/dist/lib/index/watcher.js

@@ -52,7 +52,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.WATCHER_IGNORE_GLOBS = void 0;
 exports.startWatcher = startWatcher;
 const path = __importStar(require("node:path"));
-const watcher = __importStar(require("@parcel/watcher"));
 const batch_processor_1 = require("./batch-processor");
 const cache_coherence_1 = require("./cache-coherence");
 const file_policy_1 = require("./file-policy");
@@ -135,11 +134,8 @@ function startWatcher(opts) {
             const update = subscriptionUpdate
                 .catch(() => { })
                 .then(() => __awaiter(this, void 0, void 0, function* () {
-                const ignores = [
-                    ...exports.WATCHER_IGNORE_GLOBS,
-                    ...(yield (0, watcher_ignore_1.readProjectWatcherIgnores)(projectRoot)),
-                ];
-                const policy = JSON.stringify(ignores);
+                const ignores = yield (0, watcher_ignore_1.readNativeWatcherIgnores)(projectRoot, exports.WATCHER_IGNORE_GLOBS);
+                const policy = (0, watcher_ignore_1.watcherIgnoreIdentity)(ignores);
                 if (closing ||
                     (subscription && !subscriptionFailed && subscriptionPolicy === policy))
                     return;
@@ -149,7 +145,7 @@ function startWatcher(opts) {
                 subscription = null;
                 if (closing)
                     return;
-                const next = yield watcher.subscribe(projectRoot, (err, events) => {
+                const { subscription: next, policy: attachedPolicy } = yield (0, watcher_ignore_1.subscribeWithNativeExclusions)(projectRoot, (err, events) => {
                     var _a;
                     if (closing || generation !== subscriptionGeneration)
                         return;
@@ -168,13 +164,13 @@ function startWatcher(opts) {
                     }
                     for (const event of events)
                         processor.handleFileEvent(event.type === "delete" ? "unlink" : "change", event.path);
-                }, { ignore: ignores });
+                }, exports.WATCHER_IGNORE_GLOBS, ignores);
                 if (closing) {
                     yield next.unsubscribe();
                     return;
                 }
                 subscription = next;
-                subscriptionPolicy = policy;
+                subscriptionPolicy = attachedPolicy;
             }));
             subscriptionUpdate = update;
             return update;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "grepmax",
-  "version": "0.26.67",
+  "version": "0.26.69",
   "author": "Robert Owens <78518764+reowens@users.noreply.github.com>",
   "homepage": "https://github.com/reowens/grepmax",
   "bugs": {

package/plugins/grepmax/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "grepmax",
-  "version": "0.26.67",
+  "version": "0.26.69",
   "description": "Semantic code search for Claude Code. Automatically indexes your project and provides intelligent search capabilities.",
   "author": {
     "name": "Robert Owens",