grepmax 0.26.5 → 0.26.7

package/.claude-plugin/marketplace.json

@@ -10,7 +10,7 @@
       "name": "grepmax",
       "source": "./plugins/grepmax",
       "description": "Semantic code search for Claude Code. Automatically indexes your project and provides intelligent search capabilities.",
-      "version": "0.26.5",
+      "version": "0.26.7",
       "author": {
         "name": "Robert Owens",
         "email": "robowens@me.com"

package/README.md

@@ -124,7 +124,7 @@ Plugins auto-update when you run `npm install -g grepmax@latest` — no need to
 
 ### MCP Server
 
-`gmax mcp` starts a stdio-based MCP server for clients that support MCP but can't run shell commands (Cursor, Windsurf, custom agents).
+`gmax mcp` starts a stdio-based MCP server for clients that support MCP but can't run shell commands (Cursor, Windsurf, custom agents). Search-backed tools use the singleton daemon when available, avoiding a separate embedding worker per MCP session, and fall back to an in-process worker only when the daemon is unavailable or does not support the request.
 
 | Tool | Description |
 | --- | --- |
@@ -215,7 +215,7 @@ gmax watch stop               # Stop daemon
 gmax status                   # See all projects + watcher status
 ```
 
-The daemon auto-starts when you run `gmax add`, `gmax index`, `gmax remove`, or `gmax summarize`. It shuts down after 30 minutes of inactivity.
+The daemon auto-starts when you run `gmax add`, `gmax index`, `gmax remove`, `gmax summarize`, or `gmax mcp`. It shuts down after 30 minutes of inactivity. File-change batches are processed concurrently while reserving worker capacity for searches; LanceDB compaction runs after writes rather than on every maintenance tick.
 
 ## Local LLM (optional)
 
@@ -374,9 +374,11 @@ fixtures/
 | --- | --- | --- |
 | `GMAX_EMBED_MODE` | Force `cpu` or `gpu` | Auto-detect |
 | `GMAX_WORKER_THREADS` | Worker threads for embedding | 50% of cores |
+| `GMAX_WORKER_RSS_RECYCLE_MB` | Recycle workers that remain above this RSS; `0` disables the check | `1536` |
 | `GMAX_DEBUG` | Debug logging | Off |
 | `GMAX_SUMMARIZER` | Enable summarizer auto-start (`1`) | Off |
 | `GMAX_RERANK` | Enable ColBERT rerank (`1`) — off by default since v0.17.1 ([why](docs/known-limitations.md)) | Off |
+| `GMAX_ANN` | Experimental IVF_FLAT vector index (`1`); leave off unless validating recall on your corpus | Off |
 
 ## Troubleshooting
 
@@ -389,6 +391,8 @@ gmax index --reset            # Full reindex from scratch
 gmax watch stop && gmax watch --daemon -b  # Restart daemon
 ```
 
+`gmax doctor` reports ANN index state. `ANN: vector index not built` is normal with the default exact-search configuration.
+
 ## Contributing
 
 See [CLAUDE.md](CLAUDE.md) for development setup, commands, and architecture details.

package/dist/commands/doctor.js

@@ -88,7 +88,7 @@ exports.doctor = new commander_1.Command("doctor")
     .option("--fix", "Auto-fix detected issues (compact, prune, remove stale locks)", false)
     .option("--agent", "Compact output for AI agents", false)
     .action((opts) => __awaiter(void 0, void 0, void 0, function* () {
-    var _a, _b, _c, _d, _e, _f, _g, _h;
+    var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m;
     if (!opts.agent)
         console.log("gmax Doctor\n");
     const root = config_1.PATHS.globalRoot;
@@ -228,6 +228,11 @@ exports.doctor = new commander_1.Command("doctor")
         const logicalSize = tableStats.totalBytes;
         const { numFragments, numSmallFragments } = tableStats.fragmentStats;
         const versions = yield table.listVersions();
+        const indices = yield table.listIndices();
+        const vectorIndex = indices.find((index) => index.name === "vector_idx" || index.columns.includes("vector"));
+        const vectorIndexStats = vectorIndex
+            ? yield table.indexStats(vectorIndex.name).catch(() => undefined)
+            : undefined;
         // Lock status
         const lockPath = path.join(config_1.PATHS.globalRoot, "LOCK");
         let lockStatus = "none";
@@ -273,7 +278,7 @@ exports.doctor = new commander_1.Command("doctor")
                         ? "LOW"
                         : "ok";
         }
-        catch (_j) { }
+        catch (_o) { }
         const staleChunkerProjects = projects.filter((p) => {
             var _a;
             return p.status === "indexed" &&
@@ -301,6 +306,10 @@ exports.doctor = new commander_1.Command("doctor")
                 `legacy_embedding=${legacyEmbeddingCount}`,
                 `schema_dim=${physicalDim !== null && physicalDim !== void 0 ? physicalDim : "none"}`,
                 `schema_dim_ok=${schemaGap ? "false" : "true"}`,
+                `ann_index=${vectorIndex ? "present" : "absent"}`,
+                `ann_indexed=${(_f = vectorIndexStats === null || vectorIndexStats === void 0 ? void 0 : vectorIndexStats.numIndexedRows) !== null && _f !== void 0 ? _f : 0}`,
+                `ann_unindexed=${(_g = vectorIndexStats === null || vectorIndexStats === void 0 ? void 0 : vectorIndexStats.numUnindexedRows) !== null && _g !== void 0 ? _g : 0}`,
+                `ann_metric=${(_h = vectorIndexStats === null || vectorIndexStats === void 0 ? void 0 : vectorIndexStats.distanceType) !== null && _h !== void 0 ? _h : "none"}`,
             ];
             console.log(fields.join("\t"));
             if (schemaGap) {
@@ -354,6 +363,12 @@ exports.doctor = new commander_1.Command("doctor")
             else if (physicalDim) {
                 console.log(`ok  Schema: vector table is ${physicalDim}d`);
             }
+            if (vectorIndexStats) {
+                console.log(`ok  ANN: ${vectorIndexStats.numIndexedRows.toLocaleString()} indexed, ${vectorIndexStats.numUnindexedRows.toLocaleString()} unindexed, metric=${(_j = vectorIndexStats.distanceType) !== null && _j !== void 0 ? _j : "unknown"}`);
+            }
+            else {
+                console.log("INFO  ANN: vector index not built (set GMAX_ANN=1)");
+            }
             // Disk space
             if (diskLevel !== "ok") {
                 console.log(`WARN  Disk: ${formatSize(availBytes)} available (${diskLevel})`);
@@ -468,7 +483,7 @@ exports.doctor = new commander_1.Command("doctor")
                         yield mc.close();
                     }
                 }
-                catch (_k) { }
+                catch (_p) { }
             }
         }
         // --fix auto-remediation
@@ -515,7 +530,7 @@ exports.doctor = new commander_1.Command("doctor")
                         const done = yield sendStreamingCommand({ cmd: "index", root: p.root, reset: true }, () => { }).catch((e) => ({ ok: false, error: String(e) }));
                         if (done.ok &&
                             (!("degraded" in done) || done.degraded !== true)) {
-                            const chunks = (_f = done.indexed) !== null && _f !== void 0 ? _f : 0;
+                            const chunks = (_k = done.indexed) !== null && _k !== void 0 ? _k : 0;
                             if (opts.agent) {
                                 console.log(`stale_chunker_reindexed\tname=${name}\tchunks=${chunks}`);
                             }
@@ -527,7 +542,7 @@ exports.doctor = new commander_1.Command("doctor")
                         else if (!opts.agent) {
                             const degraded = "degraded" in done && done.degraded === true;
                             const reason = degraded
-                                ? `${(_g = ("scanErrors" in done ? done.scanErrors : 0)) !== null && _g !== void 0 ? _g : 0} scan errors, ${(_h = ("failedFiles" in done ? done.failedFiles : 0)) !== null && _h !== void 0 ? _h : 0} file failures`
+                                ? `${(_l = ("scanErrors" in done ? done.scanErrors : 0)) !== null && _l !== void 0 ? _l : 0} scan errors, ${(_m = ("failedFiles" in done ? done.failedFiles : 0)) !== null && _m !== void 0 ? _m : 0} file failures`
                                 : done.error;
                             console.log(`FAIL  ${name}: reindex failed (${reason})`);
                         }

package/dist/commands/mcp.js

@@ -53,6 +53,7 @@ exports.mcp = void 0;
 exports.toStringArray = toStringArray;
 exports.ok = ok;
 exports.err = err;
+exports.shouldFallbackMcpDaemonSearch = shouldFallbackMcpDaemonSearch;
 exports.isExplicitCrossProjectSearch = isExplicitCrossProjectSearch;
 exports.resolveMcpProject = resolveMcpProject;
 exports.containMcpTarget = containMcpTarget;
@@ -84,6 +85,7 @@ const annotator_1 = require("../lib/skeleton/annotator");
 const retriever_1 = require("../lib/skeleton/retriever");
 const skeletonizer_1 = require("../lib/skeleton/skeletonizer");
 const symbol_extractor_1 = require("../lib/skeleton/symbol-extractor");
+const ann_config_1 = require("../lib/store/ann-config");
 const meta_cache_1 = require("../lib/store/meta-cache");
 const vector_db_1 = require("../lib/store/vector-db");
 const cross_project_1 = require("../lib/utils/cross-project");
@@ -121,6 +123,15 @@ function ok(text) {
 function err(text) {
     return { content: [{ type: "text", text }], isError: true };
 }
+function shouldFallbackMcpDaemonSearch(error) {
+    const message = String(error !== null && error !== void 0 ? error : "").toLowerCase();
+    return (message === "enoent" ||
+        message === "econnrefused" ||
+        message.includes("project not watched") ||
+        message.includes("daemon not ready") ||
+        message.includes("oversize") ||
+        message.includes("unknown command"));
+}
 function isExplicitCrossProjectSearch(args, isSearchAll = false) {
     return (isSearchAll ||
         args.scope === "all" ||
@@ -339,6 +350,29 @@ exports.mcp = new commander_1.Command("mcp")
             _searcher = new searcher_1.Searcher(getVectorDb());
         return _searcher;
     }
+    function daemonSearch(args) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            const { sendDaemonCommand } = yield Promise.resolve().then(() => __importStar(require("../lib/utils/daemon-client")));
+            const response = yield sendDaemonCommand(Object.assign({ cmd: ((_a = args.filters) === null || _a === void 0 ? void 0 : _a.projectRoots) ? "search-v2" : "search" }, args), { timeoutMs: 60000 });
+            if (!response.ok) {
+                if (shouldFallbackMcpDaemonSearch(response.error))
+                    return null;
+                const detail = [response.error, response.hint]
+                    .filter((value) => typeof value === "string" && value)
+                    .join(": ");
+                throw new Error(detail || "daemon search failed");
+            }
+            return {
+                data: Array.isArray(response.data)
+                    ? response.data
+                    : [],
+                warnings: Array.isArray(response.warnings)
+                    ? response.warnings.filter((warning) => typeof warning === "string")
+                    : undefined,
+            };
+        });
+    }
     function getSkeletonizer() {
         return __awaiter(this, void 0, void 0, function* () {
             if (!_skeletonizer) {
@@ -394,7 +428,7 @@ exports.mcp = new commander_1.Command("mcp")
     // --- Tool handlers ---
     function handleSemanticSearch(args_1) {
         return __awaiter(this, arguments, void 0, function* (args, isSearchAll = false) {
-            var _a, _b;
+            var _a, _b, _c;
             const query = String(args.query || "");
             if (!query)
                 return err("Missing required parameter: query");
@@ -482,7 +516,6 @@ exports.mcp = new commander_1.Command("mcp")
                     ? new Set((_a = filters.projectRoots) !== null && _a !== void 0 ? _a : [])
                     : new Set([resolvedRoot]);
                 (0, embedding_status_1.assertEmbeddingSearchCompatible)((0, project_registry_1.listProjects)().filter((project) => selectedRoots.has(project.root)), (0, index_config_1.readGlobalConfig)());
-                const searcher = getSearcher();
                 // Aider-style seeding: the agent passes its open files / discussed
                 // symbols; the searcher biases candidate generation toward them.
                 const parseSeedList = (v) => {
@@ -500,9 +533,15 @@ exports.mcp = new commander_1.Command("mcp")
                 const seeds = seedFiles || seedSymbols
                     ? { files: seedFiles, symbols: seedSymbols }
                     : undefined;
-                const result = yield searcher.search(query, limit, { rerank: process.env.GMAX_RERANK === "1", seeds }, Object.keys(filters).length > 0 ? filters : undefined, pathPrefix);
+                const effectiveFilters = Object.keys(filters).length > 0 ? filters : undefined;
+                const searchOptions = {
+                    rerank: process.env.GMAX_RERANK === "1",
+                    seeds,
+                };
+                const result = (_b = (yield daemonSearch(Object.assign({ projectRoot: resolvedRoot, query,
+                    limit, filters: effectiveFilters, pathPrefix }, searchOptions)))) !== null && _b !== void 0 ? _b : (yield getSearcher().search(query, limit, searchOptions, effectiveFilters, pathPrefix));
                 const allowedRoots = searchAll
-                    ? ((_b = filters.projectRoots) !== null && _b !== void 0 ? _b : [])
+                    ? ((_c = filters.projectRoots) !== null && _c !== void 0 ? _c : [])
                     : [resolvedRoot];
                 result.data = result.data.filter((chunk) => {
                     const candidate = chunkAbsPath(chunk);
@@ -645,7 +684,7 @@ exports.mcp = new commander_1.Command("mcp")
                         const regex = new RegExp(namePattern, "i");
                         results = results.filter((r) => r.symbols.some((s) => regex.test(s)));
                     }
-                    catch (_c) {
+                    catch (_d) {
                         // Invalid regex — skip filter
                     }
                 }
@@ -688,7 +727,7 @@ exports.mcp = new commander_1.Command("mcp")
                             output += `\n${traceLines.join("\n")}`;
                         }
                     }
-                    catch (_d) {
+                    catch (_e) {
                         // Trace failed — return search results without trace
                     }
                 }
@@ -1891,6 +1930,7 @@ exports.mcp = new commander_1.Command("mcp")
     // --- New command handlers (Phase 4) ---
     function handleDiffChanges(args) {
         return __awaiter(this, void 0, void 0, function* () {
+            var _a;
             ensureWatcher();
             const { getChangedFiles } = yield Promise.resolve().then(() => __importStar(require("../lib/utils/git")));
             const ref = typeof args.ref === "string" ? args.ref : undefined;
@@ -1910,8 +1950,11 @@ exports.mcp = new commander_1.Command("mcp")
                 }
                 const rel = (p) => p.startsWith(`${root}/`) ? p.slice(root.length + 1) : p;
                 if (query) {
-                    const searcher = getSearcher();
-                    const response = yield searcher.search(query, limit, { rerank: process.env.GMAX_RERANK === "1" }, {}, root);
+                    const searchOptions = {
+                        rerank: process.env.GMAX_RERANK === "1",
+                    };
+                    const response = (_a = (yield daemonSearch(Object.assign({ projectRoot: root, query,
+                        limit, filters: {}, pathPrefix: root }, searchOptions)))) !== null && _a !== void 0 ? _a : (yield getSearcher().search(query, limit, searchOptions, {}, root));
                     const changedSet = new Set(changedFiles);
                     // searcher.search() returns mapped chunks (path under metadata.path);
                     // changedFiles are absolute, so match on the resolved absolute path.
@@ -2100,8 +2143,7 @@ exports.mcp = new commander_1.Command("mcp")
                 const source = sourceRows[0];
                 if (!source.vector || source.vector.length === 0)
                     return ok("Source chunk has no embedding.");
-                const results = yield table
-                    .vectorSearch(source.vector)
+                const results = yield (0, ann_config_1.configureAnnVectorQuery)(table.vectorSearch(source.vector))
                     .select([
                     "path",
                     "start_line",
@@ -2132,7 +2174,7 @@ exports.mcp = new commander_1.Command("mcp")
     }
     function handleBuildContext(args) {
         return __awaiter(this, void 0, void 0, function* () {
-            var _a, _b;
+            var _a, _b, _c;
             ensureWatcher();
             const topic = String(args.topic || "");
             if (!topic)
@@ -2144,8 +2186,8 @@ exports.mcp = new commander_1.Command("mcp")
                 if (!proj)
                     return err("Project not added to gmax yet.");
                 (0, embedding_status_1.assertEmbeddingSearchCompatible)([proj], (0, index_config_1.readGlobalConfig)());
-                const searcher = getSearcher();
-                const response = yield searcher.search(topic, limit, { rerank: process.env.GMAX_RERANK === "1" }, {}, root);
+                const searchOptions = { rerank: process.env.GMAX_RERANK === "1" };
+                const response = (_a = (yield daemonSearch(Object.assign({ projectRoot: root, query: topic, limit, filters: {}, pathPrefix: root }, searchOptions)))) !== null && _a !== void 0 ? _a : (yield getSearcher().search(topic, limit, searchOptions, {}, root));
                 if (response.data.length === 0)
                     return ok(`No results found for "${topic}".`);
                 const rel = (p) => p.startsWith(`${root}/`) ? p.slice(root.length + 1) : p;
@@ -2167,7 +2209,7 @@ exports.mcp = new commander_1.Command("mcp")
                     const absP = searchResultPath(r);
                     const startLine = searchResultStartLine(r);
                     const endLine = searchResultEndLine(r, startLine);
-                    const sym = (_b = (_a = toStringArray(r.defined_symbols)) === null || _a === void 0 ? void 0 : _a[0]) !== null && _b !== void 0 ? _b : "";
+                    const sym = (_c = (_b = toStringArray(r.defined_symbols)) === null || _b === void 0 ? void 0 : _b[0]) !== null && _c !== void 0 ? _c : "";
                     try {
                         const sourcePath = (0, path_containment_1.resolveContainedFile)(root, absP);
                         const content = fs.readFileSync(sourcePath, "utf-8");
@@ -2181,7 +2223,7 @@ exports.mcp = new commander_1.Command("mcp")
                         sections.push(blob);
                         tokensUsed += estTokens(blob);
                     }
-                    catch (_c) {
+                    catch (_d) {
                         /* skip unreadable */
                     }
                 }
@@ -2769,6 +2811,9 @@ exports.mcp = new commander_1.Command("mcp")
         },
     }, handleReviewRisk);
     yield server.connect(transport);
+    // Start the singleton daemon in the background without delaying MCP startup.
+    void Promise.resolve().then(() => __importStar(require("../lib/utils/daemon-client"))).then(({ ensureDaemonRunning }) => ensureDaemonRunning())
+        .catch((e) => console.error("[MCP] Daemon startup failed:", e));
     // Kick off index readiness check and watcher in background
     ensureIndexReady().catch((e) => console.error("[MCP] Index readiness check failed:", e));
     ensureWatcher();

package/dist/commands/search-run.js

@@ -70,7 +70,7 @@ function shouldFallbackFromDaemonError(error) {
  */
 function runSearch(params) {
     return __awaiter(this, void 0, void 0, function* () {
-        var _a, _b;
+        var _a, _b, _c;
         const { pattern, options, projectRoot, effectiveRoot, paths, projectStatus, searchFilters, pathFilter, seeds, } = params;
         // Tracks a DB opened by the in-process path so it can be (a) returned to the
         // caller for closing after render, or (b) closed here if indexing throws.
@@ -201,6 +201,7 @@ function runSearch(params) {
                             return { kind: "dry-run", vectorDb: openedDb };
                         }
                         yield vectorDb.createFTSIndex();
+                        yield ((_b = vectorDb.createVectorIndex) === null || _b === void 0 ? void 0 : _b.call(vectorDb));
                         if (result.degraded) {
                             spinner.warn(`Indexing incomplete (${result.processed}/${result.total}) • ${result.scanErrors} scan errors • ${result.failedFiles} failed. Results may be partial.`);
                         }
@@ -228,7 +229,7 @@ function runSearch(params) {
                     }
                 }
                 // Ensure a watcher is running for live reindexing
-                if (!process.env.VITEST && !((_b = process.env.NODE_ENV) === null || _b === void 0 ? void 0 : _b.includes("test"))) {
+                if (!process.env.VITEST && !((_c = process.env.NODE_ENV) === null || _c === void 0 ? void 0 : _c.includes("test"))) {
                     const { launchWatcher } = yield Promise.resolve().then(() => __importStar(require("../lib/utils/watcher-launcher")));
                     const launched = yield launchWatcher(projectRoot);
                     if (!launched.ok && launched.reason === "spawn-failed") {
@@ -260,7 +261,7 @@ function runSearch(params) {
                 try {
                     yield openedDb.close();
                 }
-                catch (_c) { }
+                catch (_d) { }
             }
             throw e;
         }

package/dist/commands/similar.js

@@ -44,6 +44,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.similar = void 0;
 const commander_1 = require("commander");
+const ann_config_1 = require("../lib/store/ann-config");
 const vector_db_1 = require("../lib/store/vector-db");
 const agent_errors_1 = require("../lib/utils/agent-errors");
 const arrow_1 = require("../lib/utils/arrow");
@@ -122,8 +123,7 @@ exports.similar = new commander_1.Command("similar")
             exclude: opts.exclude,
         });
         const pathScope = buildScopeWhere(scope);
-        const results = yield table
-            .vectorSearch(sourceVector)
+        const results = yield (0, ann_config_1.configureAnnVectorQuery)(table.vectorSearch(sourceVector))
             .select([
             "path",
             "start_line",

package/dist/eval-graph-recovery-probe.js

@@ -58,6 +58,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 (_a = (_c = process.env).GMAX_WORKER_COUNT) !== null && _a !== void 0 ? _a : (_c.GMAX_WORKER_COUNT = "1");
 const path = __importStar(require("node:path"));
 const config_1 = require("./config");
+const ann_config_1 = require("./lib/store/ann-config");
 const vector_db_1 = require("./lib/store/vector-db");
 const exit_1 = require("./lib/utils/exit");
 const filter_builder_1 = require("./lib/utils/filter-builder");
@@ -114,8 +115,7 @@ function probe(table, sym, expectedFile) {
             "defined_symbols",
             "referenced_symbols",
         ];
-        const vectorRows = (yield table
-            .vectorSearch(dense)
+        const vectorRows = (yield (0, ann_config_1.configureAnnVectorQuery)(table.vectorSearch(dense))
             .select([...columns, "_distance"])
             .where(where)
             .limit(PRE_K)

package/dist/eval-graph-sanity.js

@@ -75,6 +75,7 @@ const path = __importStar(require("node:path"));
 const config_1 = require("./config");
 const callsites_1 = require("./lib/graph/callsites");
 const graph_builder_1 = require("./lib/graph/graph-builder");
+const ann_config_1 = require("./lib/store/ann-config");
 const vector_db_1 = require("./lib/store/vector-db");
 const exit_1 = require("./lib/utils/exit");
 const filter_builder_1 = require("./lib/utils/filter-builder");
@@ -128,8 +129,7 @@ function probe(target) {
             "defined_symbols",
             "referenced_symbols",
         ];
-        const vectorRows = (yield table
-            .vectorSearch(dense)
+        const vectorRows = (yield (0, ann_config_1.configureAnnVectorQuery)(table.vectorSearch(dense))
             .select([...columns, "_distance"])
             .where(where)
             .limit(PRE_K)

package/dist/lib/analysis/surprising-connections.js

@@ -60,6 +60,7 @@ exports.buildFindings = buildFindings;
 exports.analyzeSurprisingConnections = analyzeSurprisingConnections;
 const path = __importStar(require("node:path"));
 const callsites_1 = require("../graph/callsites");
+const ann_config_1 = require("../store/ann-config");
 const arrow_1 = require("../utils/arrow");
 const scope_filter_1 = require("../utils/scope-filter");
 exports.DEFAULT_SURPRISE_OPTIONS = {
@@ -520,8 +521,7 @@ function analyzeSurprisingConnections(table_1, projectRoot_1) {
         };
         const pairs = new Map();
         for (const source of anchors) {
-            const neighbors = (yield table
-                .vectorSearch(source.vector)
+            const neighbors = (yield (0, ann_config_1.configureAnnVectorQuery)(table.vectorSearch(source.vector))
                 .select([...exports.SURPRISE_COLUMNS, "_distance"])
                 .where(where)
                 .limit(opts.neighbors + 8)

package/dist/lib/daemon/daemon.js

@@ -440,6 +440,7 @@ class Daemon {
                         if (this.vectorDb) {
                             yield this.vectorDb.ensureTable();
                             yield this.vectorDb.createFTSIndex();
+                            yield this.vectorDb.createVectorIndex(false, 5, true);
                         }
                         const pool = this.workerPool;
                         if (!pool)

package/dist/lib/index/batch-processor.js

@@ -44,6 +44,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ProjectBatchProcessor = void 0;
 const path = __importStar(require("node:path"));
+const config_1 = require("../../config");
 const vector_db_1 = require("../store/vector-db");
 const cache_check_1 = require("../utils/cache-check");
 const file_utils_1 = require("../utils/file-utils");
@@ -55,9 +56,12 @@ const watcher_batch_1 = require("./watcher-batch");
 const DEBOUNCE_MS = 2000;
 const MAX_RETRIES = 5;
 const MAX_BATCH_SIZE = 50;
+// Pool priority cannot preempt an active processFile, so keep one worker free
+// for latency-sensitive encodeQuery/rerank work during indexing bursts.
+const DEFAULT_BATCH_CONCURRENCY = Math.max(1, config_1.CONFIG.WORKER_THREADS - 1);
 class ProjectBatchProcessor {
     constructor(opts) {
-        var _a, _b;
+        var _a, _b, _c;
         this.pending = new Map();
         this.retryCount = new Map();
         this.retryAt = new Map();
@@ -86,6 +90,7 @@ class ProjectBatchProcessor {
         this.filePolicy =
             (_b = opts.filePolicy) !== null && _b !== void 0 ? _b : new file_policy_1.ProjectFilePolicy(opts.projectRoot);
         this.wtag = `watch:${path.basename(opts.projectRoot)}`;
+        this.concurrency = Math.max(1, Math.floor((_c = opts.concurrency) !== null && _c !== void 0 ? _c : DEFAULT_BATCH_CONCURRENCY));
         const taskTimeoutMs = (() => {
             var _a;
             const fromEnv = Number.parseInt((_a = process.env.GMAX_WORKER_TASK_TIMEOUT_MS) !== null && _a !== void 0 ? _a : "", 10);
@@ -238,7 +243,7 @@ class ProjectBatchProcessor {
             (0, logger_1.log)(this.wtag, `Processing ${batch.size} changed files: ${filenames.join(", ")}`);
             const start = Date.now();
             let reindexed = 0;
-            let processed = 0;
+            let settled = 0;
             let backoffOverrideMs = 0;
             try {
                 // No lock needed — daemon is the single writer to LanceDB/MetaCache
@@ -268,14 +273,8 @@ class ProjectBatchProcessor {
                     }
                     this.pending.set(absPath, event);
                 };
-                for (const [absPath, event] of batch) {
-                    if (batchAc.signal.aborted)
-                        break;
-                    processed++;
-                    if (batch.size > 10 &&
-                        (processed % 10 === 0 || processed === batch.size)) {
-                        (0, logger_1.log)(this.wtag, `Progress: ${processed}/${batch.size} (${reindexed} reindexed)`);
-                    }
+                let stopDispatch = false;
+                const processOne = (absPath, event) => __awaiter(this, void 0, void 0, function* () {
                     // Reclassify every event at apply time. A catchup-derived unlink may be
                     // stale if the file was recreated after its directory was scanned.
                     try {
@@ -283,7 +282,7 @@ class ProjectBatchProcessor {
                         if (classification.status === "error") {
                             console.error(`[${this.wtag}] Policy could not classify ${absPath}:`, classification.error);
                             retryFailures.add(absPath);
-                            continue;
+                            return;
                         }
                         if (classification.status === "excluded" ||
                             classification.status === "missing") {
@@ -293,11 +292,11 @@ class ProjectBatchProcessor {
                                 reindexed++;
                             }
                             completed.add(absPath);
-                            continue;
+                            return;
                         }
                         const stats = classification.stat;
                         if (diskPressure === "critical") {
-                            continue;
+                            return;
                         }
                         const cached = this.metaCache.get(absPath);
                         const forceReprocess = batchForceGenerations.has(absPath);
@@ -305,7 +304,7 @@ class ProjectBatchProcessor {
                             (0, cache_coherence_1.isMetaEntryCacheCurrent)(cached, absPath) &&
                             (0, cache_check_1.isFileCached)(cached, stats)) {
                             completed.add(absPath);
-                            continue;
+                            return;
                         }
                         // Fast path: if only mtime changed but size matches and we have a hash,
                         // verify in-process instead of dispatching to a worker (~220ms saved).
@@ -321,7 +320,7 @@ class ProjectBatchProcessor {
                             if (hash === cached.hash) {
                                 metaUpdates.set(absPath, Object.assign(Object.assign({}, cached), { mtimeMs: snapshot.mtimeMs }));
                                 completed.add(absPath);
-                                continue;
+                                return;
                             }
                         }
                         const result = yield pool.processFile({
@@ -334,19 +333,19 @@ class ProjectBatchProcessor {
                         const current = yield this.filePolicy.classifyFile(absPath);
                         if (current.status === "error") {
                             retryFailures.add(absPath);
-                            continue;
+                            return;
                         }
                         if (current.status === "excluded" || current.status === "missing") {
                             deletes.push(absPath);
                             metaDeletes.push(absPath);
                             reindexed++;
                             completed.add(absPath);
-                            continue;
+                            return;
                         }
                         if (current.stat.mtimeMs !== result.mtimeMs ||
                             current.stat.size !== result.size) {
                             retryFailures.add(absPath);
-                            continue;
+                            return;
                         }
                         const metaEntry = {
                             hash: result.hash,
@@ -361,14 +360,14 @@ class ProjectBatchProcessor {
                             cached.hasVectors === result.vectors.length > 0) {
                             metaUpdates.set(absPath, metaEntry);
                             completed.add(absPath);
-                            continue;
+                            return;
                         }
                         if (result.shouldDelete) {
                             deletes.push(absPath);
                             metaUpdates.set(absPath, metaEntry);
                             reindexed++;
                             completed.add(absPath);
-                            continue;
+                            return;
                         }
                         deletes.push(absPath);
                         if (result.vectors.length > 0) {
@@ -380,7 +379,7 @@ class ProjectBatchProcessor {
                     }
                     catch (err) {
                         if (batchAc.signal.aborted)
-                            break;
+                            return;
                         const code = err === null || err === void 0 ? void 0 : err.code;
                         if (code === "ENOENT") {
                             deletes.push(absPath);
@@ -392,12 +391,40 @@ class ProjectBatchProcessor {
                             console.error(`[${this.wtag}] Failed to process ${absPath}:`, err);
                             if (!pool.isHealthy()) {
                                 console.error(`[${this.wtag}] Worker pool unhealthy, aborting batch`);
-                                break;
+                                stopDispatch = true;
+                                return;
                             }
                             retryFailures.add(absPath);
                         }
                     }
+                    finally {
+                        settled++;
+                        if (batch.size > 10 &&
+                            (settled % 10 === 0 || settled === batch.size)) {
+                            (0, logger_1.log)(this.wtag, `Progress: ${settled}/${batch.size} (${reindexed} reindexed)`);
+                        }
+                    }
+                });
+                const activeTasks = [];
+                const schedule = (task) => __awaiter(this, void 0, void 0, function* () {
+                    const taskPromise = task();
+                    activeTasks.push(taskPromise);
+                    const removeActiveTask = () => {
+                        const index = activeTasks.indexOf(taskPromise);
+                        if (index !== -1)
+                            activeTasks.splice(index, 1);
+                    };
+                    void taskPromise.then(removeActiveTask, removeActiveTask);
+                    if (activeTasks.length >= this.concurrency) {
+                        yield Promise.race(activeTasks);
+                    }
+                });
+                for (const [absPath, event] of batch) {
+                    if (batchAc.signal.aborted || stopDispatch)
+                        break;
+                    yield schedule(() => processOne(absPath, event));
                 }
+                yield Promise.allSettled(activeTasks);
                 const pureDeleteCandidates = new Set(metaDeletes);
                 const authoritativeMetaDeletes = new Set();
                 for (const absPath of pureDeleteCandidates) {