grepmax 0.26.8 → 0.26.9

package/.claude-plugin/marketplace.json

@@ -10,7 +10,7 @@
       "name": "grepmax",
       "source": "./plugins/grepmax",
       "description": "Semantic code search for Claude Code. Automatically indexes your project and provides intelligent search capabilities.",
-      "version": "0.26.8",
+      "version": "0.26.9",
       "author": {
         "name": "Robert Owens",
         "email": "robowens@me.com"

package/README.md

@@ -221,6 +221,15 @@ The daemon auto-starts when you run `gmax add`, `gmax index`, `gmax remove`, `gm
 
 gmax can use a local LLM (via llama-server) for agentic codebase investigation. This is entirely opt-in and disabled by default — gmax works fine without it.
 
+> **Memory footprint.** `gmax llm start`, `investigate`, `review`, and `summarize` load a multi-GB
+> GGUF model (typical coding models run 16–21 GB). On a memory-constrained machine that can stall
+> or hang the host. Start it deliberately, not as a reflex — and if you run coding agents against
+> this repo, they should be told not to invoke these on their own initiative. The bundled skill
+> already carries that instruction.
+>
+> Everything else in gmax — search, `trace`, `impact`, `context`, `--context-for-llm` — runs off
+> the local index with no model load.
+
 ```bash
 gmax llm on                   # Enable LLM features (persists to config)
 gmax llm start                # Start llama-server (auto-starts daemon too)
@@ -393,6 +402,19 @@ gmax watch stop && gmax watch --daemon -b  # Restart daemon
 
 `gmax doctor` reports ANN index state. `ANN: vector index not built` is normal with the default exact-search configuration.
 
+### Known issues
+
+Some log output looks alarming but is expected and self-healing. Before filing a bug, check
+[`docs/known-limitations.md`](docs/known-limitations.md) — it covers what each case means and
+whether you need to act.
+
+| You see | What it means |
+|---|---|
+| `Optimize panicked ... inverted/builder.rs` | Upstream Lance bug in incremental FTS merge ([lance#8310](https://github.com/lance-format/lance/issues/8310)). gmax rebuilds the index and retries automatically; no data loss, no action needed. |
+| `disabling auto-rebuild until an optimize succeeds` | Transient, not a wedge. The next successful optimize clears it. Search stays available. |
+| `ANN: vector index not built` | Normal — exact search is the default. |
+| `npm audit` reports advisories on install | Two transitive advisories (`sharp`, `adm-zip`) have no upstream fix yet; neither is on a code path gmax executes. |
+
 ## Contributing
 
 See [CLAUDE.md](CLAUDE.md) for development setup, commands, and architecture details.

package/dist/config.js

@@ -251,8 +251,28 @@ exports.DISK_LOW_BYTES = (() => {
     const gb = Number.parseFloat((_a = process.env.GMAX_DISK_LOW_GB) !== null && _a !== void 0 ? _a : "20");
     return (Number.isFinite(gb) && gb > 0 ? gb : 20) * 1024 * 1024 * 1024;
 })();
-// Trigger compaction when small (uncompacted) fragment count exceeds this
-exports.FRAGMENT_COMPACT_THRESHOLD = 50;
+// Trigger compaction when small (uncompacted) fragment count exceeds this.
+//
+// Deliberately high. Lance's optimize() merges the *whole* table into ~2
+// fragments regardless of how many small ones it absorbs, so a compaction costs
+// the same full-table rewrite at 51 fragments as at 400 — the threshold only
+// decides how far that fixed cost is amortized. At 50 it was amortized as little
+// as possible: on a continuously-watched project this fired every ~15-25min and
+// rewrote ~10-14GB each time, measured at 40 compactions / 586GB over 26h.
+//
+// Fragmentation also does not cost what you would expect. With ANN off (the
+// default), vector search is a brute-force scan that Lance parallelizes across
+// fragments, so compacting to 2 fragments collapses it to 2-way parallelism.
+// Interleaved benchmark, 395k rows / 384d, 96 samples per config:
+//
+//   fragments |   2   |  50  | 200  | 800
+//   vector p50| 165.6 | 73.4 | 61.6 | 66.8   (ms)
+//
+// Flat from 50-800; 2 is 2.5x slower. So raising this wins on writes *and*
+// latency. 400 sits inside the measured range with margin at both ends.
+// Caveat: measured on the no-ANN path — revisit if GMAX_ANN=1 becomes default,
+// since an ANN index partitions independently of fragments.
+exports.FRAGMENT_COMPACT_THRESHOLD = 400;
 // Extensions we consider for indexing to avoid binary noise and improve relevance.
 exports.INDEXABLE_EXTENSIONS = new Set([
     ".ts",

package/dist/lib/daemon/daemon.js

@@ -174,7 +174,11 @@ class Daemon {
         return new pool_1.WorkerPool(generation, embedMode);
     }
     createVectorDb(vectorDim, lease) {
-        return new vector_db_1.VectorDB(config_1.PATHS.lancedbDir, vectorDim, lease);
+        const db = new vector_db_1.VectorDB(config_1.PATHS.lancedbDir, vectorDim, lease);
+        // The daemon owns the store, so it is the one process allowed to create or
+        // rebuild the shared FTS index. Readers adopt it; see VectorDB.adoptFTSIndex.
+        db.markIndexOwner();
+        return db;
     }
     mlxMode(config) {
         if (config.embedMode !== "gpu")
@@ -318,6 +322,7 @@ class Daemon {
                 this.activeConfig = (0, index_config_1.readGlobalConfig)();
                 this.activeGeneration = (0, embedding_generation_1.resolveEmbeddingGeneration)(this.activeConfig);
                 this.vectorDb = new vector_db_1.VectorDB(config_1.PATHS.lancedbDir, this.activeGeneration.vectorDim);
+                this.vectorDb.markIndexOwner();
                 this.workerPool = this.createWorkerPool(this.activeGeneration, this.activeConfig.embedMode);
                 this.vectorDb.startMaintenanceLoop((fn) => this.runSharedOperation("store-maintenance", undefined, () => fn()));
                 console.log("[daemon] Opening MetaCache:", config_1.PATHS.lmdbPath);

package/dist/lib/search/searcher.js

@@ -15,6 +15,7 @@ exports.buildWhereClause = buildWhereClause;
 const config_1 = require("../../config");
 const ann_config_1 = require("../store/ann-config");
 const filter_builder_1 = require("../utils/filter-builder");
+const logger_1 = require("../utils/logger");
 const pool_1 = require("../workers/pool");
 const intent_1 = require("./intent");
 const pagerank_1 = require("./pagerank");
@@ -436,12 +437,15 @@ class Searcher {
                 this.ftsIndexChecked = true;
                 this.ftsLastCheckedAt = now;
                 try {
-                    yield this.db.createFTSIndex();
+                    // Adopt-only: search runs in every process, so building here raced
+                    // CreateIndex across processes and duplicated a full-size index per
+                    // winner. Owners build it (daemon pre-warm, `gmax index`, maintenance).
+                    yield this.db.adoptFTSIndex();
                     this.ftsAvailable = true;
                 }
                 catch (e) {
                     this.ftsAvailable = false;
-                    console.warn("[Searcher] Failed to ensure FTS index:", e);
+                    (0, logger_1.debug)("searcher", `FTS index unavailable, vector-only search: ${e}`);
                 }
             }
             // Phase A: Lightweight retrieval — only columns needed for RRF, cosine, boost, dedup
@@ -501,13 +505,23 @@ class Searcher {
                     this.ftsAvailable = false;
                     const msg = e instanceof Error ? e.message : String(e);
                     if (msg.includes("position")) {
-                        // FTS index lacks positional data — rebuild it
-                        try {
-                            yield this.db.createFTSIndex(true);
-                            this.ftsAvailable = true;
-                            console.warn("[Searcher] Rebuilt FTS index with position support — retry search");
+                        // FTS index lacks positional data — rebuild it, but only from the
+                        // index owner. This drops the index before recreating it, so a
+                        // non-owner (CLI fallback, MCP session) would yank the shared index
+                        // out from under every other process and, with several running at
+                        // once, race CreateIndex on the way back up. Non-owners degrade to
+                        // vector-only and let the daemon repair it.
+                        if (this.db.canBuildIndexes()) {
+                            try {
+                                yield this.db.createFTSIndex(true);
+                                this.ftsAvailable = true;
+                                console.warn("[Searcher] Rebuilt FTS index with position support — retry search");
+                            }
+                            catch (_r) { }
+                        }
+                        else {
+                            (0, logger_1.debug)("searcher", "FTS index lacks positions; deferring rebuild to the index owner");
                         }
-                        catch (_r) { }
                     }
                     else {
                         console.warn(`[Searcher] FTS search failed (will retry later): ${msg}`);

package/dist/lib/store/vector-db.js

@@ -102,6 +102,8 @@ class VectorDB {
         this.maintainedTableVersion = null;
         this.lastMaintenanceMs = 0;
         this.ftsIndexEnsured = false;
+        /** Only an index owner may create/rebuild shared indexes — see markIndexOwner. */
+        this.indexOwner = false;
         this.lastOptimizeDidWork = false;
         this.maintenanceRunner = (fn) => fn();
         /** Latched when a from-scratch FTS rebuild failed to stop an optimize
@@ -735,6 +737,48 @@ class VectorDB {
             return this.withWriteGate(() => this.createFTSIndexUnsafe(rebuild, retries));
         });
     }
+    /**
+     * Mark this instance as the store's index owner — the daemon, or a
+     * deliberately exclusive CLI operation. Only an owner may create or rebuild
+     * the shared FTS index; see `adoptFTSIndex`.
+     */
+    markIndexOwner() {
+        this.indexOwner = true;
+    }
+    /** Whether this instance may create or rebuild shared indexes. */
+    canBuildIndexes() {
+        return this.indexOwner;
+    }
+    /**
+     * Adopt the FTS index if one already exists, without ever building it.
+     *
+     * Search runs in *every* process — the daemon, each CLI invocation that falls
+     * back in-process, each MCP session. `createFTSIndex` on a miss meant N
+     * processes could issue `CreateIndex` against the same table version
+     * concurrently: Lance rejects the losers with a retryable commit conflict
+     * (observed 2026-08-17T00:03, all 5 retries lost) and every winner writes a
+     * full-size inverted index — ~1.4GB each on this store, which is how
+     * `_indices` reached 47GB across 35 near-identical copies.
+     *
+     * Building is therefore reserved for the explicit paths that already run
+     * under an owner: daemon pre-warm, `gmax index`, and post-optimize
+     * maintenance. Throws when no index exists so callers set `ftsAvailable =
+     * false` and degrade to vector-only search — never silently reports FTS as
+     * ready.
+     */
+    adoptFTSIndex() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (this.ftsIndexEnsured)
+                return;
+            const table = yield this.ensureTable();
+            const indices = yield table.listIndices();
+            const existing = indices.find((index) => index.name === "content_idx" || index.columns.includes("content"));
+            if (!existing) {
+                throw new Error("[vector-db] FTS index not built yet; this process is not the index owner");
+            }
+            this.ftsIndexEnsured = true;
+        });
+    }
     createVectorIndex() {
         return __awaiter(this, arguments, void 0, function* (rebuild = false, retries = 5, checkOnly = false) {
             if (this.checkDiskPressure() !== "ok") {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "grepmax",
-  "version": "0.26.8",
+  "version": "0.26.9",
   "author": "Robert Owens <78518764+reowens@users.noreply.github.com>",
   "homepage": "https://github.com/reowens/grepmax",
   "bugs": {

package/plugins/grepmax/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "grepmax",
-  "version": "0.26.8",
+  "version": "0.26.9",
   "description": "Semantic code search for Claude Code. Automatically indexes your project and provides intelligent search capabilities.",
   "author": {
     "name": "Robert Owens",