grepmax 0.26.37 → 0.26.40

package/.claude-plugin/marketplace.json

@@ -10,7 +10,7 @@
       "name": "grepmax",
       "source": "./plugins/grepmax",
       "description": "Semantic code search for Claude Code. Automatically indexes your project and provides intelligent search capabilities.",
-      "version": "0.26.37",
+      "version": "0.26.40",
       "author": {
         "name": "Robert Owens",
         "email": "robowens@me.com"

package/dist/lib/index/batch-processor.js

@@ -43,6 +43,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.ProjectBatchProcessor = void 0;
+exports.applyReusedEmbeddings = applyReusedEmbeddings;
 const path = __importStar(require("node:path"));
 const config_1 = require("../../config");
 const vector_db_1 = require("../store/vector-db");
@@ -172,6 +173,18 @@ class ProjectBatchProcessor {
             recentReindexed,
         };
     }
+    loadReusableEmbeddings(absPath) {
+        return __awaiter(this, void 0, void 0, function* () {
+            // An optimisation only: a failed lookup costs a full re-embed, nothing more.
+            try {
+                return yield this.vectorDb.getReusableEmbeddings(absPath);
+            }
+            catch (err) {
+                (0, logger_1.debug)(this.wtag, `embedding reuse lookup failed for ${absPath}: ${err instanceof Error ? err.message : String(err)}`);
+                return undefined;
+            }
+        });
+    }
     close() {
         return __awaiter(this, void 0, void 0, function* () {
             var _a;
@@ -360,11 +373,14 @@ class ProjectBatchProcessor {
                                 return;
                             }
                         }
-                        const result = yield pool.processFile({
-                            path: absPath,
-                            absolutePath: absPath,
-                            projectRoot: this.projectRoot,
-                        }, batchAc.signal);
+                        // A forced reprocess is a repair; it must not trust what is stored.
+                        const reusable = !forceReprocess && (cached === null || cached === void 0 ? void 0 : cached.hasVectors) !== false && (cached === null || cached === void 0 ? void 0 : cached.hash)
+                            ? yield this.loadReusableEmbeddings(absPath)
+                            : undefined;
+                        const result = yield pool.processFile(Object.assign({ path: absPath, absolutePath: absPath, projectRoot: this.projectRoot }, ((reusable === null || reusable === void 0 ? void 0 : reusable.size)
+                            ? { reusableKeys: Array.from(reusable.keys()) }
+                            : {})), batchAc.signal);
+                        applyReusedEmbeddings(result, reusable);
                         // Policy and filesystem state can change while embedding is in
                         // flight. Never commit a result that no longer describes the path.
                         const current = yield this.filePolicy.classifyFile(absPath);
@@ -652,3 +668,25 @@ class ProjectBatchProcessor {
 }
 exports.ProjectBatchProcessor = ProjectBatchProcessor;
 ProjectBatchProcessor.RECENT_BATCH_WINDOW = 5;
+/**
+ * Fill the records a worker left unembedded from the embeddings offered to it.
+ * Throws when one cannot be filled, so the file is retried rather than stored
+ * with an empty vector.
+ */
+function applyReusedEmbeddings(result, reusable) {
+    var _a;
+    if (!((_a = result.reused) === null || _a === void 0 ? void 0 : _a.length))
+        return;
+    for (const { index, key } of result.reused) {
+        const record = result.vectors[index];
+        const embedding = reusable === null || reusable === void 0 ? void 0 : reusable.get(key);
+        if (!record || !embedding) {
+            throw new Error(`worker reused an embedding that was not offered (chunk ${index})`);
+        }
+        record.vector = embedding.vector;
+        record.colbert = embedding.colbert;
+        record.colbert_scale = embedding.colbert_scale;
+        record.pooled_colbert_48d = embedding.pooled_colbert_48d;
+        record.doc_token_ids = embedding.doc_token_ids;
+    }
+}

package/dist/lib/index/embedding-reuse.js

@@ -0,0 +1,47 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.embeddingReuseKey = embeddingReuseKey;
+exports.toFloat32 = toFloat32;
+exports.toInt32 = toInt32;
+exports.toBytes = toBytes;
+const node_crypto_1 = require("node:crypto");
+/** Keyed by the exact text passed to the embedder (`PreparedChunk.content`). */
+function embeddingReuseKey(content) {
+    return (0, node_crypto_1.createHash)("sha256").update(content).digest("hex");
+}
+function unwrap(val) {
+    if (val &&
+        !ArrayBuffer.isView(val) &&
+        typeof val.toArray === "function") {
+        return val.toArray();
+    }
+    return val;
+}
+// Copies rather than views: an Arrow view pins its whole record batch buffer.
+function toFloat32(val) {
+    const v = unwrap(val);
+    if (v instanceof Float32Array)
+        return v.slice();
+    if (ArrayBuffer.isView(v) || Array.isArray(v)) {
+        return Float32Array.from(v);
+    }
+    return new Float32Array(0);
+}
+function toInt32(val) {
+    const v = unwrap(val);
+    if (v instanceof Int32Array)
+        return v.slice();
+    if (ArrayBuffer.isView(v) || Array.isArray(v)) {
+        return Int32Array.from(v);
+    }
+    return new Int32Array(0);
+}
+function toBytes(val) {
+    const v = unwrap(val);
+    if (ArrayBuffer.isView(v)) {
+        return Buffer.from(new Uint8Array(v.buffer, v.byteOffset, v.byteLength).slice());
+    }
+    if (Array.isArray(v))
+        return Buffer.from(v);
+    return Buffer.alloc(0);
+}

package/dist/lib/store/vector-db.js

@@ -51,6 +51,7 @@ const fs = __importStar(require("node:fs"));
 const lancedb = __importStar(require("@lancedb/lancedb"));
 const apache_arrow_1 = require("apache-arrow");
 const config_1 = require("../../config");
+const embedding_reuse_1 = require("../index/embedding-reuse");
 const index_config_1 = require("../index/index-config");
 const cleanup_1 = require("../utils/cleanup");
 const filter_builder_1 = require("../utils/filter-builder");
@@ -304,10 +305,15 @@ class VectorDB {
                     (0, logger_1.log)("vectordb", `Periodic maintenance failed: ${err}`);
                 }
             }));
-            this.maintenancePromise = run.finally(() => {
-                if (this.maintenancePromise === run)
+            // Compare against the promise actually stored, not `run`: `.finally()`
+            // returns a new promise, so checking `run` never matched and the field
+            // latched non-null forever — isMaintenanceActive() then read true for the
+            // daemon's whole life, silently blocking every recycle and idle shutdown.
+            const tracked = run.finally(() => {
+                if (this.maintenancePromise === tracked)
                     this.maintenancePromise = null;
             });
+            this.maintenancePromise = tracked;
         }, MAINTENANCE_INTERVAL_MS);
         this.maintenanceTimer.unref();
     }
@@ -1430,6 +1436,48 @@ class VectorDB {
             return unique;
         });
     }
+    /**
+     * The stored embeddings of one file, keyed by `embeddingReuseKey(content)`.
+     * Rows whose dense width disagrees with this table are left out rather than
+     * offered — reuse must never be the way a wrong-width vector gets copied.
+     */
+    getReusableEmbeddings(filePath) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a;
+            const table = yield this.ensureTable();
+            const rows = yield table
+                .query()
+                .select([
+                "content",
+                "vector",
+                "colbert",
+                "colbert_scale",
+                "pooled_colbert_48d",
+                "doc_token_ids",
+            ])
+                .where(`path = '${(0, filter_builder_1.escapeSqlString)(filePath)}'`)
+                .toArray();
+            const reusable = new Map();
+            for (const row of rows) {
+                const vector = (0, embedding_reuse_1.toFloat32)(row.vector);
+                const colbert = (0, embedding_reuse_1.toBytes)(row.colbert);
+                if (vector.length !== this.vectorDim || colbert.length === 0)
+                    continue;
+                const pooled = row.pooled_colbert_48d == null
+                    ? undefined
+                    : (0, embedding_reuse_1.toFloat32)(row.pooled_colbert_48d);
+                const tokenIds = row.doc_token_ids == null ? undefined : (0, embedding_reuse_1.toInt32)(row.doc_token_ids);
+                reusable.set((0, embedding_reuse_1.embeddingReuseKey)(String((_a = row.content) !== null && _a !== void 0 ? _a : "")), {
+                    vector,
+                    colbert,
+                    colbert_scale: typeof row.colbert_scale === "number" ? row.colbert_scale : 1,
+                    pooled_colbert_48d: (pooled === null || pooled === void 0 ? void 0 : pooled.length) ? pooled : undefined,
+                    doc_token_ids: tokenIds,
+                });
+            }
+            return reusable;
+        });
+    }
     getStats() {
         return __awaiter(this, void 0, void 0, function* () {
             const table = yield this.ensureTable();

package/dist/lib/workers/orchestrator.js

@@ -52,6 +52,7 @@ const uuid_1 = require("uuid");
 const config_1 = require("../../config");
 const chunker_1 = require("../index/chunker");
 const embedding_generation_1 = require("../index/embedding-generation");
+const embedding_reuse_1 = require("../index/embedding-reuse");
 const skeleton_1 = require("../skeleton");
 const file_utils_1 = require("../utils/file-utils");
 const logger_1 = require("../utils/logger");
@@ -234,6 +235,7 @@ class WorkerOrchestrator {
     }
     processFile(input, onProgress) {
         return __awaiter(this, void 0, void 0, function* () {
+            var _a;
             const fileStart = performance.now();
             const absolutePath = path.isAbsolute(input.path)
                 ? input.path
@@ -281,12 +283,32 @@ class WorkerOrchestrator {
                 return { vectors: [], hash, mtimeMs, size };
             }
             const preparedChunks = this.toPreparedChunks(input.path, hash, chunks, skeletonResult.success ? skeletonResult.skeleton : undefined);
+            const reusableKeys = ((_a = input.reusableKeys) === null || _a === void 0 ? void 0 : _a.length)
+                ? new Set(input.reusableKeys)
+                : null;
+            const reused = [];
+            const toEmbed = [];
+            preparedChunks.forEach((chunk, idx) => {
+                if (reusableKeys) {
+                    const key = (0, embedding_reuse_1.embeddingReuseKey)(chunk.content);
+                    if (reusableKeys.has(key)) {
+                        reused.push({ index: idx, key });
+                        return;
+                    }
+                }
+                toEmbed.push(idx);
+            });
             const embedStart = performance.now();
-            const hybrids = yield this.computeHybrid(preparedChunks.map((chunk) => chunk.content), onProgress);
-            (0, logger_1.debug)("orch", `embedded ${input.path} → ${hybrids.length} vectors ${(performance.now() - embedStart).toFixed(0)}ms`);
+            const embedded = yield this.computeHybrid(toEmbed.map((idx) => preparedChunks[idx].content), onProgress);
+            const hybrids = new Map();
+            toEmbed.forEach((chunkIdx, i) => {
+                if (embedded[i])
+                    hybrids.set(chunkIdx, embedded[i]);
+            });
+            (0, logger_1.debug)("orch", `embedded ${input.path} → ${embedded.length} vectors (${reused.length} reused) ${(performance.now() - embedStart).toFixed(0)}ms`);
             const vectors = preparedChunks.map((chunk, idx) => {
                 var _a;
-                const hybrid = (_a = hybrids[idx]) !== null && _a !== void 0 ? _a : {
+                const hybrid = (_a = hybrids.get(idx)) !== null && _a !== void 0 ? _a : {
                     dense: new Float32Array(),
                     colbert: new Int8Array(),
                     scale: 1,
@@ -295,7 +317,9 @@ class WorkerOrchestrator {
             });
             onProgress === null || onProgress === void 0 ? void 0 : onProgress();
             (0, logger_1.debug)("orch", `processFile done: ${input.path} ${vectors.length} vectors ${(performance.now() - fileStart).toFixed(0)}ms`);
-            return { vectors, hash, mtimeMs, size };
+            return reused.length
+                ? { vectors, hash, mtimeMs, size, reused }
+                : { vectors, hash, mtimeMs, size };
         });
     }
     encodeQuery(text) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "grepmax",
-  "version": "0.26.37",
+  "version": "0.26.40",
   "author": "Robert Owens <78518764+reowens@users.noreply.github.com>",
   "homepage": "https://github.com/reowens/grepmax",
   "bugs": {
@@ -40,7 +40,7 @@
     "typecheck": "tsc --noEmit",
     "test:typecheck": "tsc -p tsconfig.tests.json",
     "prepublishOnly": "pnpm run test:typecheck && pnpm build",
-    "preversion": "pnpm test && pnpm typecheck && pnpm run test:typecheck && pnpm run format:check",
+    "preversion": "pnpm audit --prod && pnpm test && pnpm typecheck && pnpm run test:typecheck && pnpm run format:check",
     "version": "bash scripts/sync-versions.sh && git add -A",
     "postversion": "bash scripts/postrelease.sh"
   },

package/plugins/grepmax/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "grepmax",
-  "version": "0.26.37",
+  "version": "0.26.40",
   "description": "Semantic code search for Claude Code. Automatically indexes your project and provides intelligent search capabilities.",
   "author": {
     "name": "Robert Owens",