grepmax 0.17.7 → 0.17.9

package/dist/eval-graph-recovery-probe.js

@@ -0,0 +1,170 @@
+"use strict";
+/**
+ * Phase 3 design probe. For each platform symbol-name miss, determine whether
+ * the expected DEFINITION chunk is:
+ *   (a) inside the top-200 fusion pool but lost downstream  -> a ranking fix
+ *   (b) outside the pool (but inside the 500-row retrieval)  -> recovery fix
+ *   (c) outside retrieval entirely                            -> unreachable
+ * and whether a 1-hop ref->def walk from the top-K fusion seeds reaches it
+ * (i.e. some seed's referenced_symbols contains the query symbol).
+ *
+ * Run: npx tsx src/eval-graph-recovery-probe.ts
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var _a, _b;
+var _c;
+Object.defineProperty(exports, "__esModule", { value: true });
+(_a = (_c = process.env).GMAX_WORKER_COUNT) !== null && _a !== void 0 ? _a : (_c.GMAX_WORKER_COUNT = "1");
+const path = __importStar(require("node:path"));
+const config_1 = require("./config");
+const vector_db_1 = require("./lib/store/vector-db");
+const exit_1 = require("./lib/utils/exit");
+const pool_1 = require("./lib/workers/pool");
+const filter_builder_1 = require("./lib/utils/filter-builder");
+const PLATFORM_ROOT = path.join((_b = process.env.HOME) !== null && _b !== void 0 ? _b : "", "Development/beyond/platform");
+const PRE_K = 500;
+const STAGE1_K = 200;
+const SEED_K = 20;
+const RRF_K = 60;
+// (query symbol, expected definition file) — the rank-0 platform misses.
+const CASES = [
+    ["BeyondError", "packages/shared/src/errors.ts"],
+    ["ErrorCodes", "packages/shared/src/errors.ts"],
+    ["createDb", "packages/db/src/index.ts"],
+    ["createDbAsync", "packages/db/src/index.ts"],
+    ["authMiddleware", "packages/api/src/middleware/auth.ts"],
+    ["requireAuth", "packages/api/src/middleware/auth.ts"],
+    ["resolveActor", "packages/api/src/middleware/auth.ts"],
+    ["getActor", "packages/api/src/middleware/auth.ts"],
+    ["errorHandler", "packages/api/src/middleware/error.ts"],
+    ["initializeApp", "packages/api/src/app.ts"],
+];
+function toStrArr(val) {
+    if (!val)
+        return [];
+    if (Array.isArray(val))
+        return val.filter((v) => typeof v === "string");
+    const m = val;
+    if (typeof m.toArray === "function") {
+        try {
+            const a = m.toArray();
+            return Array.isArray(a) ? a.filter((v) => typeof v === "string") : [];
+        }
+        catch (_a) {
+            return [];
+        }
+    }
+    return [];
+}
+function probe(table, sym, expectedFile) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const pool = (0, pool_1.getWorkerPool)();
+        const { dense } = yield pool.encodeQuery(sym);
+        const prefix = PLATFORM_ROOT.endsWith("/") ? PLATFORM_ROOT : `${PLATFORM_ROOT}/`;
+        const where = `path LIKE '${(0, filter_builder_1.escapeSqlString)(prefix)}%'`;
+        const columns = ["id", "path", "chunk_index", "defined_symbols", "referenced_symbols"];
+        const vectorRows = (yield table.vectorSearch(dense).select([...columns, "_distance"]).where(where).limit(PRE_K).toArray());
+        let ftsRows = [];
+        try {
+            ftsRows = (yield table.search(sym).select([...columns, "_score"]).where(where).limit(PRE_K).toArray());
+        }
+        catch (_a) { }
+        const scores = new Map();
+        const docMap = new Map();
+        const keyOf = (d) => d.id || `${d.path}:${d.chunk_index}`;
+        vectorRows.forEach((d, r) => { const k = keyOf(d); docMap.set(k, d); scores.set(k, (scores.get(k) || 0) + 1 / (RRF_K + r + 1)); });
+        ftsRows.forEach((d, r) => { const k = keyOf(d); if (!docMap.has(k))
+            docMap.set(k, d); scores.set(k, (scores.get(k) || 0) + 1 / (RRF_K + r + 1)); });
+        const fusedKeys = Array.from(scores.entries()).sort((a, b) => b[1] - a[1]).map(([k]) => k);
+        const fused = fusedKeys.map((k) => docMap.get(k));
+        // Locate the expected definition chunk among retrieved rows.
+        const isExpected = (d) => String(d.path).toLowerCase().endsWith(`/${expectedFile.toLowerCase()}`) &&
+            toStrArr(d.defined_symbols).includes(sym);
+        let defRetrievalRank = -1; // rank within full union (by fusion order)
+        let defInPool = false;
+        fused.forEach((d, i) => {
+            if (defRetrievalRank < 0 && isExpected(d)) {
+                defRetrievalRank = i + 1;
+                defInPool = i < STAGE1_K;
+            }
+        });
+        // Was the def chunk retrieved at all (in the 500 union) even if low?
+        const defInUnion = fused.some(isExpected);
+        // If not in the union, query directly to confirm it exists in the index.
+        let defExistsInIndex = defInUnion;
+        if (!defInUnion) {
+            const direct = (yield table.query().select(columns).where(`${where} AND array_contains(defined_symbols, '${(0, filter_builder_1.escapeSqlString)(sym)}')`).limit(50).toArray());
+            defExistsInIndex = direct.some((d) => String(d.path).toLowerCase().endsWith(`/${expectedFile.toLowerCase()}`));
+        }
+        // ref->def reachability: among top-SEED_K fusion seeds, how many reference `sym`?
+        const seeds = fused.slice(0, SEED_K);
+        let seedsRefSym = 0;
+        let firstRefSeedRank = -1;
+        seeds.forEach((d, i) => {
+            if (toStrArr(d.referenced_symbols).includes(sym)) {
+                seedsRefSym++;
+                if (firstRefSeedRank < 0)
+                    firstRefSeedRank = i + 1;
+            }
+        });
+        const loc = defRetrievalRank > 0
+            ? (defInPool ? `pool#${defRetrievalRank}` : `union#${defRetrievalRank}(>200)`)
+            : (defExistsInIndex ? "OUTSIDE-500" : "NOT-IN-INDEX?");
+        return { sym, expectedFile, loc, seedsRefSym, firstRefSeedRank };
+    });
+}
+function main() {
+    return __awaiter(this, void 0, void 0, function* () {
+        const db = new vector_db_1.VectorDB(config_1.PATHS.lancedbDir);
+        const table = yield db.ensureTable();
+        console.log("sym             expectedDefChunk         seeds_ref  firstRefSeed");
+        for (const [sym, file] of CASES) {
+            const r = yield probe(table, sym, file);
+            console.log(`${r.sym.padEnd(15)} ${r.loc.padEnd(24)} ${String(r.seedsRefSym).padStart(3)}/20    ${r.firstRefSeedRank > 0 ? `rank${r.firstRefSeedRank}` : "-"}`);
+        }
+        yield db.close();
+        yield (0, exit_1.gracefulExit)(0);
+    });
+}
+main().catch((e) => { console.error(e); process.exit(1); });

package/dist/eval-oss.js

@@ -133,25 +133,44 @@ const DATASETS = {
         cases: PLATFORM_CASES,
     },
 };
-// A chunk matches when its file path ends with the expected file AND the
-// expected line falls within [start_line, end_line]. end_line falls back
-// to start_line + 200 when missing (lodash's mega-IIFE chunks are
-// typically much smaller than this, but the guard avoids false negatives
-// from short tail chunks).
-function chunkMatches(chunk, expectedFile, expectedLine) {
+// A chunk matches when its file path ends with the expected file AND either:
+//   (b) it declares the queried symbol (`defined_symbols` includes it), OR
+//   (a) the expected line falls within [start_line, end_line].
+//
+// (b) is the primary, drift-robust criterion for these symbol-lookup cases: it
+// credits the searcher for surfacing the chunk that *defines* the symbol,
+// regardless of where the hand-curated `expectedLine` lands relative to
+// post-reindex chunk boundaries. Before this, stale expectedLine values (e.g.
+// `requireAuth` def moved to lines 57-76 but the case said 45) and one-line
+// boundary off-by-ones (BeyondError chunk starts at line 37; the line check
+// tested `36 >= 37`) scored 7/15 platform cases as misses even though the
+// defining chunk was returned at ranks 1-3 — masking real recall (0.333 → 0.800).
+// (a) is kept as a fallback for re-export / binding-site cases (express `query`,
+// lodash `merge`) whose answer chunk legitimately doesn't carry the symbol in
+// `defined_symbols`. end_line falls back to start_line + 200 when missing.
+function chunkMatches(chunk, expectedFile, expectedLine, expectedSymbol) {
     var _a, _b, _c, _d, _e, _f;
     const path = String(((_a = chunk.metadata) === null || _a === void 0 ? void 0 : _a.path) || "").toLowerCase();
     if (!path.endsWith(`/${expectedFile.toLowerCase()}`) && !path.endsWith(expectedFile.toLowerCase())) {
         return false;
     }
+    // (b) defining-chunk match
+    if (expectedSymbol) {
+        const defs = chunk.defined_symbols;
+        if (Array.isArray(defs) && defs.includes(expectedSymbol))
+            return true;
+    }
+    // (a) line-range match — chunks are 0-indexed start_line; expected line is 1-indexed
     const start = Number((_d = (_c = (_b = chunk.generated_metadata) === null || _b === void 0 ? void 0 : _b.start_line) !== null && _c !== void 0 ? _c : chunk.start_line) !== null && _d !== void 0 ? _d : 0);
     const numLines = Number((_f = (_e = chunk.generated_metadata) === null || _e === void 0 ? void 0 : _e.num_lines) !== null && _f !== void 0 ? _f : 0);
     const end = numLines > 0 ? start + numLines : start + 200;
-    // chunks are 0-indexed start_line; expected line is 1-indexed
     return expectedLine - 1 >= start && expectedLine - 1 <= end;
 }
 function evaluateOss(response, c, timeMs) {
-    const idx = response.data.findIndex((chunk) => chunkMatches(chunk, c.expectedFile, c.expectedLine));
+    const idx = response.data.findIndex((chunk) => 
+    // `query` is the symbol name (sverklo P1 shape), so it doubles as the
+    // expected defined-symbol for the drift-robust match branch.
+    chunkMatches(chunk, c.expectedFile, c.expectedLine, c.query));
     const rank = idx + 1; // 0 = miss
     const rr = rank > 0 ? 1 / rank : 0;
     const recall10 = rank > 0 && rank <= 10 ? 1 : 0;

package/dist/lib/search/searcher.js

@@ -10,12 +10,40 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.Searcher = void 0;
+exports.asSymbolQuery = asSymbolQuery;
 exports.buildWhereClause = buildWhereClause;
 const config_1 = require("../../config");
 const filter_builder_1 = require("../utils/filter-builder");
 const pool_1 = require("../workers/pool");
 const intent_1 = require("./intent");
 const pagerank_1 = require("./pagerank");
+// Reads a defined_symbols / referenced_symbols column that may arrive as a plain
+// array or a LanceDB Arrow proxy (.toArray()).
+function readSymbolArray(val) {
+    if (!val)
+        return [];
+    if (Array.isArray(val))
+        return val.filter((v) => typeof v === "string");
+    const maybe = val;
+    if (typeof maybe.toArray === "function") {
+        try {
+            const a = maybe.toArray();
+            return Array.isArray(a) ? a.filter((v) => typeof v === "string") : [];
+        }
+        catch (_a) {
+            return [];
+        }
+    }
+    return [];
+}
+// A query that is a single bare identifier ("BeyondError", "requireAuth", "map")
+// is almost always a symbol lookup — the user wants the chunk that *defines*
+// that symbol. Returns the trimmed identifier, or null for natural-language
+// queries. Drives the symbol-definition promotion in search().
+function asSymbolQuery(query) {
+    const q = query.trim();
+    return /^[A-Za-z_$][A-Za-z0-9_$]*$/.test(q) ? q : null;
+}
 function buildWhereClause(pathPrefix, filters, searchIntent) {
     var _a;
     const parts = [];
@@ -339,7 +367,7 @@ class Searcher {
     }
     search(query, top_k, _search_options, _filters, pathPrefix, intent, signal) {
         return __awaiter(this, void 0, void 0, function* () {
-            var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l;
+            var _a, _b, _c, _d, _e, _f, _g, _h, _j, _k, _l, _m;
             const finalLimit = top_k !== null && top_k !== void 0 ? top_k : 10;
             // ColBERT rerank is opt-in as of v0.17.1. On the 97-case eval it
             // regresses MRR@10 by ~3% and doubles query latency; sweep across
@@ -348,6 +376,8 @@ class Searcher {
             let doRerank = (_a = _search_options === null || _search_options === void 0 ? void 0 : _search_options.rerank) !== null && _a !== void 0 ? _a : false;
             const explain = (_b = _search_options === null || _search_options === void 0 ? void 0 : _search_options.explain) !== null && _b !== void 0 ? _b : false;
             const searchIntent = intent || (0, intent_1.detectIntent)(query);
+            // Bare-identifier queries get symbol-definition promotion (see below).
+            const symbolQuery = asSymbolQuery(query);
             const pool = (0, pool_1.getWorkerPool)();
             if (signal === null || signal === void 0 ? void 0 : signal.aborted) {
                 const err = new Error("Aborted");
@@ -372,7 +402,7 @@ class Searcher {
             try {
                 table = yield this.db.ensureTable();
             }
-            catch (_m) {
+            catch (_o) {
                 return { data: [] };
             }
             // Ensure FTS index exists (lazy init, retry periodically on failure)
@@ -396,11 +426,14 @@ class Searcher {
             // it in the lightweight path only when the flag is on so we don't bloat the
             // default query path.
             const pagerankEnabled = process.env.GMAX_PAGERANK === "1" && !!pathPrefix;
+            // Symbol-definition promotion needs defined_symbols on every candidate, not
+            // just the final display set — load it for bare-symbol queries too.
+            const needDefinedSymbols = pagerankEnabled || symbolQuery !== null;
             const LIGHTWEIGHT_COLUMNS = [
                 "id", "path", "hash", "chunk_index", "start_line", "end_line",
                 "is_anchor", "chunk_type", "role", "complexity", "is_exported",
                 "content", "parent_symbol", "referenced_symbols", "pooled_colbert_48d",
-                ...(pagerankEnabled ? ["defined_symbols"] : []),
+                ...(needDefinedSymbols ? ["defined_symbols"] : []),
             ];
             // _distance is auto-added by vectorSearch, _score by FTS — include each
             // in the respective query to suppress LanceDB deprecation warnings
@@ -438,7 +471,7 @@ class Searcher {
                             this.ftsAvailable = true;
                             console.warn("[Searcher] Rebuilt FTS index with position support — retry search");
                         }
-                        catch (_o) { }
+                        catch (_p) { }
                     }
                     else {
                         console.warn(`[Searcher] FTS search failed (will retry later): ${msg}`);
@@ -552,6 +585,30 @@ class Searcher {
                 return { data: [] };
             }
             const rerankCandidates = stage2Candidates.slice(0, RERANK_TOP);
+            // Symbol-definition promotion (1/2): membership. For a bare-symbol query,
+            // ensure the chunk(s) that actually DEFINE the symbol reach the rerank set
+            // even when the cosine / RERANK_TOP cuts would drop them — e.g. ErrorCodes
+            // sits at pooled-cosine rank 24 (> RERANK_TOP=20) and resolveActor at fusion
+            // rank 91 (> the stage-2 cut). Pulled from the top-200 fusion pool, bounded
+            // so the rerank batch stays small. Must run before Phase B so the injected
+            // chunks get their colbert data fetched for reranking. The score boost in
+            // (2/2) below then lets them win dedup over their own method-child chunks.
+            if (symbolQuery && rerankCandidates.length > 0) {
+                const present = new Set(rerankCandidates.map((d) => d.id).filter(Boolean));
+                const MAX_INJECT = 5;
+                let injected = 0;
+                for (const d of topCandidates) {
+                    if (injected >= MAX_INJECT)
+                        break;
+                    if (!d.id || present.has(d.id))
+                        continue;
+                    if (readSymbolArray(d.defined_symbols).includes(symbolQuery)) {
+                        rerankCandidates.push(d);
+                        present.add(d.id);
+                        injected++;
+                    }
+                }
+            }
             // Phase B: Lazy-load colbert data only for the ~20 rerank candidates
             if (doRerank && rerankCandidates.length > 0) {
                 const rerankIds = rerankCandidates
@@ -599,13 +656,25 @@ class Searcher {
                     // Small tie-breaker so later items don't all share 0
                     return fusedScore || 1 / (idx + 1);
                 });
+            // Symbol-definition promotion (2/2): score. Multiplicatively boost any
+            // candidate that defines the queried symbol so the definition chunk outranks
+            // its own method-child chunks (e.g. the `BeyondError` class chunk vs its
+            // constructor/toJSON, which otherwise score higher on the literal and evict
+            // the parent in overlap dedup). Multiplicative keeps it scale-invariant
+            // across the rerank-on (ColBERT maxsim) and rerank-off (fusion) score ranges.
+            const envDefBoost = Number.parseFloat((_k = process.env.GMAX_DEF_BOOST) !== null && _k !== void 0 ? _k : "");
+            const DEF_MATCH_BOOST = Number.isFinite(envDefBoost) && envDefBoost >= 1 ? envDefBoost : 5;
             const scored = rerankCandidates.map((doc, idx) => {
                 var _a, _b;
                 const base = (_a = scores === null || scores === void 0 ? void 0 : scores[idx]) !== null && _a !== void 0 ? _a : 0;
                 const key = doc.id || `${doc.path}:${doc.chunk_index}`;
                 const fusedScore = (_b = candidateScores.get(key)) !== null && _b !== void 0 ? _b : 0;
                 const blended = base + FUSED_WEIGHT * fusedScore;
-                const boosted = this.applyStructureBoost(doc, blended, searchIntent);
+                let boosted = this.applyStructureBoost(doc, blended, searchIntent);
+                if (symbolQuery &&
+                    readSymbolArray(doc.defined_symbols).includes(symbolQuery)) {
+                    boosted *= DEF_MATCH_BOOST;
+                }
                 return {
                     record: doc,
                     score: boosted,
@@ -622,7 +691,7 @@ class Searcher {
                 try {
                     const { scores: prScores, max: prMax } = yield (0, pagerank_1.loadOrComputePageRank)(this.db, pathPrefix);
                     if (prMax > 0) {
-                        const envWeight = Number.parseFloat((_k = process.env.GMAX_PR_WEIGHT) !== null && _k !== void 0 ? _k : "");
+                        const envWeight = Number.parseFloat((_l = process.env.GMAX_PR_WEIGHT) !== null && _l !== void 0 ? _l : "");
                         const PR_WEIGHT = Number.isFinite(envWeight) && envWeight >= 0 ? envWeight : 0.05;
                         for (const item of scored) {
                             const raw = item.record.defined_symbols;
@@ -637,7 +706,7 @@ class Searcher {
                                         defs = arr.filter((v) => typeof v === "string");
                                     }
                                 }
-                                catch (_p) { }
+                                catch (_q) { }
                             }
                             const norm = (0, pagerank_1.pageRankBoostForSymbols)(defs, prScores, prMax);
                             item.score += PR_WEIGHT * norm;
@@ -655,7 +724,7 @@ class Searcher {
             // Item 10: Per-file diversification
             const seenFiles = new Map();
             const diversified = [];
-            const envMaxPerFile = Number.parseInt((_l = process.env.GMAX_MAX_PER_FILE) !== null && _l !== void 0 ? _l : "", 10);
+            const envMaxPerFile = Number.parseInt((_m = process.env.GMAX_MAX_PER_FILE) !== null && _m !== void 0 ? _m : "", 10);
             const MAX_PER_FILE = Number.isFinite(envMaxPerFile) && envMaxPerFile > 0 ? envMaxPerFile : 3;
             for (const item of uniqueScored) {
                 const path = item.record.path || "";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "grepmax",
-  "version": "0.17.7",
+  "version": "0.17.9",
   "author": "Robert Owens <78518764+reowens@users.noreply.github.com>",
   "homepage": "https://github.com/reowens/grepmax",
   "bugs": {

package/plugins/grepmax/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "grepmax",
-  "version": "0.17.7",
+  "version": "0.17.9",
   "description": "Semantic code search for Claude Code. Automatically indexes your project and provides intelligent search capabilities.",
   "author": {
     "name": "Robert Owens",