grepmax 0.17.23 → 0.18.0

package/dist/config.js

@@ -33,7 +33,9 @@ var __importStar = (this && this.__importStar) || (function () {
     };
 })();
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.INDEXABLE_EXTENSIONS = exports.FRAGMENT_COMPACT_THRESHOLD = exports.DISK_LOW_BYTES = exports.DISK_CRITICAL_BYTES = exports.MAX_FILE_SIZE_BYTES = exports.PATHS = exports.MAX_WORKER_MEMORY_MB = exports.WORKER_BOOT_TIMEOUT_MS = exports.WORKER_TIMEOUT_MS = exports.CONFIG = exports.MODEL_IDS = exports.DEFAULT_MODEL_TIER = exports.MODEL_TIERS = void 0;
+exports.INDEXABLE_EXTENSIONS = exports.FRAGMENT_COMPACT_THRESHOLD = exports.DISK_LOW_BYTES = exports.DISK_CRITICAL_BYTES = exports.MAX_FILE_SIZE_BYTES = exports.PATHS = exports.MAX_WORKER_MEMORY_MB = exports.WORKER_BOOT_TIMEOUT_MS = exports.WORKER_TIMEOUT_MS = exports.CHUNKER_VERSION_HISTORY = exports.CONFIG = exports.MODEL_IDS = exports.DEFAULT_MODEL_TIER = exports.MODEL_TIERS = void 0;
+exports.describeChunkerGap = describeChunkerGap;
+exports.describeEmbeddingGap = describeEmbeddingGap;
 const os = __importStar(require("node:os"));
 const path = __importStar(require("node:path"));
 exports.MODEL_TIERS = {
@@ -77,11 +79,84 @@ exports.CONFIG = {
     WORKER_THREADS: DEFAULT_WORKER_THREADS,
     QUERY_PREFIX: "",
     // Bump when chunk metadata semantics change in a way that requires a full
-    // reindex to take effect. v2: sub-chunk symbol lists scoped to the chunk's
-    // own content (previously every split sub-chunk inherited the parent's
-    // full defined/referenced symbol lists, fabricating graph edges).
-    CHUNKER_VERSION: 2,
+    // reindex to take effect. Must equal the latest entry's `v` in
+    // CHUNKER_VERSION_HISTORY below — see that list for per-version severity and
+    // the user-facing note rendered by `gmax doctor` and the staleness hint.
+    CHUNKER_VERSION: 3,
 };
+/**
+ * Per-version record of what changed in the chunker and how much it matters to
+ * an already-built index. `severity` drives tone: an "additive" change only
+ * adds new metadata (older indexes under-cover but aren't wrong), while a
+ * "breaking" change means older indexes carry incorrect metadata until a
+ * reindex. The `note` is shown verbatim to the user for the versions their
+ * index is missing. CONFIG.CHUNKER_VERSION must equal the highest `v` here.
+ */
+exports.CHUNKER_VERSION_HISTORY = [
+    {
+        v: 2,
+        severity: "breaking",
+        note: "sub-chunk symbol scoping; graph overcounted callers before this.",
+    },
+    {
+        v: 3,
+        severity: "additive",
+        note: "type-position edges; dead/trace miss type-only callers until reindex.",
+    },
+];
+/**
+ * Describe the gap between an index's stamped chunker version and the current
+ * one, or null when the index is already current. Shared by `gmax doctor` and
+ * the query-time staleness hint so both render the same severity + notes.
+ */
+function describeChunkerGap(indexedVersion) {
+    const fromVersion = indexedVersion !== null && indexedVersion !== void 0 ? indexedVersion : 1;
+    if (fromVersion >= exports.CONFIG.CHUNKER_VERSION)
+        return null;
+    const missed = exports.CHUNKER_VERSION_HISTORY.filter((h) => h.v > fromVersion && h.v <= exports.CONFIG.CHUNKER_VERSION);
+    const severity = missed.some((h) => h.severity === "breaking")
+        ? "breaking"
+        : "additive";
+    return {
+        fromVersion,
+        toVersion: exports.CONFIG.CHUNKER_VERSION,
+        severity,
+        notes: missed.map((h) => h.note),
+    };
+}
+/**
+ * Describe the gap between an index's stored embedding identity (model tier +
+ * vector dim) and the current global config, or null when they already agree.
+ *
+ * A dimension change is `breaking`: vectors of differing widths cannot be
+ * compared, and the single fixed-dim `chunks` table silently pads/truncates a
+ * mismatched query embedding, yielding meaningless scores. A same-width model
+ * swap is `additive`: the stored vectors are a different model's output but the
+ * same width, so search keeps working with mixed-model quality until a re-embed.
+ *
+ * Mirrors describeChunkerGap so `gmax doctor` and the query-time hint render the
+ * same severity. Pure over MODEL_TIERS — callers pass the current identity (read
+ * from the global config) so this module stays free of config I/O.
+ */
+function describeEmbeddingGap(stored, current) {
+    var _a, _b, _c, _d, _e, _f, _g, _h;
+    const fromModel = (_a = stored.modelTier) !== null && _a !== void 0 ? _a : exports.DEFAULT_MODEL_TIER;
+    const toModel = (_b = current.modelTier) !== null && _b !== void 0 ? _b : exports.DEFAULT_MODEL_TIER;
+    const fromDim = (_e = (_c = stored.vectorDim) !== null && _c !== void 0 ? _c : (_d = exports.MODEL_TIERS[fromModel]) === null || _d === void 0 ? void 0 : _d.vectorDim) !== null && _e !== void 0 ? _e : exports.CONFIG.VECTOR_DIM;
+    const toDim = (_h = (_f = current.vectorDim) !== null && _f !== void 0 ? _f : (_g = exports.MODEL_TIERS[toModel]) === null || _g === void 0 ? void 0 : _g.vectorDim) !== null && _h !== void 0 ? _h : exports.CONFIG.VECTOR_DIM;
+    const dimChanged = fromDim !== toDim;
+    const modelChanged = fromModel !== toModel;
+    if (!dimChanged && !modelChanged)
+        return null;
+    return {
+        fromModel,
+        toModel,
+        fromDim,
+        toDim,
+        dimChanged,
+        severity: dimChanged ? "breaking" : "additive",
+    };
+}
 exports.WORKER_TIMEOUT_MS = Number.parseInt(process.env.GMAX_WORKER_TIMEOUT_MS || "60000", 10);
 exports.WORKER_BOOT_TIMEOUT_MS = Number.parseInt(process.env.GMAX_WORKER_BOOT_TIMEOUT_MS || "300000", 10);
 exports.MAX_WORKER_MEMORY_MB = (() => {

package/dist/eval-graph-nav.js

@@ -0,0 +1,374 @@
+"use strict";
+/**
+ * Navigation-precision fixture — the measurement gate for the type-position
+ * reference cluster (chunker type-position edges, Phase 10 graph-distance,
+ * `gmax dead` precision). Both standing plans defer that cluster behind one
+ * unmet condition: a `trace --inbound` / `dead` truth set that EXISTS and
+ * MOVES. This is that instrument.
+ *
+ * WHAT IT MEASURES. The call graph is built from tree-sitter call-expression
+ * captures, so `referenced_symbols` holds call/construct sites (`foo()`,
+ * `new Foo()`) but NOT type-position uses (`: Foo`, `<Foo>`, `extends Foo`,
+ * `x as Foo`). `gmax trace --inbound Foo` and `gmax dead Foo` both read that
+ * graph, so a type used purely in annotations looks like it has no callers —
+ * trace under-reports, dead false-positives. This harness quantifies the gap by
+ * contrast: call-position recall (the baseline the graph achieves on the shapes
+ * it does capture) vs type-position recall (≈0 today).
+ *
+ * IT IS A GAUGE, NOT A PASS/FAIL GATE. Call-position recall is already <100%
+ * (chunk rollup, uncaptured shapes) — that is the status quo, not a regression,
+ * so the run always exits 0. The load-bearing number is type-position recall:
+ * low today = standing evidence to build chunker type-position edges; re-run
+ * after building them to confirm it moved. (For a true regression gate on the
+ * call graph, see the caller-count guard in eval-graph-sanity.ts.)
+ *
+ * HOW IT STAYS HONEST (no rot). Ground truth is derived live: for each symbol,
+ * `git grep` every reference and classify each file as a call-position caller or
+ * a type-position-only caller; `getCallers` is the measured set. The only
+ * hand-annotation is each symbol's character (callable vs type-only). The
+ * harness excludes its own file from both grep and the graph so listing the
+ * fixture symbols here can't contaminate the measurement. Run after a reindex.
+ *
+ * Usage:
+ *   npx tsx src/eval-graph-nav.ts            # table output
+ *   npx tsx src/eval-graph-nav.ts --json     # machine-readable
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var _a;
+var _b;
+Object.defineProperty(exports, "__esModule", { value: true });
+(_a = (_b = process.env).GMAX_WORKER_COUNT) !== null && _a !== void 0 ? _a : (_b.GMAX_WORKER_COUNT = "1");
+const node_child_process_1 = require("node:child_process");
+const path = __importStar(require("node:path"));
+const config_1 = require("./config");
+const graph_builder_1 = require("./lib/graph/graph-builder");
+const vector_db_1 = require("./lib/store/vector-db");
+const exit_1 = require("./lib/utils/exit");
+const filter_builder_1 = require("./lib/utils/filter-builder");
+const GMAX_ROOT = path.resolve(__dirname, "..");
+const SELF = "src/eval-graph-nav.ts"; // exclude from grep + graph (self-reference)
+// Files that embed the fixture symbols as *string* data (template-literal
+// source, `.toContain("…")` assertions) rather than real type/call references.
+// grep can't tell those apart from genuine uses, so they would manufacture
+// phantom "expected callers" that no correct graph ever surfaces. Excluded from
+// both the grep truth and the getCallers set so the exclusion stays symmetric.
+const FIXTURE_FILES = new Set([SELF, "tests/graph-edges.type-position.test.ts"]);
+const rel = (p) => p.replace(`${GMAX_ROOT}/`, "");
+const INBOUND_SYMBOLS = [
+    // ── Baselines: called/constructed, so the graph captures them. ──────────────
+    {
+        symbol: "getWorkerPool",
+        character: "callable",
+        note: "plain function call sites",
+    },
+    {
+        symbol: "GraphBuilder",
+        character: "callable",
+        note: "class via `new GraphBuilder(...)`",
+    },
+    {
+        symbol: "withQueryTimeout",
+        character: "callable",
+        note: "wrapper function call sites",
+    },
+    {
+        symbol: "isFileCached",
+        character: "callable",
+        note: "predicate function call sites",
+    },
+    // ── The gap: interfaces / type aliases used only in `: T` / `<T>` position. ──
+    {
+        symbol: "SearchResponse",
+        character: "type-only",
+        note: "return/param annotation across search + eval",
+    },
+    {
+        symbol: "VectorRecord",
+        character: "type-only",
+        note: "row interface annotated across index/store/worker",
+    },
+    {
+        symbol: "GraphNode",
+        character: "type-only",
+        note: "graph row interface in dead/formatter consumers",
+    },
+    {
+        symbol: "NeighborHit",
+        character: "type-only",
+        note: "traversal interface, type position only",
+    },
+    {
+        symbol: "CallerTree",
+        character: "type-only",
+        note: "recursive caller interface, type position only",
+    },
+    {
+        symbol: "EdgeDirection",
+        character: "type-only",
+        note: "string-literal type alias, pure type position",
+    },
+    // ── Python (mlx-embed-server/server.py): proves Shape 6 closes the gap in a
+    //    grammar with no `type_identifier` node. ─────────────────────────────────
+    {
+        symbol: "EmbedResponse",
+        character: "callable",
+        note: "Python: Pydantic model via `EmbedResponse(...)` construct site",
+    },
+    {
+        symbol: "EmbedRequest",
+        character: "type-only",
+        note: "Python: Pydantic model in `request: EmbedRequest` annotation only",
+    },
+];
+// Dead-precision truth is derived (LIVE iff grep finds any use beyond def/import).
+const DEAD_SYMBOLS = [
+    "getWorkerPool", // LIVE baseline (call users) — dead must read LIVE
+    "GraphBuilder", // LIVE baseline (construct users)
+    "DeadResult", // non-exported, type-only users → today DEAD (false positive)
+    "EdgeDirection", // exported, type-only users → today PUBLIC_EXPORT (masked)
+    "ResolvedCaller", // exported, type-only users → today PUBLIC_EXPORT (masked)
+    "EmbedRequest", // Python: annotation-only Pydantic model — the canonical false-dead, now LIVE via Shape 6
+];
+function classify(symbol, text) {
+    const s = symbol.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    if (/\bimport\b/.test(text))
+        return "import";
+    if (new RegExp(`\\b(interface|type|class|function|enum)\\s+${s}\\b`).test(text))
+        return "def";
+    if (new RegExp(`\\b(const|let|var)\\s+${s}\\b\\s*[:=]`).test(text))
+        return "def";
+    if (new RegExp(`(\\bnew\\s+${s}\\b|\\b${s}\\s*\\()`).test(text))
+        return "call";
+    if (new RegExp(`([:<,|&(]\\s*${s}\\b|\\bextends\\s+${s}\\b|\\bimplements\\s+${s}\\b|\\bas\\s+${s}\\b|\\b${s}\\s*[[<])`).test(text))
+        return "type";
+    return "other";
+}
+/**
+ * Live ground truth for `symbol`: files that reference it from a call position
+ * vs a type-only position. Definition-only / import-only files are not callers.
+ * A file with any call site counts as a call caller even if it also uses the
+ * symbol as a type (the graph should find it via the call). The harness's own
+ * file is excluded so the fixture list can't contaminate the truth.
+ */
+function referenceTruth(symbol) {
+    let raw;
+    try {
+        raw = (0, node_child_process_1.execFileSync)("git", [
+            "grep",
+            "-nwF",
+            symbol,
+            "--",
+            "*.ts",
+            "*.py",
+            ...[...FIXTURE_FILES].map((f) => `:!${f}`),
+        ], { cwd: GMAX_ROOT, encoding: "utf-8" });
+    }
+    catch (_a) {
+        return { callFiles: new Set(), typeOnlyFiles: new Set() };
+    }
+    const callFiles = new Set();
+    const typeFiles = new Set();
+    for (const line of raw.split("\n")) {
+        if (!line)
+            continue;
+        const parts = line.split(":");
+        const file = parts[0];
+        const text = parts.slice(2).join(":");
+        const kind = classify(symbol, text);
+        if (kind === "call")
+            callFiles.add(file);
+        else if (kind === "type")
+            typeFiles.add(file);
+    }
+    const typeOnlyFiles = new Set([...typeFiles].filter((f) => !callFiles.has(f)));
+    return { callFiles, typeOnlyFiles };
+}
+/** getCallers file set, excluding the harness's own (possibly stale) chunks. */
+function callerFiles(builder, symbol) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const callers = yield builder.getCallers(symbol);
+        return new Set(callers.map((g) => rel(g.file)).filter((f) => !FIXTURE_FILES.has(f)));
+    });
+}
+function recall(found, expected) {
+    let hit = 0;
+    for (const f of expected)
+        if (found.has(f))
+            hit++;
+    return { hit, total: expected.size };
+}
+function runInbound(builder) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const out = [];
+        for (const c of INBOUND_SYMBOLS) {
+            const { callFiles, typeOnlyFiles } = referenceTruth(c.symbol);
+            const found = yield callerFiles(builder, c.symbol);
+            out.push({
+                symbol: c.symbol,
+                character: c.character,
+                call: recall(found, callFiles),
+                type: recall(found, typeOnlyFiles),
+                missedType: [...typeOnlyFiles].filter((f) => !found.has(f)),
+                note: c.note,
+            });
+        }
+        return out;
+    });
+}
+function runDead(db, builder) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const table = yield db.ensureTable();
+        const out = [];
+        for (const symbol of DEAD_SYMBOLS) {
+            const { callFiles, typeOnlyFiles } = referenceTruth(symbol);
+            const trueLive = callFiles.size + typeOnlyFiles.size > 0;
+            const defRows = yield table
+                .query()
+                .select(["is_exported"])
+                .where(`array_contains(defined_symbols, '${(0, filter_builder_1.escapeSqlString)(symbol)}')`)
+                .limit(1)
+                .toArray();
+            const found = yield callerFiles(builder, symbol);
+            let status;
+            if (defRows.length === 0)
+                status = "NO_DEF";
+            else if (found.size > 0)
+                status = "LIVE";
+            else
+                status = defRows[0].is_exported ? "PUBLIC_EXPORT" : "DEAD";
+            let verdict;
+            if (trueLive && status === "LIVE")
+                verdict = "correct";
+            else if (!trueLive && status === "DEAD")
+                verdict = "correct";
+            else if (trueLive && status === "DEAD")
+                verdict = "false-dead";
+            else if (trueLive && status === "PUBLIC_EXPORT")
+                verdict = "masked-by-export";
+            else
+                verdict = "over-live";
+            out.push({ symbol, trueLive, status, verdict });
+        }
+        return out;
+    });
+}
+function pool(rows) {
+    const hit = rows.reduce((a, r) => a + r.hit, 0);
+    const total = rows.reduce((a, r) => a + r.total, 0);
+    return { hit, total, pct: total === 0 ? 0 : (100 * hit) / total };
+}
+function main() {
+    return __awaiter(this, void 0, void 0, function* () {
+        const jsonMode = process.argv.includes("--json") || process.env.GMAX_EVAL_JSON === "1";
+        const db = new vector_db_1.VectorDB(config_1.PATHS.lancedbDir);
+        const builder = new graph_builder_1.GraphBuilder(db, GMAX_ROOT);
+        const inbound = yield runInbound(builder);
+        const dead = yield runDead(db, builder);
+        yield db.close();
+        const callBaseline = pool(inbound.filter((r) => r.character === "callable").map((r) => r.call));
+        const typeGap = pool(inbound.filter((r) => r.character === "type-only").map((r) => r.type));
+        const falseDead = dead.filter((r) => r.verdict === "false-dead").length;
+        const maskedByExport = dead.filter((r) => r.verdict === "masked-by-export").length;
+        const summary = {
+            callPositionRecall: callBaseline,
+            typePositionRecall: typeGap,
+            deadFalsePositives: falseDead,
+            deadMaskedByExport: maskedByExport,
+        };
+        if (jsonMode) {
+            process.stdout.write(`${JSON.stringify({ summary, inbound, dead }, null, 2)}\n`);
+            yield (0, exit_1.gracefulExit)(0);
+            return;
+        }
+        const pct = (r) => r.total === 0
+            ? "  n/a"
+            : `${Math.round((100 * r.hit) / r.total)}%`.padStart(4);
+        console.log(`Navigation-precision fixture (trace --inbound / dead)`);
+        console.log(`root: ${GMAX_ROOT}\n`);
+        console.log(`Inbound caller recall (getCallers vs grep truth, by reference position):`);
+        for (const r of inbound) {
+            const tag = r.character === "callable" ? "callable " : "type-only";
+            console.log(`  ${r.symbol.padEnd(16)} [${tag}] ` +
+                `call ${pct(r.call)} (${String(r.call.hit).padStart(2)}/${String(r.call.total).padStart(2)})  ` +
+                `type ${pct(r.type)} (${String(r.type.hit).padStart(2)}/${String(r.type.total).padStart(2)})` +
+                (r.character === "type-only" && r.missedType.length
+                    ? `  misses: ${r.missedType.slice(0, 3).join(", ")}${r.missedType.length > 3 ? ", …" : ""}`
+                    : ""));
+        }
+        console.log(`\nDead-code precision (truth derived from grep):`);
+        for (const r of dead) {
+            const flag = r.verdict === "false-dead"
+                ? "  ← FALSE POSITIVE (really LIVE)"
+                : r.verdict === "masked-by-export"
+                    ? "  ← type-only users hidden behind PUBLIC EXPORT"
+                    : "";
+            console.log(`  ${r.symbol.padEnd(16)} truth=${(r.trueLive ? "LIVE" : "DEAD").padEnd(4)}  reported=${r.status.padEnd(13)}${flag}`);
+        }
+        console.log(`\nHeadline` +
+            `\n  call-position recall (baseline): ${callBaseline.hit}/${callBaseline.total} (${Math.round(callBaseline.pct)}%)` +
+            `\n  type-position recall (the gap):  ${typeGap.hit}/${typeGap.total} (${Math.round(typeGap.pct)}%)` +
+            `\n  dead false-positives: ${falseDead}   ·   dead masked-by-export: ${maskedByExport}`);
+        const delta = Math.round(callBaseline.pct - typeGap.pct);
+        if (callBaseline.pct < 40) {
+            console.log(`\nVerdict: INCONCLUSIVE — call-position baseline is unexpectedly low (${Math.round(callBaseline.pct)}%); ` +
+                `the index may be stale. Reindex (gmax index) and re-run.`);
+        }
+        else if (typeGap.pct < 50) {
+            console.log(`\nVerdict: GAP OPEN (${delta} pts) — navigation recovers call sites at ${Math.round(callBaseline.pct)}% ` +
+                `but type-position references at only ${Math.round(typeGap.pct)}%. Standing evidence to build chunker ` +
+                `type-position edges; re-run after to confirm the gap closes.`);
+        }
+        else {
+            console.log(`\nVerdict: GAP CLOSING — type-position recall is up to ${Math.round(typeGap.pct)}% ` +
+                `(call baseline ${Math.round(callBaseline.pct)}%).`);
+        }
+        yield (0, exit_1.gracefulExit)(0);
+    });
+}
+if (require.main === module) {
+    main().catch((e) => {
+        console.error(e);
+        process.exit(1);
+    });
+}