@davesheffer/hunch 0.21.1 → 0.23.0

package/dist/cli/index.js

@@ -42,6 +42,7 @@ import { healClaudeConfigCaseSplit } from "../integrations/claudeConfig.js";
 import { formatContext } from "../core/format.js";
 import { readConfig, writeConfig, FIRMNESS_LEVELS, isFirmness } from "../core/config.js";
 import { blockingInScope, vetoInScope, proposedEditLines } from "../core/hookpolicy.js";
+import { loadGoldenSet, evaluateGraphLift } from "../eval/harness.js";
 import { draftTripwires, knownRepoDeps } from "../synthesis/tripwires.js";
 import { constraintId } from "../core/ids.js";
 import { readManifest, writeManifest, SCHEMA_VERSION } from "../core/migrate.js";
@@ -409,6 +410,49 @@ program
     }
     store.close();
 });
+// ---- eval (retrieval quality; measures the graph-stream lift) --------------
+program
+    .command("eval")
+    .description("Measure retrieval quality (Recall@k, MRR) over a golden set, and A/B the dependency-graph stream.")
+    .requiredOption("--file <path>", "golden set JSON: [{ query, expected: [refs], note? }]")
+    .option("--k <n>", "top-k cutoff", "10")
+    .option("--semantic", "also blend the semantic stream (requires `hunch embed`; default is deterministic FTS + graph)")
+    .action(async (opts) => {
+    const { store } = storeFor();
+    store.reindex(); // reflect any out-of-band JSON edits before scoring
+    let cases;
+    try {
+        cases = loadGoldenSet(readFileSync(opts.file, "utf8"));
+    }
+    catch (e) {
+        store.close();
+        return fail(`could not load golden set: ${e.message}`);
+    }
+    if (!cases.length) {
+        store.close();
+        return fail("golden set is empty");
+    }
+    const k = Math.max(1, parseInt(opts.k, 10) || 10);
+    // Default is deterministic (FTS + graph, no model). --semantic only adds the
+    // semantic leg when embeddings actually exist; otherwise it's still FTS + graph.
+    const embedder = opts.semantic ? await selectEmbedder() : undefined;
+    const lift = await evaluateGraphLift(store, cases, { k, embedder });
+    const pct = (x) => `${(x * 100).toFixed(1)}%`;
+    const dpt = (x) => `${x >= 0 ? "+" : ""}${(x * 100).toFixed(1)}pt`;
+    const dnum = (x) => `${x >= 0 ? "+" : ""}${x.toFixed(3)}`;
+    console.log(`Eval over ${cases.length} case(s), k=${k}${opts.semantic ? " (semantic + graph + FTS)" : " (FTS + graph)"}\n`);
+    console.log(`                Recall@${k}    MRR      hit-rate`);
+    console.log(`  graph OFF     ${pct(lift.off.recallAtK).padStart(7)}    ${lift.off.mrr.toFixed(3)}    ${pct(lift.off.hitRate)}`);
+    console.log(`  graph ON      ${pct(lift.on.recallAtK).padStart(7)}    ${lift.on.mrr.toFixed(3)}    ${pct(lift.on.hitRate)}`);
+    console.log(`  graph LIFT    ${dpt(lift.recallDelta).padStart(7)}    ${dnum(lift.mrrDelta)}`);
+    const misses = lift.on.perCase.filter((c) => c.found === 0);
+    if (misses.length) {
+        console.log(`\n  ${misses.length} case(s) with no expected hit — curate or tune:`);
+        for (const m of misses.slice(0, 10))
+            console.log(`    · "${m.query}"`);
+    }
+    store.close();
+});
 // ---- embed (opt-in semantic search) ---------------------------------------
 program
     .command("embed")

package/dist/eval/harness.js

@@ -0,0 +1,55 @@
+/** Score a golden set: Recall@k, MRR, hit-rate. Deterministic when no embedder. */
+export async function evaluateRetrieval(store, cases, opts = {}) {
+    const k = opts.k ?? 10;
+    const perCase = [];
+    for (const c of cases) {
+        const hits = await store.hybridSearch(c.query, k, { embedder: opts.embedder, graphWeight: opts.graphWeight });
+        const top = hits.slice(0, k).map((h) => h.ref);
+        const expected = new Set(c.expected);
+        let found = 0;
+        let rr = 0;
+        top.forEach((ref, i) => {
+            if (!expected.has(ref))
+                return;
+            found++;
+            if (rr === 0)
+                rr = 1 / (i + 1); // first expected hit sets the reciprocal rank
+        });
+        perCase.push({
+            query: c.query,
+            expected: c.expected.length,
+            found,
+            recall: c.expected.length ? found / c.expected.length : 0,
+            rr,
+        });
+    }
+    const n = perCase.length || 1;
+    return {
+        n: perCase.length,
+        k,
+        recallAtK: perCase.reduce((s, r) => s + r.recall, 0) / n,
+        mrr: perCase.reduce((s, r) => s + r.rr, 0) / n,
+        hitRate: perCase.reduce((s, r) => s + (r.found > 0 ? 1 : 0), 0) / n,
+        perCase,
+    };
+}
+/** Compare graph-OFF vs graph-ON on the same golden set — the #1 lift measurement. */
+export async function evaluateGraphLift(store, cases, opts = {}) {
+    const off = await evaluateRetrieval(store, cases, { ...opts, graphWeight: 0 });
+    // graphWeight undefined -> hybridSearch uses the configured default; an explicit
+    // value tunes it. Either way "on" is whatever ships, "off" is the baseline.
+    const on = await evaluateRetrieval(store, cases, opts);
+    return { off, on, recallDelta: on.recallAtK - off.recallAtK, mrrDelta: on.mrr - off.mrr };
+}
+/** Parse + validate a golden-set JSON string (array of {query, expected[]}). */
+export function loadGoldenSet(raw) {
+    const data = JSON.parse(raw);
+    if (!Array.isArray(data))
+        throw new Error("golden set must be a JSON array of { query, expected[] }");
+    return data.map((c, i) => {
+        if (!c || typeof c.query !== "string" || !Array.isArray(c.expected))
+            throw new Error(`golden case ${i} must be { query: string, expected: string[] }`);
+        return { query: c.query, expected: c.expected.map(String), note: typeof c.note === "string" ? c.note : undefined };
+    });
+}
+//# sourceMappingURL=harness.js.map

package/dist/store/hunchStore.js

@@ -296,23 +296,35 @@ export class HunchStore {
      *  the lean install and fallback regressions are unaffected. Pass
      *  `embedder: null` to FORCE FTS-only without auto-selecting. */
     async hybridSearch(query, limit = 12, opts = {}) {
-        const embedder = opts.embedder !== undefined ? opts.embedder : await selectEmbedder();
-        if (!this.semanticReady(embedder))
+        // Explicit `embedder: null` forces pure FTS-only (no semantic, no graph) — the
+        // documented escape hatch and the lean-fallback regression guard.
+        if (opts.embedder === null)
             return this.search(query, limit);
+        const embedder = opts.embedder !== undefined ? opts.embedder : await selectEmbedder();
+        // graphWeight override lets the eval harness A/B the graph stream (0 = off) on one
+        // store without re-loading the module const; defaults to the configured weight.
+        const gw = opts.graphWeight ?? RRF_W_GRAPH;
         const fts = this.search(query, Math.max(limit, 50));
-        try {
-            // The whole semantic leg (query embedding + decode + cosine + fuse) is guarded:
-            // any failure — model load, a corrupt/dim-mismatched vector — degrades to the
-            // lexical results rather than failing the query.
-            const [qvec] = await embedder.embed([query]);
-            if (!qvec)
-                return fts.slice(0, limit);
-            const sem = this.cosineRank(qvec, embedder.id, 50);
-            return this.rrfFuse(fts, sem, limit);
+        let sem = [];
+        if (embedder && this.semanticReady(embedder)) {
+            try {
+                // The semantic leg (query embedding + decode + cosine) is guarded: any failure —
+                // model load, a corrupt/dim-mismatched vector — degrades to lexical + graph.
+                const [qvec] = await embedder.embed([query]);
+                if (qvec)
+                    sem = this.cosineRank(qvec, embedder.id, 50);
+            }
+            catch {
+                sem = [];
+            }
         }
-        catch {
+        // The graph stream is model-free, so it contributes even on a lean (no-embeddings)
+        // install. With neither semantic nor graph signal, return pure FTS so the
+        // zero-fusion-overhead fast path is preserved.
+        const graph = this.graphExpand([...fts, ...sem], 50, gw);
+        if (!sem.length && !graph.length)
             return fts.slice(0, limit);
-        }
+        return this.rrfFuse(fts, sem, graph, limit, gw);
     }
     /** Brute-force exact cosine top-n over stored vectors for one model. Vectors are
      *  pre-normalized, so cosine == dot product. Scoped to `dim = qvec.length` so a
@@ -347,10 +359,12 @@ export class HunchStore {
             return { ref: s.ref, kind: s.kind, title: m?.title ?? s.ref, snippet: (m?.body ?? "").slice(0, 120), score: s.score };
         });
     }
-    /** Rank-based Reciprocal Rank Fusion of the FTS and semantic lists. Ranks (not
-     *  raw scores) erase the bm25-vs-cosine scale mismatch; a small lexical weight
-     *  keeps exact symbol/path matches from being displaced by paraphrase hits. */
-    rrfFuse(fts, sem, limit) {
+    /** Rank-based Reciprocal Rank Fusion of the FTS, semantic, and graph lists. Ranks
+     *  (not raw scores) erase the bm25-vs-cosine-vs-graph scale mismatch; a small lexical
+     *  weight keeps exact symbol/path matches from being displaced by paraphrase or
+     *  neighbor hits. An empty list contributes nothing, so 2-stream behavior is exactly
+     *  preserved when graph (or sem) is absent. */
+    rrfFuse(fts, sem, graph, limit, graphWeight = RRF_W_GRAPH) {
         const acc = new Map();
         const add = (list, weight) => list.forEach((hit, i) => {
             const e = acc.get(hit.ref) ?? { hit, score: 0 };
@@ -359,8 +373,51 @@ export class HunchStore {
         });
         add(fts, RRF_W_FTS);
         add(sem, RRF_W_SEM);
+        add(graph, graphWeight);
         return [...acc.values()].sort((a, b) => b.score - a.score).slice(0, limit).map((e) => ({ ...e.hit, score: e.score }));
     }
+    /** Graph retrieval stream (roadmap #1): 1-hop expansion over the dependency graph
+     *  from the lexical/semantic seed hits. For each seed SYMBOL, surface its direct
+     *  neighbors (callers/callees, importers/imported, container) — the cross-file
+     *  evidence a "why" question needs but that neither bm25 nor cosine reaches. Each
+     *  neighbor accrues GAMMA-decayed support per linking seed (one pulled in by several
+     *  top seeds ranks higher); seeds themselves are excluded, so this only ADDS context.
+     *  Deterministic, model-free (runs on a lean install too), one indexed query per seed. */
+    graphExpand(seeds, n, weight = RRF_W_GRAPH) {
+        if (weight <= 0)
+            return [];
+        const symSeeds = seeds.filter((h) => h.ref.startsWith("sym_"));
+        if (!symSeeds.length)
+            return [];
+        const seen = new Set(seeds.map((h) => h.ref)); // never re-surface a seed
+        const nbStmt = this.db.prepare(
+        /* sql */ `
+      SELECT e."to"   AS nb FROM edges e WHERE e."from" = ? AND e.type IN ('calls','depends_on','imports','contains')
+      UNION
+      SELECT e."from" AS nb FROM edges e WHERE e."to"   = ? AND e.type IN ('calls','depends_on','imports','contains')`);
+        const score = new Map();
+        symSeeds.forEach((h, i) => {
+            const contrib = GRAPH_GAMMA / (RRF_K + i + 1);
+            for (const r of nbStmt.all(h.ref, h.ref)) {
+                if (seen.has(r.nb))
+                    continue;
+                score.set(r.nb, (score.get(r.nb) ?? 0) + contrib);
+            }
+        });
+        if (!score.size)
+            return [];
+        const top = [...score.entries()].sort((a, b) => b[1] - a[1]).slice(0, n);
+        // Hydrate title/snippet from the FTS table in ONE query (mirrors cosineRank).
+        const placeholders = top.map(() => "?").join(",");
+        const meta = new Map();
+        for (const row of this.db.prepare(`SELECT ref, title, body FROM search WHERE ref IN (${placeholders})`).all(...top.map(([ref]) => ref))) {
+            meta.set(row.ref, { title: row.title, body: row.body });
+        }
+        return top.map(([ref, s]) => {
+            const m = meta.get(ref);
+            return { ref, kind: ref.startsWith("cmp_") ? "component" : "symbol", title: m?.title ?? ref, snippet: (m?.body ?? "").slice(0, 120), score: s };
+        });
+    }
     /** All decisions/bugs/constraints/symbols/components touching a file path or
      *  symbol name (hunch_why). Pass `{ asOf }` (an ISO instant) to TIME-TRAVEL:
      *  return only decisions/constraints whose valid-time window contained that
@@ -971,11 +1028,17 @@ function round(n) {
     return Math.round(n * 100) / 100;
 }
 // --- semantic-search helpers ---------------------------------------------
-/** RRF tuning (env-overridable). Lexical weight ≥ semantic so exact matches win
- *  ties while semantic adds paraphrase recall. */
+/** RRF tuning (env-overridable). Lexical weight ≥ semantic ≥ graph: exact matches
+ *  win ties, semantic adds paraphrase recall, and the graph stream adds the
+ *  dependency-neighbor evidence a "why" question needs across files. The graph
+ *  weight is conservative + measurement-gated (set HUNCH_RRF_W_GRAPH=0 to disable);
+ *  GAMMA only decays a neighbor's cross-seed support, so its absolute value is
+ *  normalized away by the rank-based fusion. */
 const RRF_K = numEnv("HUNCH_RRF_K", 60);
 const RRF_W_FTS = numEnv("HUNCH_RRF_W_FTS", 1);
 const RRF_W_SEM = numEnv("HUNCH_RRF_W_SEM", 0.7);
+const RRF_W_GRAPH = numEnv("HUNCH_RRF_W_GRAPH", 0.5);
+const GRAPH_GAMMA = numEnv("HUNCH_GRAPH_GAMMA", 0.25);
 function numEnv(name, dflt) {
     const v = Number(process.env[name]);
     return Number.isFinite(v) && v > 0 ? v : dflt;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@davesheffer/hunch",
-  "version": "0.21.1",
+  "version": "0.23.0",
   "license": "Apache-2.0",
   "author": "Dave Sheffer <dave.sheffer1@gmail.com>",
   "description": "Hunch — an Engineering Memory OS: a persistent, git-native reasoning graph over a codebase, exposed to Claude Code via MCP.",