@davesheffer/hunch 0.22.0 → 0.23.0

package/dist/cli/index.js

@@ -42,6 +42,7 @@ import { healClaudeConfigCaseSplit } from "../integrations/claudeConfig.js";
 import { formatContext } from "../core/format.js";
 import { readConfig, writeConfig, FIRMNESS_LEVELS, isFirmness } from "../core/config.js";
 import { blockingInScope, vetoInScope, proposedEditLines } from "../core/hookpolicy.js";
+import { loadGoldenSet, evaluateGraphLift } from "../eval/harness.js";
 import { draftTripwires, knownRepoDeps } from "../synthesis/tripwires.js";
 import { constraintId } from "../core/ids.js";
 import { readManifest, writeManifest, SCHEMA_VERSION } from "../core/migrate.js";
@@ -409,6 +410,49 @@ program
     }
     store.close();
 });
+// ---- eval (retrieval quality; measures the graph-stream lift) --------------
+program
+    .command("eval")
+    .description("Measure retrieval quality (Recall@k, MRR) over a golden set, and A/B the dependency-graph stream.")
+    .requiredOption("--file <path>", "golden set JSON: [{ query, expected: [refs], note? }]")
+    .option("--k <n>", "top-k cutoff", "10")
+    .option("--semantic", "also blend the semantic stream (requires `hunch embed`; default is deterministic FTS + graph)")
+    .action(async (opts) => {
+    const { store } = storeFor();
+    store.reindex(); // reflect any out-of-band JSON edits before scoring
+    let cases;
+    try {
+        cases = loadGoldenSet(readFileSync(opts.file, "utf8"));
+    }
+    catch (e) {
+        store.close();
+        return fail(`could not load golden set: ${e.message}`);
+    }
+    if (!cases.length) {
+        store.close();
+        return fail("golden set is empty");
+    }
+    const k = Math.max(1, parseInt(opts.k, 10) || 10);
+    // Default is deterministic (FTS + graph, no model). --semantic only adds the
+    // semantic leg when embeddings actually exist; otherwise it's still FTS + graph.
+    const embedder = opts.semantic ? await selectEmbedder() : undefined;
+    const lift = await evaluateGraphLift(store, cases, { k, embedder });
+    const pct = (x) => `${(x * 100).toFixed(1)}%`;
+    const dpt = (x) => `${x >= 0 ? "+" : ""}${(x * 100).toFixed(1)}pt`;
+    const dnum = (x) => `${x >= 0 ? "+" : ""}${x.toFixed(3)}`;
+    console.log(`Eval over ${cases.length} case(s), k=${k}${opts.semantic ? " (semantic + graph + FTS)" : " (FTS + graph)"}\n`);
+    console.log(`                Recall@${k}    MRR      hit-rate`);
+    console.log(`  graph OFF     ${pct(lift.off.recallAtK).padStart(7)}    ${lift.off.mrr.toFixed(3)}    ${pct(lift.off.hitRate)}`);
+    console.log(`  graph ON      ${pct(lift.on.recallAtK).padStart(7)}    ${lift.on.mrr.toFixed(3)}    ${pct(lift.on.hitRate)}`);
+    console.log(`  graph LIFT    ${dpt(lift.recallDelta).padStart(7)}    ${dnum(lift.mrrDelta)}`);
+    const misses = lift.on.perCase.filter((c) => c.found === 0);
+    if (misses.length) {
+        console.log(`\n  ${misses.length} case(s) with no expected hit — curate or tune:`);
+        for (const m of misses.slice(0, 10))
+            console.log(`    · "${m.query}"`);
+    }
+    store.close();
+});
 // ---- embed (opt-in semantic search) ---------------------------------------
 program
     .command("embed")

package/dist/eval/harness.js

@@ -0,0 +1,55 @@
+/** Score a golden set: Recall@k, MRR, hit-rate. Deterministic when no embedder. */
+export async function evaluateRetrieval(store, cases, opts = {}) {
+    const k = opts.k ?? 10;
+    const perCase = [];
+    for (const c of cases) {
+        const hits = await store.hybridSearch(c.query, k, { embedder: opts.embedder, graphWeight: opts.graphWeight });
+        const top = hits.slice(0, k).map((h) => h.ref);
+        const expected = new Set(c.expected);
+        let found = 0;
+        let rr = 0;
+        top.forEach((ref, i) => {
+            if (!expected.has(ref))
+                return;
+            found++;
+            if (rr === 0)
+                rr = 1 / (i + 1); // first expected hit sets the reciprocal rank
+        });
+        perCase.push({
+            query: c.query,
+            expected: c.expected.length,
+            found,
+            recall: c.expected.length ? found / c.expected.length : 0,
+            rr,
+        });
+    }
+    const n = perCase.length || 1;
+    return {
+        n: perCase.length,
+        k,
+        recallAtK: perCase.reduce((s, r) => s + r.recall, 0) / n,
+        mrr: perCase.reduce((s, r) => s + r.rr, 0) / n,
+        hitRate: perCase.reduce((s, r) => s + (r.found > 0 ? 1 : 0), 0) / n,
+        perCase,
+    };
+}
+/** Compare graph-OFF vs graph-ON on the same golden set — the #1 lift measurement. */
+export async function evaluateGraphLift(store, cases, opts = {}) {
+    const off = await evaluateRetrieval(store, cases, { ...opts, graphWeight: 0 });
+    // graphWeight undefined -> hybridSearch uses the configured default; an explicit
+    // value tunes it. Either way "on" is whatever ships, "off" is the baseline.
+    const on = await evaluateRetrieval(store, cases, opts);
+    return { off, on, recallDelta: on.recallAtK - off.recallAtK, mrrDelta: on.mrr - off.mrr };
+}
+/** Parse + validate a golden-set JSON string (array of {query, expected[]}). */
+export function loadGoldenSet(raw) {
+    const data = JSON.parse(raw);
+    if (!Array.isArray(data))
+        throw new Error("golden set must be a JSON array of { query, expected[] }");
+    return data.map((c, i) => {
+        if (!c || typeof c.query !== "string" || !Array.isArray(c.expected))
+            throw new Error(`golden case ${i} must be { query: string, expected: string[] }`);
+        return { query: c.query, expected: c.expected.map(String), note: typeof c.note === "string" ? c.note : undefined };
+    });
+}
+//# sourceMappingURL=harness.js.map

package/dist/store/hunchStore.js

@@ -301,6 +301,9 @@ export class HunchStore {
         if (opts.embedder === null)
             return this.search(query, limit);
         const embedder = opts.embedder !== undefined ? opts.embedder : await selectEmbedder();
+        // graphWeight override lets the eval harness A/B the graph stream (0 = off) on one
+        // store without re-loading the module const; defaults to the configured weight.
+        const gw = opts.graphWeight ?? RRF_W_GRAPH;
         const fts = this.search(query, Math.max(limit, 50));
         let sem = [];
         if (embedder && this.semanticReady(embedder)) {
@@ -318,10 +321,10 @@ export class HunchStore {
         // The graph stream is model-free, so it contributes even on a lean (no-embeddings)
         // install. With neither semantic nor graph signal, return pure FTS so the
         // zero-fusion-overhead fast path is preserved.
-        const graph = this.graphExpand([...fts, ...sem], 50);
+        const graph = this.graphExpand([...fts, ...sem], 50, gw);
         if (!sem.length && !graph.length)
             return fts.slice(0, limit);
-        return this.rrfFuse(fts, sem, graph, limit);
+        return this.rrfFuse(fts, sem, graph, limit, gw);
     }
     /** Brute-force exact cosine top-n over stored vectors for one model. Vectors are
      *  pre-normalized, so cosine == dot product. Scoped to `dim = qvec.length` so a
@@ -361,7 +364,7 @@ export class HunchStore {
      *  weight keeps exact symbol/path matches from being displaced by paraphrase or
      *  neighbor hits. An empty list contributes nothing, so 2-stream behavior is exactly
      *  preserved when graph (or sem) is absent. */
-    rrfFuse(fts, sem, graph, limit) {
+    rrfFuse(fts, sem, graph, limit, graphWeight = RRF_W_GRAPH) {
         const acc = new Map();
         const add = (list, weight) => list.forEach((hit, i) => {
             const e = acc.get(hit.ref) ?? { hit, score: 0 };
@@ -370,7 +373,7 @@ export class HunchStore {
         });
         add(fts, RRF_W_FTS);
         add(sem, RRF_W_SEM);
-        add(graph, RRF_W_GRAPH);
+        add(graph, graphWeight);
         return [...acc.values()].sort((a, b) => b.score - a.score).slice(0, limit).map((e) => ({ ...e.hit, score: e.score }));
     }
     /** Graph retrieval stream (roadmap #1): 1-hop expansion over the dependency graph
@@ -380,8 +383,8 @@ export class HunchStore {
      *  neighbor accrues GAMMA-decayed support per linking seed (one pulled in by several
      *  top seeds ranks higher); seeds themselves are excluded, so this only ADDS context.
      *  Deterministic, model-free (runs on a lean install too), one indexed query per seed. */
-    graphExpand(seeds, n) {
-        if (RRF_W_GRAPH <= 0)
+    graphExpand(seeds, n, weight = RRF_W_GRAPH) {
+        if (weight <= 0)
             return [];
         const symSeeds = seeds.filter((h) => h.ref.startsWith("sym_"));
         if (!symSeeds.length)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@davesheffer/hunch",
-  "version": "0.22.0",
+  "version": "0.23.0",
   "license": "Apache-2.0",
   "author": "Dave Sheffer <dave.sheffer1@gmail.com>",
   "description": "Hunch — an Engineering Memory OS: a persistent, git-native reasoning graph over a codebase, exposed to Claude Code via MCP.",