@davesheffer/hunch 0.22.0 → 0.24.0

package/dist/cli/index.js

@@ -42,6 +42,8 @@ import { healClaudeConfigCaseSplit } from "../integrations/claudeConfig.js";
 import { formatContext } from "../core/format.js";
 import { readConfig, writeConfig, FIRMNESS_LEVELS, isFirmness } from "../core/config.js";
 import { blockingInScope, vetoInScope, proposedEditLines } from "../core/hookpolicy.js";
+import { loadGoldenSet, evaluateGraphLift } from "../eval/harness.js";
+import { computeDrift } from "../core/drift.js";
 import { draftTripwires, knownRepoDeps } from "../synthesis/tripwires.js";
 import { constraintId } from "../core/ids.js";
 import { readManifest, writeManifest, SCHEMA_VERSION } from "../core/migrate.js";
@@ -409,6 +411,49 @@ program
     }
     store.close();
 });
+// ---- eval (retrieval quality; measures the graph-stream lift) --------------
+program
+    .command("eval")
+    .description("Measure retrieval quality (Recall@k, MRR) over a golden set, and A/B the dependency-graph stream.")
+    .requiredOption("--file <path>", "golden set JSON: [{ query, expected: [refs], note? }]")
+    .option("--k <n>", "top-k cutoff", "10")
+    .option("--semantic", "also blend the semantic stream (requires `hunch embed`; default is deterministic FTS + graph)")
+    .action(async (opts) => {
+    const { store } = storeFor();
+    store.reindex(); // reflect any out-of-band JSON edits before scoring
+    let cases;
+    try {
+        cases = loadGoldenSet(readFileSync(opts.file, "utf8"));
+    }
+    catch (e) {
+        store.close();
+        return fail(`could not load golden set: ${e.message}`);
+    }
+    if (!cases.length) {
+        store.close();
+        return fail("golden set is empty");
+    }
+    const k = Math.max(1, parseInt(opts.k, 10) || 10);
+    // Default is deterministic (FTS + graph, no model). --semantic only adds the
+    // semantic leg when embeddings actually exist; otherwise it's still FTS + graph.
+    const embedder = opts.semantic ? await selectEmbedder() : undefined;
+    const lift = await evaluateGraphLift(store, cases, { k, embedder });
+    const pct = (x) => `${(x * 100).toFixed(1)}%`;
+    const dpt = (x) => `${x >= 0 ? "+" : ""}${(x * 100).toFixed(1)}pt`;
+    const dnum = (x) => `${x >= 0 ? "+" : ""}${x.toFixed(3)}`;
+    console.log(`Eval over ${cases.length} case(s), k=${k}${opts.semantic ? " (semantic + graph + FTS)" : " (FTS + graph)"}\n`);
+    console.log(`                Recall@${k}    MRR      hit-rate`);
+    console.log(`  graph OFF     ${pct(lift.off.recallAtK).padStart(7)}    ${lift.off.mrr.toFixed(3)}    ${pct(lift.off.hitRate)}`);
+    console.log(`  graph ON      ${pct(lift.on.recallAtK).padStart(7)}    ${lift.on.mrr.toFixed(3)}    ${pct(lift.on.hitRate)}`);
+    console.log(`  graph LIFT    ${dpt(lift.recallDelta).padStart(7)}    ${dnum(lift.mrrDelta)}`);
+    const misses = lift.on.perCase.filter((c) => c.found === 0);
+    if (misses.length) {
+        console.log(`\n  ${misses.length} case(s) with no expected hit — curate or tune:`);
+        for (const m of misses.slice(0, 10))
+            console.log(`    · "${m.query}"`);
+    }
+    store.close();
+});
 // ---- embed (opt-in semantic search) ---------------------------------------
 program
     .command("embed")
@@ -1229,6 +1274,20 @@ program
     }
     // Windows: detect/heal the Claude Code ~/.claude.json drive-letter case-split
     // that silently hides the hunch_* MCP tools. No-op (silent) off Windows.
+    // Memory drift: deterministic, advisory smoke detector for memory that has
+    // fallen out of sync with the code/docs (dead file refs, dangling supersedes,
+    // docs still marked "proposed"). Never blocks; never auto-fixes.
+    const drift = computeDrift(store, root);
+    if (drift.findings.length) {
+        console.log(`drift:      ⚠ ${drift.findings.length} finding(s) — memory may be out of sync with the code:`);
+        for (const f of drift.findings.slice(0, 20))
+            console.log(`              · [${f.kind}] ${f.id} — ${f.detail}`);
+        if (drift.findings.length > 20)
+            console.log(dim(`              … and ${drift.findings.length - 20} more`));
+    }
+    else {
+        console.log(`drift:      ✓ no stale refs, dangling supersedes, or stale "proposed" docs`);
+    }
     reportClaudeConfigHeal();
     store.close();
 });

package/dist/core/drift.js

@@ -0,0 +1,98 @@
+/**
+ * Memory drift checks (roadmap #7, dec_2a53072620). Deterministic, model-free
+ * comparisons of the curated graph against the actual code/docs — a smoke detector
+ * for stale memory, NOT a robot that rewrites it. Advisory only: `hunch doctor`
+ * prints findings; nothing blocks and nothing is auto-fixed. Each check maps to drift
+ * observed in practice:
+ *   - dead-ref:   an in-force decision points at a file that no longer exists.
+ *   - supersede:  A claims to supersede B, but B was never properly closed.
+ *   - doc-stale:  a doc marked "proposed / not yet implemented" references shipped code.
+ */
+import { existsSync, readFileSync, readdirSync } from "node:fs";
+import { join, extname } from "node:path";
+const STALE_MARKER = /\b(proposed|not yet implemented|no code yet)\b/i;
+const SRC_REF = /\bsrc\/[A-Za-z0-9_\-/]+\.ts\b/g;
+export function computeDrift(store, root) {
+    const findings = [];
+    const decisions = store.recs("decisions");
+    const byId = new Map(decisions.map((d) => [d.id, d]));
+    for (const d of decisions) {
+        // 1. DEAD-REFERENCE — only for in-force decisions; a superseded one referencing
+        //    a since-deleted file is legitimate history, not drift.
+        const inForce = d.status !== "superseded" && !d.superseded_by;
+        if (inForce) {
+            for (const f of d.related_files ?? []) {
+                if (!f || f.includes("*"))
+                    continue; // skip globs / empties
+                if (!existsSync(join(root, f))) {
+                    findings.push({ kind: "dead-ref", id: d.id, detail: `references missing file "${f}"` });
+                }
+            }
+        }
+        // 2. SUPERSEDE-INTEGRITY — a contradiction class: A.supersedes = B, but B is
+        //    either gone or still in force (the private-supersede bug shape).
+        if (d.supersedes) {
+            const target = byId.get(d.supersedes);
+            if (!target) {
+                findings.push({ kind: "supersede", id: d.id, detail: `supersedes "${d.supersedes}", which does not exist` });
+            }
+            else if (target.status !== "superseded" || target.superseded_by !== d.id) {
+                findings.push({
+                    kind: "supersede",
+                    id: d.id,
+                    detail: `supersedes "${d.supersedes}", but it is still in force (status=${target.status}, superseded_by=${target.superseded_by ?? "null"})`,
+                });
+            }
+        }
+    }
+    // 3. DOC-STALE — a doc that still advertises "proposed / not implemented" while
+    //    referencing code that exists. Heuristic + advisory; scoped to the repo's own
+    //    markdown (node_modules and sub-projects skipped).
+    for (const doc of markdownDocs(root)) {
+        const text = safeRead(doc.path);
+        if (!STALE_MARKER.test(text.slice(0, 1500)))
+            continue;
+        const existing = (text.match(SRC_REF) ?? []).find((r) => existsSync(join(root, r)));
+        if (existing) {
+            findings.push({ kind: "doc-stale", id: doc.rel, detail: `marked proposed/not-implemented but references shipped code (${existing})` });
+        }
+    }
+    return { findings };
+}
+function safeRead(path) {
+    try {
+        return readFileSync(path, "utf8");
+    }
+    catch {
+        return "";
+    }
+}
+const SKIP_DIRS = new Set(["node_modules", ".git", ".hunch", ".hunch-private", "dist", "vscode-extension", "site"]);
+/** Bounded walk for repo markdown (root + docs/, depth-limited; heavy/irrelevant trees skipped). */
+function markdownDocs(root) {
+    const out = [];
+    const walk = (dir, rel, depth) => {
+        if (depth > 4)
+            return;
+        let entries;
+        try {
+            entries = readdirSync(dir, { withFileTypes: true });
+        }
+        catch {
+            return;
+        }
+        for (const e of entries) {
+            if (e.isDirectory()) {
+                if (e.name.startsWith(".") || SKIP_DIRS.has(e.name))
+                    continue;
+                walk(join(dir, e.name), rel ? `${rel}/${e.name}` : e.name, depth + 1);
+            }
+            else if (extname(e.name) === ".md") {
+                out.push({ path: join(dir, e.name), rel: rel ? `${rel}/${e.name}` : e.name });
+            }
+        }
+    };
+    walk(root, "", 0);
+    return out;
+}
+//# sourceMappingURL=drift.js.map

package/dist/eval/harness.js

@@ -0,0 +1,55 @@
+/** Score a golden set: Recall@k, MRR, hit-rate. Deterministic when no embedder. */
+export async function evaluateRetrieval(store, cases, opts = {}) {
+    const k = opts.k ?? 10;
+    const perCase = [];
+    for (const c of cases) {
+        const hits = await store.hybridSearch(c.query, k, { embedder: opts.embedder, graphWeight: opts.graphWeight });
+        const top = hits.slice(0, k).map((h) => h.ref);
+        const expected = new Set(c.expected);
+        let found = 0;
+        let rr = 0;
+        top.forEach((ref, i) => {
+            if (!expected.has(ref))
+                return;
+            found++;
+            if (rr === 0)
+                rr = 1 / (i + 1); // first expected hit sets the reciprocal rank
+        });
+        perCase.push({
+            query: c.query,
+            expected: c.expected.length,
+            found,
+            recall: c.expected.length ? found / c.expected.length : 0,
+            rr,
+        });
+    }
+    const n = perCase.length || 1;
+    return {
+        n: perCase.length,
+        k,
+        recallAtK: perCase.reduce((s, r) => s + r.recall, 0) / n,
+        mrr: perCase.reduce((s, r) => s + r.rr, 0) / n,
+        hitRate: perCase.reduce((s, r) => s + (r.found > 0 ? 1 : 0), 0) / n,
+        perCase,
+    };
+}
+/** Compare graph-OFF vs graph-ON on the same golden set — the #1 lift measurement. */
+export async function evaluateGraphLift(store, cases, opts = {}) {
+    const off = await evaluateRetrieval(store, cases, { ...opts, graphWeight: 0 });
+    // graphWeight undefined -> hybridSearch uses the configured default; an explicit
+    // value tunes it. Either way "on" is whatever ships, "off" is the baseline.
+    const on = await evaluateRetrieval(store, cases, opts);
+    return { off, on, recallDelta: on.recallAtK - off.recallAtK, mrrDelta: on.mrr - off.mrr };
+}
+/** Parse + validate a golden-set JSON string (array of {query, expected[]}). */
+export function loadGoldenSet(raw) {
+    const data = JSON.parse(raw);
+    if (!Array.isArray(data))
+        throw new Error("golden set must be a JSON array of { query, expected[] }");
+    return data.map((c, i) => {
+        if (!c || typeof c.query !== "string" || !Array.isArray(c.expected))
+            throw new Error(`golden case ${i} must be { query: string, expected: string[] }`);
+        return { query: c.query, expected: c.expected.map(String), note: typeof c.note === "string" ? c.note : undefined };
+    });
+}
+//# sourceMappingURL=harness.js.map

package/dist/store/hunchStore.js

@@ -301,6 +301,9 @@ export class HunchStore {
         if (opts.embedder === null)
             return this.search(query, limit);
         const embedder = opts.embedder !== undefined ? opts.embedder : await selectEmbedder();
+        // graphWeight override lets the eval harness A/B the graph stream (0 = off) on one
+        // store without re-loading the module const; defaults to the configured weight.
+        const gw = opts.graphWeight ?? RRF_W_GRAPH;
         const fts = this.search(query, Math.max(limit, 50));
         let sem = [];
         if (embedder && this.semanticReady(embedder)) {
@@ -318,10 +321,10 @@ export class HunchStore {
         // The graph stream is model-free, so it contributes even on a lean (no-embeddings)
         // install. With neither semantic nor graph signal, return pure FTS so the
         // zero-fusion-overhead fast path is preserved.
-        const graph = this.graphExpand([...fts, ...sem], 50);
+        const graph = this.graphExpand([...fts, ...sem], 50, gw);
         if (!sem.length && !graph.length)
             return fts.slice(0, limit);
-        return this.rrfFuse(fts, sem, graph, limit);
+        return this.rrfFuse(fts, sem, graph, limit, gw);
     }
     /** Brute-force exact cosine top-n over stored vectors for one model. Vectors are
      *  pre-normalized, so cosine == dot product. Scoped to `dim = qvec.length` so a
@@ -361,7 +364,7 @@ export class HunchStore {
      *  weight keeps exact symbol/path matches from being displaced by paraphrase or
      *  neighbor hits. An empty list contributes nothing, so 2-stream behavior is exactly
      *  preserved when graph (or sem) is absent. */
-    rrfFuse(fts, sem, graph, limit) {
+    rrfFuse(fts, sem, graph, limit, graphWeight = RRF_W_GRAPH) {
         const acc = new Map();
         const add = (list, weight) => list.forEach((hit, i) => {
             const e = acc.get(hit.ref) ?? { hit, score: 0 };
@@ -370,7 +373,7 @@ export class HunchStore {
         });
         add(fts, RRF_W_FTS);
         add(sem, RRF_W_SEM);
-        add(graph, RRF_W_GRAPH);
+        add(graph, graphWeight);
         return [...acc.values()].sort((a, b) => b.score - a.score).slice(0, limit).map((e) => ({ ...e.hit, score: e.score }));
     }
     /** Graph retrieval stream (roadmap #1): 1-hop expansion over the dependency graph
@@ -380,8 +383,8 @@ export class HunchStore {
      *  neighbor accrues GAMMA-decayed support per linking seed (one pulled in by several
      *  top seeds ranks higher); seeds themselves are excluded, so this only ADDS context.
      *  Deterministic, model-free (runs on a lean install too), one indexed query per seed. */
-    graphExpand(seeds, n) {
-        if (RRF_W_GRAPH <= 0)
+    graphExpand(seeds, n, weight = RRF_W_GRAPH) {
+        if (weight <= 0)
             return [];
         const symSeeds = seeds.filter((h) => h.ref.startsWith("sym_"));
         if (!symSeeds.length)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@davesheffer/hunch",
-  "version": "0.22.0",
+  "version": "0.24.0",
   "license": "Apache-2.0",
   "author": "Dave Sheffer <dave.sheffer1@gmail.com>",
   "description": "Hunch — an Engineering Memory OS: a persistent, git-native reasoning graph over a codebase, exposed to Claude Code via MCP.",