@davesheffer/hunch 0.18.1 → 0.19.1

package/README.md

@@ -87,11 +87,14 @@ afterward to pick up the `hunch_*` tools. Each teammate runs `hunch init` once;
 > **never** a pay-per-token API key — and falls back to a deterministic heuristic if no CLI
 > is present. Details: [Synthesis & billing](https://hunch-pi.vercel.app/docs#synthesis).
 >
-> **Deep Synthesis** (`backfill --deep` / `sync --deep`): if you're signed into more than one
-> CLI, fan the commit out to *all* of them and reconcile the drafts into one — confidence is
-> **agreement-weighted** (capped below the enforcement threshold, so it stays advisory).
-> Subscription-only, never on the guard path, and it degrades to the single-provider path with
-> one CLI.
+> **Deep Synthesis** (`backfill --deep` / `sync --deep`): reconcile multiple independent drafts
+> into one — fan out across every signed-in CLI, or, with a single CLI, sample it N times for
+> **self-consistency** (`--samples`, default 2). Confidence is **agreement-weighted** (capped
+> below the enforcement threshold, so it stays advisory). Add `--verify` (auto under `--deep`)
+> for a **Critic pass** that audits each draft against its commit — pruning unsupported
+> rejected-alternatives before they become tripwires and down-weighting weak grounding; it only
+> ever *lowers* confidence, never arming enforcement. Subscription-only, never on the guard path;
+> degrades to the single-provider draft when no CLI is available.
 > On Windows, prefer `hunch init` over a global `claude mcp add`; if tools don't appear,
 > `hunch doctor` heals it ([why](https://hunch-pi.vercel.app/docs#windows)).
 

package/dist/cli/index.js

@@ -180,6 +180,13 @@ async function mapPool(items, limit, fn) {
     };
     await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker()));
 }
+/** Parse a `--samples` flag into a finite positive count, or undefined so the ensemble
+ *  uses its default depth. A typo'd value (`--samples abc`) must NOT become NaN — that
+ *  would silently collapse --deep to the deterministic fallback. */
+function parseSamples(v) {
+    const n = Number(v);
+    return v != null && Number.isFinite(n) && n > 0 ? Math.trunc(n) : undefined;
+}
 program
     .command("backfill")
     .description("Replay git history to seed decisions (cold-start fix).")
@@ -187,6 +194,8 @@ program
     .option("--max <n>", "max commits to process", "40")
     .option("--concurrency <n>", "commits to synthesize in parallel (the LLM call is the bottleneck)", "4")
     .option("--deep", "Deep Synthesis: ensemble every available subscription CLI per commit and reconcile their drafts (slower, higher-quality; advisory)")
+    .option("--verify", "Critic pass: audit each draft against its commit, prune unsupported alternatives/consequences, down-weight weak grounding (extra subscription call; advisory)")
+    .option("--samples <n>", "self-consistency depth when only one CLI is installed: sample it n times per commit and reconcile (default 2 under --deep)")
     .action(async (opts) => {
     const { store, root } = storeFor();
     if (!isGitRepo(root))
@@ -200,11 +209,13 @@ program
     // each commit drafts independently and writes its OWN decision file atomically,
     // and the store's JS-side reads/writes run synchronously between awaits (single
     // thread) — only the LLM spawns overlap. reindex() runs once, after the pool.
+    const samples = parseSamples(opts.samples);
     await mapPool(commits, conc, async (sha) => {
-        const r = await syncCommit(store, root, sha, { deep: opts.deep });
+        const r = await syncCommit(store, root, sha, { deep: opts.deep, verify: opts.verify, samples });
         if (r.status === "written") {
             written++;
-            if (r.provider === "claude-cli")
+            // Any non-deterministic provider (claude/codex/cursor/ensemble) is an LLM draft.
+            if (r.provider && r.provider !== "deterministic")
                 llm++;
             else
                 heuristic++;
@@ -231,6 +242,8 @@ program
     .option("--private", "write the synthesized decision into the private overlay (HUNCH_PRIVATE_DIR), not the public repo — for a repo whose memory is kept private")
     .option("--commit", "after a capture, also git add+commit+push the repo the decision landed in (opt-in; best-effort) — the private store under --private, else this repo")
     .option("--deep", "Deep Synthesis: ensemble every available subscription CLI and reconcile their drafts (agreement-weighted, advisory). Slower; subscription-only")
+    .option("--verify", "Critic pass: audit the draft against its commit, prune unsupported alternatives/consequences, down-weight weak grounding (extra subscription call; advisory)")
+    .option("--samples <n>", "self-consistency depth when only one CLI is installed: sample it n times and reconcile (default 2 under --deep)")
     .action(async (sha, opts) => {
     const { store, root } = storeFor();
     if (!isGitRepo(root))
@@ -240,7 +253,7 @@ program
         return opts.quiet ? undefined : fail("--private needs HUNCH_PRIVATE_DIR set to a private store");
     }
     store.json.ensureDirs();
-    const r = await syncCommit(store, root, sha ?? headSha(root), { force: opts.force, private: opts.private, deep: opts.deep });
+    const r = await syncCommit(store, root, sha ?? headSha(root), { force: opts.force, private: opts.private, deep: opts.deep, verify: opts.verify, samples: parseSamples(opts.samples) });
     if (r.status === "written") {
         store.reindex();
         // Don't rewrite CLAUDE.md from the hook — it would dirty the working tree
@@ -986,7 +999,12 @@ program
         else {
             console.log(`${drafts.length} draft(s) awaiting review (lowest confidence first):\n`);
             for (const d of drafts) {
-                console.log(`  ${d.id} [${d.status}, ${d.provenance.source} ${d.provenance.confidence}]\n      ${d.title}\n      ${d.decision.slice(0, 120)}`);
+                // Surface synthesis telemetry (provider / reconciliation breadth / verifier
+                // grounding) parked in evidence, so the reviewer sees WHY the confidence is
+                // what it is and can confirm or reject at a glance.
+                const synth = (d.provenance.evidence ?? []).find((e) => e.startsWith("synth:"));
+                const synthLine = synth ? `\n      ↳ ${synth.slice("synth:".length).trim()}` : "";
+                console.log(`  ${d.id} [${d.status}, ${d.provenance.source} ${d.provenance.confidence}]\n      ${d.title}\n      ${d.decision.slice(0, 120)}${synthLine}`);
             }
             console.log(`\nAccept:  hunch review --accept <id>\nReject:  hunch review --reject <id>`);
         }

package/dist/synthesis/provider.js

@@ -131,6 +131,27 @@ const BUG_TOOL = {
         required: ["title", "symptom", "root_cause", "severity"],
     },
 };
+const VERIFY_TOOL = {
+    name: "emit_verdict",
+    description: "Emit a skeptical audit of a synthesized decision against its commit.",
+    input_schema: {
+        type: "object",
+        properties: {
+            grounded: { type: "number", description: "0..1: how well decision+consequences are supported by the ACTUAL diff. Be strict." },
+            unsupported_alternatives: {
+                type: "array",
+                items: { type: "string" },
+                description: "VERBATIM entries from alternatives_rejected that the diff/message does NOT evidence (likely hallucinated). Copy them exactly.",
+            },
+            unsupported_claims: {
+                type: "array",
+                items: { type: "string" },
+                description: "VERBATIM consequences not supported by the diff.",
+            },
+        },
+        required: ["grounded", "unsupported_alternatives", "unsupported_claims"],
+    },
+};
 // --------------------------------------------------------------------------
 // Base for headless-CLI SUBSCRIPTION providers. Each one drives a coding-assistant
 // CLI billed to the user's own subscription (never a pay-per-token API key — see
@@ -176,6 +197,17 @@ class CliSynthProvider {
             throw new Error(`${this.name}: no usable bug JSON in output`);
         return draft;
     }
+    /** The Critic pass: audit a draft against its commit. Same subscription-only
+     *  run() path (API keys stripped), so this never bills the pay-per-token API.
+     *  Throws on unusable output so verifyDecisionSafe degrades to the un-audited
+     *  draft (a verifier failure must never lose the draft — dec_18a81c8291). */
+    async verifyDecision(input, draft) {
+        const text = await this.run(`${VERIFY_SYSTEM}\n\n${verifyPrompt(input, draft)}\n\n${jsonInstruction(VERIFY_TOOL.input_schema)}`);
+        const verdict = verdictFromText(text);
+        if (!verdict)
+            throw new Error(`${this.name}: no usable verdict JSON in output`);
+        return verdict;
+    }
 }
 // A model id comes from a HUNCH_*_MODEL env var and ends up as an argv token that,
 // on Windows, pexecIn joins into the cmd.exe line (shell:true, to resolve the npm
@@ -460,7 +492,8 @@ const dedupLines = (xs) => [...new Set(xs.map((s) => s.trim()).filter(Boolean))]
 /** Reconcile N worker drafts into one. DETERMINISTIC (no second LLM call): the richest
  *  draft is the spine; alternatives/consequences are unioned; confidence is AGREEMENT-
  *  WEIGHTED and CAPPED at 0.78 — below STRICT_MIN_CONFIDENCE (0.8) — so an ensemble
- *  auto-draft can never arm enforcement. */
+ *  auto-draft can never arm enforcement. `samples`/`agreement` ride along as advisory
+ *  telemetry for `hunch review` (not schema-bound). */
 export function mergeDecisionDrafts(drafts) {
     const primary = [...drafts].sort((a, b) => b.confidence - a.confidence || b.decision.length - a.decision.length)[0];
     const agreement = meanAgreement(drafts);
@@ -472,19 +505,40 @@ export function mergeDecisionDrafts(drafts) {
         alternatives_rejected: dedupLines(drafts.flatMap((d) => d.alternatives_rejected)),
         confidence: Math.min(0.78, 0.55 + 0.23 * agreement),
         source: "llm_draft+ensemble",
+        samples: drafts.length,
+        agreement: Math.round(agreement * 100) / 100,
     };
 }
+// Default self-consistency depth when only ONE subscription CLI is installed (the
+// common case): sample it this many times and reconcile, so single-CLI users get
+// ensemble-like robustness. Tunable per-call via `--samples`.
+const DEFAULT_SAMPLES = 2;
 export class EnsembleProvider {
     workers;
     name = "ensemble";
-    constructor(workers) {
+    samples;
+    constructor(workers, opts = {}) {
         this.workers = workers;
+        // Default 1 (single worker → passthrough); the self-consistency policy default
+        // lives at the selection layer (selectEnsemble). Coerce to a finite integer in a
+        // sane 1..5 band — a NaN here would make decisionTasks build ZERO tasks and throw,
+        // silently collapsing --deep to the deterministic fallback (callers also sanitize).
+        const n = Math.trunc(Number(opts.samples));
+        this.samples = Number.isFinite(n) ? Math.max(1, Math.min(5, n)) : 1;
     }
     async available() { return this.workers.length > 0; }
+    /** The draft tasks to fan out: one per distinct CLI when several are installed
+     *  (cross-model ensemble), else N self-consistency samples of the single CLI. */
+    decisionTasks(input) {
+        if (this.workers.length >= 2)
+            return this.workers.map((w) => () => w.draftDecision(input));
+        const w = this.workers[0];
+        return Array.from({ length: this.samples }, () => () => w.draftDecision(input));
+    }
     async draftDecision(input) {
         if (!this.workers.length)
             throw new Error("ensemble: no subscription CLI workers available");
-        const settled = await Promise.allSettled(this.workers.map((w) => w.draftDecision(input)));
+        const settled = await Promise.allSettled(this.decisionTasks(input).map((t) => t()));
         const drafts = settled.flatMap((s) => (s.status === "fulfilled" ? [s.value] : []));
         if (!drafts.length)
             throw new Error("ensemble: all workers failed");
@@ -502,11 +556,112 @@ export class EnsembleProvider {
     }
 }
 /** Build the Deep-Synthesis provider, or null if no subscription CLI is available
- *  (the caller then falls back to the normal single-provider path). */
-export async function selectEnsemble() {
+ *  (the caller then falls back to the normal single-provider path). `samples` sets
+ *  the self-consistency depth for the single-CLI case. */
+export async function selectEnsemble(opts = {}) {
+    const workers = await selectWorkers();
+    // The self-consistency policy default (DEFAULT_SAMPLES) is applied HERE, not in the
+    // provider — so a single CLI under --deep is sampled N times, while direct
+    // construction stays passthrough. `--samples 1` opts back out.
+    return workers.length ? new EnsembleProvider(workers, { samples: opts.samples ?? DEFAULT_SAMPLES }) : null;
+}
+/** Pick a CLI provider to run the Critic pass (subscription-only, like the workers).
+ *  Returns null when no assistant CLI is installed — verification then no-ops and the
+ *  un-audited draft stands (graceful degradation; dec_18a81c8291). */
+export async function selectVerifier() {
     const workers = await selectWorkers();
-    return workers.length ? new EnsembleProvider(workers) : null;
+    return workers[0] ?? null;
+}
+// ---- Verification (the Critic pass) ---------------------------------------
+// Audit a draft against the commit it came from, then PRUNE unsupported
+// alternatives/consequences and LOWER confidence on weak grounding. It may only
+// reduce trust, never raise it past the cap — auto-drafts stay advisory and a human
+// `hunch review --accept` remains the ONLY path to enforcement (dec_9a2f2fe72a).
+const VERIFY_SYSTEM = `You are a skeptical auditor for an Engineering Memory OS. You are given a
+synthesized decision record and the ACTUAL commit it was derived from. Your job is to
+flag everything the record asserts that the evidence does NOT support — be strict; when
+in doubt, flag it. Do not invent new content; only judge what is present.`;
+function verifyPrompt(input, draft) {
+    const alts = draft.alternatives_rejected.length
+        ? draft.alternatives_rejected.map((a, i) => `  ${i + 1}. ${a}`).join("\n")
+        : "  (none)";
+    const cons = draft.consequences.length ? draft.consequences.map((c) => `  - ${c}`).join("\n") : "  (none)";
+    return [
+        `COMMIT SUBJECT: ${input.subject}`,
+        input.body ? `COMMIT BODY:\n${input.body}` : "",
+        input.analysis ? `STRUCTURED CHANGES: ${summarizeDiff(input.analysis)}` : "",
+        renderDiff(input),
+        `CANDIDATE DECISION UNDER AUDIT:`,
+        `  decision: ${draft.decision}`,
+        `  consequences:\n${cons}`,
+        `  alternatives_rejected:\n${alts}`,
+        `\nReturn grounded (0..1) and the VERBATIM alternatives_rejected / consequences the evidence does NOT support.`,
+    ].filter(Boolean).join("\n\n");
+}
+/** Map model text → VerifyVerdict, or null when nothing usable parses (→ the caller
+ *  keeps the un-audited draft). Tolerant of arrays-as-strings and missing fields. */
+export function verdictFromText(text) {
+    for (const obj of extractJsonObjects(text)) {
+        const hasGrounded = typeof obj.grounded === "number";
+        const ua = asStrArr(obj.unsupported_alternatives);
+        const uc = asStrArr(obj.unsupported_claims);
+        if (!hasGrounded && !ua.length && !uc.length)
+            continue; // unrelated object
+        const grounded = typeof obj.grounded === "number" ? clamp01(obj.grounded) : 1;
+        return { grounded, unsupported_alternatives: ua, unsupported_claims: uc };
+    }
+    return null;
+}
+const norm = (s) => s.trim().toLowerCase().replace(/\s+/g, " ");
+/** True if `flagged` names `entry` — exact normalized match, or a substantial
+ *  (≥8 char) containment either way, to absorb minor rewording by the auditor
+ *  without nuking unrelated entries. */
+function flaggedMatches(entry, flagged) {
+    const e = norm(entry);
+    if (!e)
+        return false;
+    return flagged.some((f) => {
+        const n = norm(f);
+        if (!n)
+            return false;
+        if (n === e)
+            return true;
+        return n.length >= 8 && e.length >= 8 && (e.includes(n) || n.includes(e));
+    });
+}
+/** Apply a verdict to a draft: drop unsupported alternatives (so they never scaffold
+ *  tripwires) and consequences, and scale confidence DOWN by grounding. Confidence is
+ *  clamped so it can only fall — verification never arms a stronger claim than the
+ *  draft already made (R2). Records `grounded` as advisory telemetry. */
+export function applyVerdict(draft, v) {
+    const alternatives_rejected = draft.alternatives_rejected.filter((a) => !flaggedMatches(a, v.unsupported_alternatives));
+    const consequences = draft.consequences.filter((c) => !flaggedMatches(c, v.unsupported_claims));
+    const grounded = clamp01(v.grounded);
+    // Penalize weak grounding; (0.5 + 0.5*grounded) ∈ [0.5,1], so this only lowers.
+    const confidence = Math.min(draft.confidence, Math.round(draft.confidence * (0.5 + 0.5 * grounded) * 100) / 100);
+    const source = draft.source.includes("verified") ? draft.source : `${draft.source}+verified`;
+    return { ...draft, alternatives_rejected, consequences, confidence, grounded, source, verifyOutcome: "applied" };
+}
+/** Run the Critic pass and apply it, degrading to the un-audited draft when the
+ *  provider can't verify (deterministic / no CLI) or the call keeps failing. Never
+ *  throws. Marks the OUTCOME on the draft (applied / unavailable / failed) so the
+ *  degradation is visible in telemetry instead of silent. Retries once: under --deep
+ *  the Critic call stacks after sampling, and a single transient failure on that
+ *  extra call shouldn't drop the audit. */
+export async function verifyDecisionSafe(verifier, input, draft) {
+    if (!verifier?.verifyDecision)
+        return { ...draft, verifyOutcome: "unavailable" };
+    for (let attempt = 0; attempt < 2; attempt++) {
+        try {
+            return applyVerdict(draft, await verifier.verifyDecision(input, draft));
+        }
+        catch {
+            /* transient (network / unparseable verdict) — retry once, then give up */
+        }
+    }
+    return { ...draft, verifyOutcome: "failed" };
 }
+const clamp01 = (n) => (Number.isFinite(n) ? Math.max(0, Math.min(1, n)) : 1);
 // ---- prompt + parsing helpers --------------------------------------------
 // Above this size we stop shipping the raw patch and lean on the deterministic
 // STRUCTURED CHANGES summary + a small sample. A truncated head-slice of a giant

package/dist/synthesis/synthesize.js

@@ -1,6 +1,6 @@
 import { commitMeta, commitDiff, headSha } from "../extractors/git.js";
 import { analyzeDiff } from "../extractors/diff.js";
-import { selectProvider, selectEnsemble, DeterministicProvider } from "./provider.js";
+import { selectProvider, selectEnsemble, selectVerifier, verifyDecisionSafe, DeterministicProvider } from "./provider.js";
 import { decisionId, bugId, constraintId } from "../core/ids.js";
 import { pathMatchesGlob } from "../core/glob.js";
 import { draftTripwires, knownRepoDeps } from "./tripwires.js";
@@ -69,13 +69,36 @@ export async function syncCommit(store, root, sha, opts = {}) {
     // Deep Synthesis (--deep): ensemble every available subscription CLI and reconcile
     // their drafts (agreement-weighted, confidence capped below the strict gate). Falls
     // back to the normal single-provider path when no CLI is available. Opt-in only.
+    // --verify forces the LLM provider (auditing a deterministic draft is pointless) and,
+    // like --deep, runs the Critic pass below. Subscription-only throughout (con_2ce3f2a547).
+    const wantVerify = !!(opts.verify || opts.deep);
     const provider = opts.deep
-        ? (await selectEnsemble()) ?? await selectProvider()
-        : opts.force || isSignificant(meta, analysis, codeFiles)
+        ? (await selectEnsemble({ samples: opts.samples })) ?? await selectProvider()
+        : opts.force || opts.verify || isSignificant(meta, analysis, codeFiles)
             ? await selectProvider()
             : new DeterministicProvider();
     const input = { subject: meta.subject, body: meta.body, files: codeFiles, diff, analysis };
-    const draft = await draftDecisionSafe(provider, input);
+    let draft = await draftDecisionSafe(provider, input);
+    // The Critic pass: audit the draft against the commit, PRUNE unsupported alternatives
+    // (BEFORE they scaffold tripwires below) and consequences, and lower confidence on weak
+    // grounding. No-ops when no assistant CLI is available; never raises trust (dec_9a2f2fe72a).
+    if (wantVerify)
+        draft = await verifyDecisionSafe(await selectVerifier(), input, draft);
+    // Advisory synthesis telemetry for `hunch review` — which provider ran, how many drafts
+    // were reconciled, their agreement, and the verifier's grounding. Rides in `evidence`
+    // (no schema change → respects forward-migration invariant con_947c578b2c).
+    const synthBits = [`provider=${provider.name}`];
+    if (draft.samples)
+        synthBits.push(`samples=${draft.samples}`);
+    if (draft.agreement != null)
+        synthBits.push(`agreement=${draft.agreement}`);
+    if (draft.grounded != null)
+        synthBits.push(`grounded=${draft.grounded}`);
+    // The Critic was requested (--verify/--deep) but didn't apply — surface WHY
+    // (unavailable / failed) so a skipped audit is never mistaken for a clean one.
+    else if (draft.verifyOutcome && draft.verifyOutcome !== "applied")
+        synthBits.push(`verify=${draft.verifyOutcome}`);
+    const synthEvidence = `synth:${synthBits.join(" ")}`;
     const components = store.json.loadAll("components");
     const relatedComponents = components
         .filter((c) => codeFiles.some((f) => c.paths.some((g) => pathMatchesGlob(f, g))))
@@ -119,7 +142,7 @@ export async function syncCommit(store, root, sha, opts = {}) {
         provenance: {
             source: draft.source,
             confidence: draft.confidence,
-            evidence: [`commit:${meta.shortSha}`, ...codeFiles.slice(0, 8)],
+            evidence: [`commit:${meta.shortSha}`, synthEvidence, ...codeFiles.slice(0, 8)],
             last_verified: new Date().toISOString(), // when the Hunch last re-derived this
         },
         date: meta.date, // the commit date

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@davesheffer/hunch",
-  "version": "0.18.1",
+  "version": "0.19.1",
   "license": "Apache-2.0",
   "author": "Dave Sheffer <dave.sheffer1@gmail.com>",
   "description": "Hunch — an Engineering Memory OS: a persistent, git-native reasoning graph over a codebase, exposed to Claude Code via MCP.",