@davesheffer/hunch 0.18.0 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -7
- package/dist/cli/index.js +37 -11
- package/dist/extractors/git.js +17 -0
- package/dist/mcp/server.js +15 -3
- package/dist/store/hunchStore.js +13 -7
- package/dist/synthesis/provider.js +154 -6
- package/dist/synthesis/synthesize.js +24 -5
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -87,11 +87,14 @@ afterward to pick up the `hunch_*` tools. Each teammate runs `hunch init` once;
|
|
|
87
87
|
> **never** a pay-per-token API key — and falls back to a deterministic heuristic if no CLI
|
|
88
88
|
> is present. Details: [Synthesis & billing](https://hunch-pi.vercel.app/docs#synthesis).
|
|
89
89
|
>
|
|
90
|
-
> **Deep Synthesis** (`backfill --deep` / `sync --deep`):
|
|
91
|
-
>
|
|
92
|
-
> **
|
|
93
|
-
>
|
|
94
|
-
>
|
|
90
|
+
> **Deep Synthesis** (`backfill --deep` / `sync --deep`): reconcile multiple independent drafts
|
|
91
|
+
> into one — fan out across every signed-in CLI, or, with a single CLI, sample it N times for
|
|
92
|
+
> **self-consistency** (`--samples`, default 2). Confidence is **agreement-weighted** (capped
|
|
93
|
+
> below the enforcement threshold, so it stays advisory). Add `--verify` (auto under `--deep`)
|
|
94
|
+
> for a **Critic pass** that audits each draft against its commit — pruning unsupported
|
|
95
|
+
> rejected-alternatives before they become tripwires and down-weighting weak grounding; it only
|
|
96
|
+
> ever *lowers* confidence, never arming enforcement. Subscription-only, never on the guard path;
|
|
97
|
+
> degrades to the single-provider draft when no CLI is available.
|
|
95
98
|
> On Windows, prefer `hunch init` over a global `claude mcp add`; if tools don't appear,
|
|
96
99
|
> `hunch doctor` heals it ([why](https://hunch-pi.vercel.app/docs#windows)).
|
|
97
100
|
|
|
@@ -221,7 +224,7 @@ src/
|
|
|
221
224
|
|
|
222
225
|
Everything lives under `.hunch/` as git-tracked JSON (the source of truth); SQLite is a
|
|
223
226
|
throwaway derived index. → [storage layout](https://hunch-pi.vercel.app/docs#storage) ·
|
|
224
|
-
[
|
|
227
|
+
[the docs](https://hunch-pi.vercel.app/docs) for the full conceptual model.
|
|
225
228
|
|
|
226
229
|
## Notable engineering decisions
|
|
227
230
|
|
|
@@ -247,5 +250,5 @@ npm run build # compile to dist/ (the published artifact)
|
|
|
247
250
|
```
|
|
248
251
|
|
|
249
252
|
Hunch is pure TypeScript ESM, Node ≥ 20, licensed **Apache-2.0**. See
|
|
250
|
-
[CONTRIBUTING.md](CONTRIBUTING.md)
|
|
253
|
+
[CONTRIBUTING.md](CONTRIBUTING.md) and the full
|
|
251
254
|
[developer docs](https://hunch-pi.vercel.app/docs#develop).
|
package/dist/cli/index.js
CHANGED
|
@@ -28,7 +28,7 @@ import { indexRepo } from "../extractors/indexer.js";
|
|
|
28
28
|
import { syncCommit, recordFailure, captureTestRun } from "../synthesis/synthesize.js";
|
|
29
29
|
import { parseTestReport } from "../extractors/testreport.js";
|
|
30
30
|
import { selectProvider } from "../synthesis/provider.js";
|
|
31
|
-
import { isGitRepo, headSha, logSince, lastChangeDate, stagedFiles, commitFiles, asOfDate, stagedDiff, commitDiff, rangeFiles, rangeDiff, revExists } from "../extractors/git.js";
|
|
31
|
+
import { isGitRepo, headSha, logSince, lastChangeDate, stagedFiles, commitFiles, asOfDate, stagedDiff, commitDiff, rangeFiles, rangeDiff, revExists, commitAndPushHunch } from "../extractors/git.js";
|
|
32
32
|
import { renderText, renderMarkdown, reportFailsStrict } from "../core/checkreport.js";
|
|
33
33
|
import { installPostCommitHook, installPreCommitHook } from "../integrations/hooks.js";
|
|
34
34
|
import { installMergeDriver } from "../integrations/mergeDriver.js";
|
|
@@ -180,6 +180,13 @@ async function mapPool(items, limit, fn) {
|
|
|
180
180
|
};
|
|
181
181
|
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, () => worker()));
|
|
182
182
|
}
|
|
183
|
+
/** Parse a `--samples` flag into a finite positive count, or undefined so the ensemble
|
|
184
|
+
* uses its default depth. A typo'd value (`--samples abc`) must NOT become NaN — that
|
|
185
|
+
* would silently collapse --deep to the deterministic fallback. */
|
|
186
|
+
function parseSamples(v) {
|
|
187
|
+
const n = Number(v);
|
|
188
|
+
return v != null && Number.isFinite(n) && n > 0 ? Math.trunc(n) : undefined;
|
|
189
|
+
}
|
|
183
190
|
program
|
|
184
191
|
.command("backfill")
|
|
185
192
|
.description("Replay git history to seed decisions (cold-start fix).")
|
|
@@ -187,6 +194,8 @@ program
|
|
|
187
194
|
.option("--max <n>", "max commits to process", "40")
|
|
188
195
|
.option("--concurrency <n>", "commits to synthesize in parallel (the LLM call is the bottleneck)", "4")
|
|
189
196
|
.option("--deep", "Deep Synthesis: ensemble every available subscription CLI per commit and reconcile their drafts (slower, higher-quality; advisory)")
|
|
197
|
+
.option("--verify", "Critic pass: audit each draft against its commit, prune unsupported alternatives/consequences, down-weight weak grounding (extra subscription call; advisory)")
|
|
198
|
+
.option("--samples <n>", "self-consistency depth when only one CLI is installed: sample it n times per commit and reconcile (default 2 under --deep)")
|
|
190
199
|
.action(async (opts) => {
|
|
191
200
|
const { store, root } = storeFor();
|
|
192
201
|
if (!isGitRepo(root))
|
|
@@ -200,11 +209,13 @@ program
|
|
|
200
209
|
// each commit drafts independently and writes its OWN decision file atomically,
|
|
201
210
|
// and the store's JS-side reads/writes run synchronously between awaits (single
|
|
202
211
|
// thread) — only the LLM spawns overlap. reindex() runs once, after the pool.
|
|
212
|
+
const samples = parseSamples(opts.samples);
|
|
203
213
|
await mapPool(commits, conc, async (sha) => {
|
|
204
|
-
const r = await syncCommit(store, root, sha, { deep: opts.deep });
|
|
214
|
+
const r = await syncCommit(store, root, sha, { deep: opts.deep, verify: opts.verify, samples });
|
|
205
215
|
if (r.status === "written") {
|
|
206
216
|
written++;
|
|
207
|
-
|
|
217
|
+
// Any non-deterministic provider (claude/codex/cursor/ensemble) is an LLM draft.
|
|
218
|
+
if (r.provider && r.provider !== "deterministic")
|
|
208
219
|
llm++;
|
|
209
220
|
else
|
|
210
221
|
heuristic++;
|
|
@@ -231,6 +242,8 @@ program
|
|
|
231
242
|
.option("--private", "write the synthesized decision into the private overlay (HUNCH_PRIVATE_DIR), not the public repo — for a repo whose memory is kept private")
|
|
232
243
|
.option("--commit", "after a capture, also git add+commit+push the repo the decision landed in (opt-in; best-effort) — the private store under --private, else this repo")
|
|
233
244
|
.option("--deep", "Deep Synthesis: ensemble every available subscription CLI and reconcile their drafts (agreement-weighted, advisory). Slower; subscription-only")
|
|
245
|
+
.option("--verify", "Critic pass: audit the draft against its commit, prune unsupported alternatives/consequences, down-weight weak grounding (extra subscription call; advisory)")
|
|
246
|
+
.option("--samples <n>", "self-consistency depth when only one CLI is installed: sample it n times and reconcile (default 2 under --deep)")
|
|
234
247
|
.action(async (sha, opts) => {
|
|
235
248
|
const { store, root } = storeFor();
|
|
236
249
|
if (!isGitRepo(root))
|
|
@@ -240,7 +253,7 @@ program
|
|
|
240
253
|
return opts.quiet ? undefined : fail("--private needs HUNCH_PRIVATE_DIR set to a private store");
|
|
241
254
|
}
|
|
242
255
|
store.json.ensureDirs();
|
|
243
|
-
const r = await syncCommit(store, root, sha ?? headSha(root), { force: opts.force, private: opts.private, deep: opts.deep });
|
|
256
|
+
const r = await syncCommit(store, root, sha ?? headSha(root), { force: opts.force, private: opts.private, deep: opts.deep, verify: opts.verify, samples: parseSamples(opts.samples) });
|
|
244
257
|
if (r.status === "written") {
|
|
245
258
|
store.reindex();
|
|
246
259
|
// Don't rewrite CLAUDE.md from the hook — it would dirty the working tree
|
|
@@ -254,10 +267,7 @@ program
|
|
|
254
267
|
// hook (no recursion, including on a manual `hunch sync --commit`).
|
|
255
268
|
const commitTarget = opts.commit ? (opts.private ? store.privateDir : hunchPaths(root).hunch) : undefined;
|
|
256
269
|
if (commitTarget) {
|
|
257
|
-
|
|
258
|
-
g(["add", "--", "."]);
|
|
259
|
-
g(["commit", "-m", `hunch: capture ${r.decision?.id ?? "decision"}`]);
|
|
260
|
-
g(["push"]);
|
|
270
|
+
commitAndPushHunch(commitTarget, `hunch: capture ${r.decision?.id ?? "decision"}`);
|
|
261
271
|
if (!opts.quiet)
|
|
262
272
|
console.log(` ↳ committed + pushed ${r.decision?.id} (${commitTarget})`);
|
|
263
273
|
}
|
|
@@ -275,9 +285,20 @@ program
|
|
|
275
285
|
.description("Enable a PRIVATE memory overlay — sensitive decisions/bugs/constraints kept in a separate location, unioned into local queries, never committed here. Writes a gitignored .hunch/local.json so it's auto-detected (no env var needed).")
|
|
276
286
|
.option("--repo <url>", "clone a private git repo to use as the store (into ./.hunch-private)")
|
|
277
287
|
.option("--no-hook", "don't switch the post-commit hook to private sync")
|
|
278
|
-
.option("--auto-commit", "opt-in:
|
|
288
|
+
.option("--auto-commit", "opt-in: also git add+commit+push the private repo after each capture (post-commit hook AND MCP private writes)")
|
|
289
|
+
.option("--sync", "flush the configured private store now (git add+commit+push) — catches records made via MCP between commits")
|
|
279
290
|
.action((dir, opts) => {
|
|
280
291
|
const root = findRoot();
|
|
292
|
+
if (opts.sync) {
|
|
293
|
+
const s = new HunchStore(hunchPaths(root));
|
|
294
|
+
const target = s.privateDir;
|
|
295
|
+
s.close();
|
|
296
|
+
if (!target)
|
|
297
|
+
return fail("no private overlay configured — run `hunch private` first");
|
|
298
|
+
commitAndPushHunch(target, "hunch: sync private memory");
|
|
299
|
+
console.log(`✓ flushed private store → ${target}`);
|
|
300
|
+
return;
|
|
301
|
+
}
|
|
281
302
|
const paths = hunchPaths(root);
|
|
282
303
|
// 1) resolve the private store's hunch dir (holds decisions/, bugs/, …)
|
|
283
304
|
let hunchDir;
|
|
@@ -303,7 +324,7 @@ program
|
|
|
303
324
|
// a store elsewhere on disk. Resolution (env || local.json) re-resolves against root.
|
|
304
325
|
const rel = relative(root, hunchDir);
|
|
305
326
|
const stored = rel && !rel.startsWith("..") && !isAbsolute(rel) ? toPosixTarget(rel) : hunchDir;
|
|
306
|
-
writeFileAtomic(join(paths.hunch, "local.json"), JSON.stringify({ privateDir: stored }, null, 2) + "\n");
|
|
327
|
+
writeFileAtomic(join(paths.hunch, "local.json"), JSON.stringify({ privateDir: stored, autoCommit: !!opts.autoCommit }, null, 2) + "\n");
|
|
307
328
|
ensureGitignore(root); // keeps .hunch/local.json + .hunch-private/ out of git
|
|
308
329
|
// 4) route post-commit synthesis to the overlay (local hook, never committed)
|
|
309
330
|
let hookNote = "";
|
|
@@ -978,7 +999,12 @@ program
|
|
|
978
999
|
else {
|
|
979
1000
|
console.log(`${drafts.length} draft(s) awaiting review (lowest confidence first):\n`);
|
|
980
1001
|
for (const d of drafts) {
|
|
981
|
-
|
|
1002
|
+
// Surface synthesis telemetry (provider / reconciliation breadth / verifier
|
|
1003
|
+
// grounding) parked in evidence, so the reviewer sees WHY the confidence is
|
|
1004
|
+
// what it is and can confirm or reject at a glance.
|
|
1005
|
+
const synth = (d.provenance.evidence ?? []).find((e) => e.startsWith("synth:"));
|
|
1006
|
+
const synthLine = synth ? `\n ↳ ${synth.slice("synth:".length).trim()}` : "";
|
|
1007
|
+
console.log(` ${d.id} [${d.status}, ${d.provenance.source} ${d.provenance.confidence}]\n ${d.title}\n ${d.decision.slice(0, 120)}${synthLine}`);
|
|
982
1008
|
}
|
|
983
1009
|
console.log(`\nAccept: hunch review --accept <id>\nReject: hunch review --reject <id>`);
|
|
984
1010
|
}
|
package/dist/extractors/git.js
CHANGED
|
@@ -19,6 +19,23 @@ function gitSafe(args, cwd, maxBuffer) {
|
|
|
19
19
|
export function isGitRepo(cwd) {
|
|
20
20
|
return gitSafe(["rev-parse", "--is-inside-work-tree"], cwd) === "true";
|
|
21
21
|
}
|
|
22
|
+
/** Best-effort: stage ONLY the hunch dir, commit, and push the repo it lives in. Shared by
|
|
23
|
+
* the post-commit auto-commit (CLI sync --commit), MCP private writes, and `hunch private
|
|
24
|
+
* --sync`. HUNCH_SYNC=1 stops the created commit from re-triggering the post-commit hook
|
|
25
|
+
* (no recursion). Stages with a pathspec scoped to `hunchDir`, so it never sweeps unrelated
|
|
26
|
+
* working-tree changes. Never throws — a non-repo dir / offline push just no-ops. */
|
|
27
|
+
export function commitAndPushHunch(hunchDir, message) {
|
|
28
|
+
const env = { ...process.env, HUNCH_SYNC: "1" };
|
|
29
|
+
const run = (args) => {
|
|
30
|
+
try {
|
|
31
|
+
execFileSync("git", ["-C", hunchDir, ...args], { stdio: "ignore", env });
|
|
32
|
+
}
|
|
33
|
+
catch { /* best-effort: nothing staged / not a repo / offline */ }
|
|
34
|
+
};
|
|
35
|
+
run(["add", "--", "."]);
|
|
36
|
+
run(["commit", "-m", message]);
|
|
37
|
+
run(["push"]);
|
|
38
|
+
}
|
|
22
39
|
export function headSha(cwd) {
|
|
23
40
|
return gitSafe(["rev-parse", "HEAD"], cwd);
|
|
24
41
|
}
|
package/dist/mcp/server.js
CHANGED
|
@@ -14,7 +14,7 @@ import { HunchStore } from "../store/hunchStore.js";
|
|
|
14
14
|
import { selectEmbedder } from "../store/embedder.js";
|
|
15
15
|
import { decisionId } from "../core/ids.js";
|
|
16
16
|
import { buildCorrectionConstraint } from "../core/correction.js";
|
|
17
|
-
import { revParse, asOfDate, revExists, lastChangeDate, rangeFiles, rangeDiff, commitFiles, commitDiff, stagedFiles, stagedDiff } from "../extractors/git.js";
|
|
17
|
+
import { revParse, asOfDate, revExists, lastChangeDate, rangeFiles, rangeDiff, commitFiles, commitDiff, stagedFiles, stagedDiff, commitAndPushHunch } from "../extractors/git.js";
|
|
18
18
|
import { formatContext } from "../core/format.js";
|
|
19
19
|
import { renderMarkdown, verdict } from "../core/checkreport.js";
|
|
20
20
|
import { HUNCH_VERSION } from "../core/version.js";
|
|
@@ -292,9 +292,16 @@ export function buildServer(root) {
|
|
|
292
292
|
// the public store, so skip it for a private record (a v1 limitation, not a leak).
|
|
293
293
|
const superseded = decision.supersedes && !decision.private ? store.supersede(decision.supersedes, rec) : null;
|
|
294
294
|
store.reindex();
|
|
295
|
+
// Auto-flush the private repo when configured (hunch private --auto-commit), so a
|
|
296
|
+
// record made via MCP between public commits is committed+pushed immediately.
|
|
297
|
+
let flushed = "";
|
|
298
|
+
if (decision.private && store.privateAutoCommit && store.privateDir) {
|
|
299
|
+
commitAndPushHunch(store.privateDir, `hunch: capture ${id}`);
|
|
300
|
+
flushed = " (committed + pushed to the private repo)";
|
|
301
|
+
}
|
|
295
302
|
const supNote = superseded ? ` Superseded ${superseded.id} (window closed at ${rec.valid_from}).` : "";
|
|
296
303
|
const note = decision.commit && !fullSha ? ` (note: commit "${decision.commit}" could not be resolved — recorded as a standalone decision, not linked to a commit)` : "";
|
|
297
|
-
const where = decision.private ?
|
|
304
|
+
const where = decision.private ? ` [PRIVATE overlay — not committed to this repo]${flushed}` : "";
|
|
298
305
|
return ok(`Recorded decision ${id}: "${rec.title}" (status ${rec.status}, ${source}).${where}${supNote}${note}`);
|
|
299
306
|
}
|
|
300
307
|
catch (e) {
|
|
@@ -328,10 +335,15 @@ export function buildServer(root) {
|
|
|
328
335
|
else
|
|
329
336
|
store.json.put("constraints", rec);
|
|
330
337
|
store.reindex();
|
|
338
|
+
let flushed = "";
|
|
339
|
+
if (input.private && store.privateAutoCommit && store.privateDir) {
|
|
340
|
+
commitAndPushHunch(store.privateDir, `hunch: capture ${rec.id}`);
|
|
341
|
+
flushed = " (committed + pushed to the private repo)";
|
|
342
|
+
}
|
|
331
343
|
const enforce = rec.severity === "blocking"
|
|
332
344
|
? "blocks a DIRECT edit to its scope at strict firmness, and fails a PR whose diff touches that scope (CI guard); blast-radius hits and lower firmness stay advisory"
|
|
333
345
|
: "flags violating edits and PRs (advisory)";
|
|
334
|
-
const where = input.private ?
|
|
346
|
+
const where = input.private ? ` [PRIVATE overlay — not committed to this repo]${flushed}` : "";
|
|
335
347
|
return ok(`${existing ? "Updated" : "Recorded"} ${rec.severity} constraint ${rec.id}: "${rec.statement}" (scope: ${rec.scope.join(", ")}).${where} It now ${enforce}.`);
|
|
336
348
|
}
|
|
337
349
|
catch (e) {
|
package/dist/store/hunchStore.js
CHANGED
|
@@ -31,6 +31,9 @@ export class HunchStore {
|
|
|
31
31
|
/** The resolved private-overlay hunch dir (from env or .hunch/local.json), or undefined
|
|
32
32
|
* when no overlay is configured. Surfaced so `hunch doctor` reflects the true state. */
|
|
33
33
|
privateDir;
|
|
34
|
+
/** Whether private writes should auto commit+push the private repo (from local.json
|
|
35
|
+
* `autoCommit`, set by `hunch private --auto-commit`). Read by the MCP write tools. */
|
|
36
|
+
privateAutoCommit;
|
|
34
37
|
/** When true, recs() ignores the private overlay (public-only). Set transiently by
|
|
35
38
|
* buildCheckReport({publicOnly}) so any PUBLICLY-POSTED report (the CI PR comment)
|
|
36
39
|
* can never render a private record — a publicly-posted output is a leak surface
|
|
@@ -43,24 +46,27 @@ export class HunchStore {
|
|
|
43
46
|
// Private overlay location: env (override, for CI / portability) → else a gitignored
|
|
44
47
|
// local config (.hunch/local.json) so `hunch private` enables it with NO env var, and
|
|
45
48
|
// the MCP server / hook pick it up automatically. Relative paths resolve from root.
|
|
46
|
-
const
|
|
49
|
+
const local = this.localConfig();
|
|
50
|
+
const priv = process.env.HUNCH_PRIVATE_DIR?.trim() || local.privateDir;
|
|
47
51
|
if (priv) {
|
|
48
52
|
this.privateDir = resolve(this.paths.root, priv);
|
|
49
53
|
this.privateJson = new JsonStore(hunchPathsForDir(this.privateDir));
|
|
50
54
|
}
|
|
55
|
+
this.privateAutoCommit = !!(priv && local.autoCommit);
|
|
51
56
|
}
|
|
52
|
-
/** The private-overlay
|
|
53
|
-
* never committed). Tolerant:
|
|
54
|
-
|
|
57
|
+
/** The private-overlay config from the gitignored `.hunch/local.json` (per-machine,
|
|
58
|
+
* never committed). Tolerant: returns {} on missing/invalid so reads never crash. */
|
|
59
|
+
localConfig() {
|
|
55
60
|
try {
|
|
56
61
|
const f = join(this.paths.hunch, "local.json");
|
|
57
62
|
if (!existsSync(f))
|
|
58
|
-
return
|
|
63
|
+
return {};
|
|
59
64
|
const v = JSON.parse(readFileSync(f, "utf8"));
|
|
60
|
-
|
|
65
|
+
const privateDir = typeof v.privateDir === "string" && v.privateDir.trim() ? v.privateDir.trim() : undefined;
|
|
66
|
+
return { privateDir, autoCommit: v.autoCommit === true };
|
|
61
67
|
}
|
|
62
68
|
catch {
|
|
63
|
-
return
|
|
69
|
+
return {};
|
|
64
70
|
}
|
|
65
71
|
}
|
|
66
72
|
/** Merged read: public ∪ private overlay (private wins on id collision). Every
|
|
@@ -131,6 +131,27 @@ const BUG_TOOL = {
|
|
|
131
131
|
required: ["title", "symptom", "root_cause", "severity"],
|
|
132
132
|
},
|
|
133
133
|
};
|
|
134
|
+
const VERIFY_TOOL = {
|
|
135
|
+
name: "emit_verdict",
|
|
136
|
+
description: "Emit a skeptical audit of a synthesized decision against its commit.",
|
|
137
|
+
input_schema: {
|
|
138
|
+
type: "object",
|
|
139
|
+
properties: {
|
|
140
|
+
grounded: { type: "number", description: "0..1: how well decision+consequences are supported by the ACTUAL diff. Be strict." },
|
|
141
|
+
unsupported_alternatives: {
|
|
142
|
+
type: "array",
|
|
143
|
+
items: { type: "string" },
|
|
144
|
+
description: "VERBATIM entries from alternatives_rejected that the diff/message does NOT evidence (likely hallucinated). Copy them exactly.",
|
|
145
|
+
},
|
|
146
|
+
unsupported_claims: {
|
|
147
|
+
type: "array",
|
|
148
|
+
items: { type: "string" },
|
|
149
|
+
description: "VERBATIM consequences not supported by the diff.",
|
|
150
|
+
},
|
|
151
|
+
},
|
|
152
|
+
required: ["grounded", "unsupported_alternatives", "unsupported_claims"],
|
|
153
|
+
},
|
|
154
|
+
};
|
|
134
155
|
// --------------------------------------------------------------------------
|
|
135
156
|
// Base for headless-CLI SUBSCRIPTION providers. Each one drives a coding-assistant
|
|
136
157
|
// CLI billed to the user's own subscription (never a pay-per-token API key — see
|
|
@@ -176,6 +197,17 @@ class CliSynthProvider {
|
|
|
176
197
|
throw new Error(`${this.name}: no usable bug JSON in output`);
|
|
177
198
|
return draft;
|
|
178
199
|
}
|
|
200
|
+
/** The Critic pass: audit a draft against its commit. Same subscription-only
|
|
201
|
+
* run() path (API keys stripped), so this never bills the pay-per-token API.
|
|
202
|
+
* Throws on unusable output so verifyDecisionSafe degrades to the un-audited
|
|
203
|
+
* draft (a verifier failure must never lose the draft — dec_18a81c8291). */
|
|
204
|
+
async verifyDecision(input, draft) {
|
|
205
|
+
const text = await this.run(`${VERIFY_SYSTEM}\n\n${verifyPrompt(input, draft)}\n\n${jsonInstruction(VERIFY_TOOL.input_schema)}`);
|
|
206
|
+
const verdict = verdictFromText(text);
|
|
207
|
+
if (!verdict)
|
|
208
|
+
throw new Error(`${this.name}: no usable verdict JSON in output`);
|
|
209
|
+
return verdict;
|
|
210
|
+
}
|
|
179
211
|
}
|
|
180
212
|
// A model id comes from a HUNCH_*_MODEL env var and ends up as an argv token that,
|
|
181
213
|
// on Windows, pexecIn joins into the cmd.exe line (shell:true, to resolve the npm
|
|
@@ -460,7 +492,8 @@ const dedupLines = (xs) => [...new Set(xs.map((s) => s.trim()).filter(Boolean))]
|
|
|
460
492
|
/** Reconcile N worker drafts into one. DETERMINISTIC (no second LLM call): the richest
|
|
461
493
|
* draft is the spine; alternatives/consequences are unioned; confidence is AGREEMENT-
|
|
462
494
|
* WEIGHTED and CAPPED at 0.78 — below STRICT_MIN_CONFIDENCE (0.8) — so an ensemble
|
|
463
|
-
* auto-draft can never arm enforcement.
|
|
495
|
+
* auto-draft can never arm enforcement. `samples`/`agreement` ride along as advisory
|
|
496
|
+
* telemetry for `hunch review` (not schema-bound). */
|
|
464
497
|
export function mergeDecisionDrafts(drafts) {
|
|
465
498
|
const primary = [...drafts].sort((a, b) => b.confidence - a.confidence || b.decision.length - a.decision.length)[0];
|
|
466
499
|
const agreement = meanAgreement(drafts);
|
|
@@ -472,19 +505,40 @@ export function mergeDecisionDrafts(drafts) {
|
|
|
472
505
|
alternatives_rejected: dedupLines(drafts.flatMap((d) => d.alternatives_rejected)),
|
|
473
506
|
confidence: Math.min(0.78, 0.55 + 0.23 * agreement),
|
|
474
507
|
source: "llm_draft+ensemble",
|
|
508
|
+
samples: drafts.length,
|
|
509
|
+
agreement: Math.round(agreement * 100) / 100,
|
|
475
510
|
};
|
|
476
511
|
}
|
|
512
|
+
// Default self-consistency depth when only ONE subscription CLI is installed (the
|
|
513
|
+
// common case): sample it this many times and reconcile, so single-CLI users get
|
|
514
|
+
// ensemble-like robustness. Tunable per-call via `--samples`.
|
|
515
|
+
const DEFAULT_SAMPLES = 2;
|
|
477
516
|
export class EnsembleProvider {
|
|
478
517
|
workers;
|
|
479
518
|
name = "ensemble";
|
|
480
|
-
|
|
519
|
+
samples;
|
|
520
|
+
constructor(workers, opts = {}) {
|
|
481
521
|
this.workers = workers;
|
|
522
|
+
// Default 1 (single worker → passthrough); the self-consistency policy default
|
|
523
|
+
// lives at the selection layer (selectEnsemble). Coerce to a finite integer in a
|
|
524
|
+
// sane 1..5 band — a NaN here would make decisionTasks build ZERO tasks and throw,
|
|
525
|
+
// silently collapsing --deep to the deterministic fallback (callers also sanitize).
|
|
526
|
+
const n = Math.trunc(Number(opts.samples));
|
|
527
|
+
this.samples = Number.isFinite(n) ? Math.max(1, Math.min(5, n)) : 1;
|
|
482
528
|
}
|
|
483
529
|
async available() { return this.workers.length > 0; }
|
|
530
|
+
/** The draft tasks to fan out: one per distinct CLI when several are installed
|
|
531
|
+
* (cross-model ensemble), else N self-consistency samples of the single CLI. */
|
|
532
|
+
decisionTasks(input) {
|
|
533
|
+
if (this.workers.length >= 2)
|
|
534
|
+
return this.workers.map((w) => () => w.draftDecision(input));
|
|
535
|
+
const w = this.workers[0];
|
|
536
|
+
return Array.from({ length: this.samples }, () => () => w.draftDecision(input));
|
|
537
|
+
}
|
|
484
538
|
async draftDecision(input) {
|
|
485
539
|
if (!this.workers.length)
|
|
486
540
|
throw new Error("ensemble: no subscription CLI workers available");
|
|
487
|
-
const settled = await Promise.allSettled(this.
|
|
541
|
+
const settled = await Promise.allSettled(this.decisionTasks(input).map((t) => t()));
|
|
488
542
|
const drafts = settled.flatMap((s) => (s.status === "fulfilled" ? [s.value] : []));
|
|
489
543
|
if (!drafts.length)
|
|
490
544
|
throw new Error("ensemble: all workers failed");
|
|
@@ -502,11 +556,105 @@ export class EnsembleProvider {
|
|
|
502
556
|
}
|
|
503
557
|
}
|
|
504
558
|
/** Build the Deep-Synthesis provider, or null if no subscription CLI is available
|
|
505
|
-
* (the caller then falls back to the normal single-provider path).
|
|
506
|
-
|
|
559
|
+
* (the caller then falls back to the normal single-provider path). `samples` sets
|
|
560
|
+
* the self-consistency depth for the single-CLI case. */
|
|
561
|
+
export async function selectEnsemble(opts = {}) {
|
|
562
|
+
const workers = await selectWorkers();
|
|
563
|
+
// The self-consistency policy default (DEFAULT_SAMPLES) is applied HERE, not in the
|
|
564
|
+
// provider — so a single CLI under --deep is sampled N times, while direct
|
|
565
|
+
// construction stays passthrough. `--samples 1` opts back out.
|
|
566
|
+
return workers.length ? new EnsembleProvider(workers, { samples: opts.samples ?? DEFAULT_SAMPLES }) : null;
|
|
567
|
+
}
|
|
568
|
+
/** Pick a CLI provider to run the Critic pass (subscription-only, like the workers).
|
|
569
|
+
* Returns null when no assistant CLI is installed — verification then no-ops and the
|
|
570
|
+
* un-audited draft stands (graceful degradation; dec_18a81c8291). */
|
|
571
|
+
export async function selectVerifier() {
|
|
507
572
|
const workers = await selectWorkers();
|
|
508
|
-
return workers
|
|
573
|
+
return workers[0] ?? null;
|
|
574
|
+
}
|
|
575
|
+
// ---- Verification (the Critic pass) ---------------------------------------
|
|
576
|
+
// Audit a draft against the commit it came from, then PRUNE unsupported
|
|
577
|
+
// alternatives/consequences and LOWER confidence on weak grounding. It may only
|
|
578
|
+
// reduce trust, never raise it past the cap — auto-drafts stay advisory and a human
|
|
579
|
+
// `hunch review --accept` remains the ONLY path to enforcement (dec_9a2f2fe72a).
|
|
580
|
+
const VERIFY_SYSTEM = `You are a skeptical auditor for an Engineering Memory OS. You are given a
|
|
581
|
+
synthesized decision record and the ACTUAL commit it was derived from. Your job is to
|
|
582
|
+
flag everything the record asserts that the evidence does NOT support — be strict; when
|
|
583
|
+
in doubt, flag it. Do not invent new content; only judge what is present.`;
|
|
584
|
+
function verifyPrompt(input, draft) {
|
|
585
|
+
const alts = draft.alternatives_rejected.length
|
|
586
|
+
? draft.alternatives_rejected.map((a, i) => ` ${i + 1}. ${a}`).join("\n")
|
|
587
|
+
: " (none)";
|
|
588
|
+
const cons = draft.consequences.length ? draft.consequences.map((c) => ` - ${c}`).join("\n") : " (none)";
|
|
589
|
+
return [
|
|
590
|
+
`COMMIT SUBJECT: ${input.subject}`,
|
|
591
|
+
input.body ? `COMMIT BODY:\n${input.body}` : "",
|
|
592
|
+
input.analysis ? `STRUCTURED CHANGES: ${summarizeDiff(input.analysis)}` : "",
|
|
593
|
+
renderDiff(input),
|
|
594
|
+
`CANDIDATE DECISION UNDER AUDIT:`,
|
|
595
|
+
` decision: ${draft.decision}`,
|
|
596
|
+
` consequences:\n${cons}`,
|
|
597
|
+
` alternatives_rejected:\n${alts}`,
|
|
598
|
+
`\nReturn grounded (0..1) and the VERBATIM alternatives_rejected / consequences the evidence does NOT support.`,
|
|
599
|
+
].filter(Boolean).join("\n\n");
|
|
600
|
+
}
|
|
601
|
+
/** Map model text → VerifyVerdict, or null when nothing usable parses (→ the caller
|
|
602
|
+
* keeps the un-audited draft). Tolerant of arrays-as-strings and missing fields. */
|
|
603
|
+
export function verdictFromText(text) {
|
|
604
|
+
for (const obj of extractJsonObjects(text)) {
|
|
605
|
+
const hasGrounded = typeof obj.grounded === "number";
|
|
606
|
+
const ua = asStrArr(obj.unsupported_alternatives);
|
|
607
|
+
const uc = asStrArr(obj.unsupported_claims);
|
|
608
|
+
if (!hasGrounded && !ua.length && !uc.length)
|
|
609
|
+
continue; // unrelated object
|
|
610
|
+
const grounded = typeof obj.grounded === "number" ? clamp01(obj.grounded) : 1;
|
|
611
|
+
return { grounded, unsupported_alternatives: ua, unsupported_claims: uc };
|
|
612
|
+
}
|
|
613
|
+
return null;
|
|
614
|
+
}
|
|
615
|
+
const norm = (s) => s.trim().toLowerCase().replace(/\s+/g, " ");
|
|
616
|
+
/** True if `flagged` names `entry` — exact normalized match, or a substantial
|
|
617
|
+
* (≥8 char) containment either way, to absorb minor rewording by the auditor
|
|
618
|
+
* without nuking unrelated entries. */
|
|
619
|
+
function flaggedMatches(entry, flagged) {
|
|
620
|
+
const e = norm(entry);
|
|
621
|
+
if (!e)
|
|
622
|
+
return false;
|
|
623
|
+
return flagged.some((f) => {
|
|
624
|
+
const n = norm(f);
|
|
625
|
+
if (!n)
|
|
626
|
+
return false;
|
|
627
|
+
if (n === e)
|
|
628
|
+
return true;
|
|
629
|
+
return n.length >= 8 && e.length >= 8 && (e.includes(n) || n.includes(e));
|
|
630
|
+
});
|
|
631
|
+
}
|
|
632
|
+
/** Apply a verdict to a draft: drop unsupported alternatives (so they never scaffold
|
|
633
|
+
* tripwires) and consequences, and scale confidence DOWN by grounding. Confidence is
|
|
634
|
+
* clamped so it can only fall — verification never arms a stronger claim than the
|
|
635
|
+
* draft already made (R2). Records `grounded` as advisory telemetry. */
|
|
636
|
+
export function applyVerdict(draft, v) {
|
|
637
|
+
const alternatives_rejected = draft.alternatives_rejected.filter((a) => !flaggedMatches(a, v.unsupported_alternatives));
|
|
638
|
+
const consequences = draft.consequences.filter((c) => !flaggedMatches(c, v.unsupported_claims));
|
|
639
|
+
const grounded = clamp01(v.grounded);
|
|
640
|
+
// Penalize weak grounding; (0.5 + 0.5*grounded) ∈ [0.5,1], so this only lowers.
|
|
641
|
+
const confidence = Math.min(draft.confidence, Math.round(draft.confidence * (0.5 + 0.5 * grounded) * 100) / 100);
|
|
642
|
+
const source = draft.source.includes("verified") ? draft.source : `${draft.source}+verified`;
|
|
643
|
+
return { ...draft, alternatives_rejected, consequences, confidence, grounded, source };
|
|
644
|
+
}
|
|
645
|
+
/** Run the Critic pass and apply it, degrading to the un-audited draft on any failure
|
|
646
|
+
* or when the provider can't verify (deterministic / no CLI). Never throws. */
|
|
647
|
+
export async function verifyDecisionSafe(verifier, input, draft) {
|
|
648
|
+
if (!verifier?.verifyDecision)
|
|
649
|
+
return draft;
|
|
650
|
+
try {
|
|
651
|
+
return applyVerdict(draft, await verifier.verifyDecision(input, draft));
|
|
652
|
+
}
|
|
653
|
+
catch {
|
|
654
|
+
return draft;
|
|
655
|
+
}
|
|
509
656
|
}
|
|
657
|
+
const clamp01 = (n) => (Number.isFinite(n) ? Math.max(0, Math.min(1, n)) : 1);
|
|
510
658
|
// ---- prompt + parsing helpers --------------------------------------------
|
|
511
659
|
// Above this size we stop shipping the raw patch and lean on the deterministic
|
|
512
660
|
// STRUCTURED CHANGES summary + a small sample. A truncated head-slice of a giant
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { commitMeta, commitDiff, headSha } from "../extractors/git.js";
|
|
2
2
|
import { analyzeDiff } from "../extractors/diff.js";
|
|
3
|
-
import { selectProvider, selectEnsemble, DeterministicProvider } from "./provider.js";
|
|
3
|
+
import { selectProvider, selectEnsemble, selectVerifier, verifyDecisionSafe, DeterministicProvider } from "./provider.js";
|
|
4
4
|
import { decisionId, bugId, constraintId } from "../core/ids.js";
|
|
5
5
|
import { pathMatchesGlob } from "../core/glob.js";
|
|
6
6
|
import { draftTripwires, knownRepoDeps } from "./tripwires.js";
|
|
@@ -69,13 +69,32 @@ export async function syncCommit(store, root, sha, opts = {}) {
|
|
|
69
69
|
// Deep Synthesis (--deep): ensemble every available subscription CLI and reconcile
|
|
70
70
|
// their drafts (agreement-weighted, confidence capped below the strict gate). Falls
|
|
71
71
|
// back to the normal single-provider path when no CLI is available. Opt-in only.
|
|
72
|
+
// --verify forces the LLM provider (auditing a deterministic draft is pointless) and,
|
|
73
|
+
// like --deep, runs the Critic pass below. Subscription-only throughout (con_2ce3f2a547).
|
|
74
|
+
const wantVerify = !!(opts.verify || opts.deep);
|
|
72
75
|
const provider = opts.deep
|
|
73
|
-
? (await selectEnsemble()) ?? await selectProvider()
|
|
74
|
-
: opts.force || isSignificant(meta, analysis, codeFiles)
|
|
76
|
+
? (await selectEnsemble({ samples: opts.samples })) ?? await selectProvider()
|
|
77
|
+
: opts.force || opts.verify || isSignificant(meta, analysis, codeFiles)
|
|
75
78
|
? await selectProvider()
|
|
76
79
|
: new DeterministicProvider();
|
|
77
80
|
const input = { subject: meta.subject, body: meta.body, files: codeFiles, diff, analysis };
|
|
78
|
-
|
|
81
|
+
let draft = await draftDecisionSafe(provider, input);
|
|
82
|
+
// The Critic pass: audit the draft against the commit, PRUNE unsupported alternatives
|
|
83
|
+
// (BEFORE they scaffold tripwires below) and consequences, and lower confidence on weak
|
|
84
|
+
// grounding. No-ops when no assistant CLI is available; never raises trust (dec_9a2f2fe72a).
|
|
85
|
+
if (wantVerify)
|
|
86
|
+
draft = await verifyDecisionSafe(await selectVerifier(), input, draft);
|
|
87
|
+
// Advisory synthesis telemetry for `hunch review` — which provider ran, how many drafts
|
|
88
|
+
// were reconciled, their agreement, and the verifier's grounding. Rides in `evidence`
|
|
89
|
+
// (no schema change → respects forward-migration invariant con_947c578b2c).
|
|
90
|
+
const synthBits = [`provider=${provider.name}`];
|
|
91
|
+
if (draft.samples)
|
|
92
|
+
synthBits.push(`samples=${draft.samples}`);
|
|
93
|
+
if (draft.agreement != null)
|
|
94
|
+
synthBits.push(`agreement=${draft.agreement}`);
|
|
95
|
+
if (draft.grounded != null)
|
|
96
|
+
synthBits.push(`grounded=${draft.grounded}`);
|
|
97
|
+
const synthEvidence = `synth:${synthBits.join(" ")}`;
|
|
79
98
|
const components = store.json.loadAll("components");
|
|
80
99
|
const relatedComponents = components
|
|
81
100
|
.filter((c) => codeFiles.some((f) => c.paths.some((g) => pathMatchesGlob(f, g))))
|
|
@@ -119,7 +138,7 @@ export async function syncCommit(store, root, sha, opts = {}) {
|
|
|
119
138
|
provenance: {
|
|
120
139
|
source: draft.source,
|
|
121
140
|
confidence: draft.confidence,
|
|
122
|
-
evidence: [`commit:${meta.shortSha}`, ...codeFiles.slice(0, 8)],
|
|
141
|
+
evidence: [`commit:${meta.shortSha}`, synthEvidence, ...codeFiles.slice(0, 8)],
|
|
123
142
|
last_verified: new Date().toISOString(), // when the Hunch last re-derived this
|
|
124
143
|
},
|
|
125
144
|
date: meta.date, // the commit date
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@davesheffer/hunch",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.19.0",
|
|
4
4
|
"license": "Apache-2.0",
|
|
5
5
|
"author": "Dave Sheffer <dave.sheffer1@gmail.com>",
|
|
6
6
|
"description": "Hunch — an Engineering Memory OS: a persistent, git-native reasoning graph over a codebase, exposed to Claude Code via MCP.",
|