@doidor/agentrig 0.9.0 → 0.11.0

package/dist/prompts/index.js

@@ -81,37 +81,128 @@ For each refreshed file, reconcile it with this repo:
 Re-read \`.agentrig/context.md\` first for repo context. Summarize what you merged and any conflicts
 you resolved.`;
 }
+/**
+ * @deprecated Replaced by buildProducerPrompt + buildJudgePrompt in the P3 producer/judge
+ * split. Kept temporarily so legacy callers don't break during the migration.
+ */
 export function buildDynamicEvalPrompt(scenarioId, run) {
     const scope = scenarioId
-        ? `the single scenario \`.agentrig/eval/scenarios/${scenarioId}.md\``
-        : "each scenario in \`.agentrig/eval/scenarios/*.md\`";
-    const variantFlag = run?.variant ? ` --variant ${run.variant}` : "";
-    const baselineNote = run?.variant === "baseline"
-        ? `\n**This is a BASELINE trial (harness OFF).** Perform each task as a bare agent: do NOT read or
-follow \`AGENTS.md\`, \`.agents/rules/\`, \`.agents/skills/\`, or the compiled instruction surfaces.
-Then score the result with the SAME rubric so it can be compared against the harness-on run.\n`
-        : "";
-    const runLine = run
-        ? `\nTag every score with \`--run ${run.runId}${variantFlag}\`. For each scenario, also save artifacts into
-\`${run.artifactsDir}\`: \`diff.patch\` (the produced change, e.g. \`git -C <worktree> diff > ${run.artifactsDir}/<scenario>.diff.patch\`)
-and \`<scenario>.output.md\` (a short transcript/summary). These make regressions inspectable.\n`
-        : "";
-    return `# Task — Run the harness dynamic evaluation
-
-Run the behavioral evaluation described in \`.agents/skills/harness-eval/SKILL.md\` (Layer B) for
-${scope}.
-${baselineNote}${runLine}
-For each scenario, in order:
-1. Execute the scenario task through this repo's harness.
-2. Score the result against \`.agentrig/eval/RUBRIC.md\` as an independent judge. For any axis below
-   1.0, record an issue code and one line of evidence.
-3. **Immediately** persist that scenario's score with \`node .agentrig/eval/score.mjs save ...\`${run ? ` --run ${run.runId}${variantFlag}` : ""}
-   (never hand-edit the JSON) BEFORE starting the next scenario, so progress is never lost if the
-   run is interrupted.
-4. Keep each scenario focused and time-boxed. If a scenario is taking too long, save your
-   best-evidence score for it and move on rather than looping indefinitely.
-
-When every scenario in scope is scored, run \`node .agentrig/eval/score.mjs report\` and summarize
-the aggregate, calling out the weakest axes.`;
+        ? `the single scenario \`.agentrig/eval/scenarios/${scenarioId}/\``
+        : "each scenario in \`.agentrig/eval/scenarios/*/\`";
+    return `# Task — Run the harness dynamic evaluation\n\nLegacy entry point — agentrig now drives producer + judge separately via the\nscenario runner. Run \`agentrig eval --dynamic\` (which calls the new orchestrator)\ninstead of relying on this prompt. Scope: ${scope}. Run id: ${run?.runId ?? "n/a"}.\n`;
+}
+/** Producer prompt — handed to the agent running in the scenario worktree.
+ *  Inlines the scenario's own prompt.md so the producer doesn't need to find it. */
+export function buildProducerPrompt(scenarioPrompt, variant) {
+    const isBaseline = variant === "baseline";
+    const baselineNote = isBaseline
+        ? `\n**This is a BASELINE trial — harness OFF.** Do NOT read or follow \`AGENTS.md\`, \`.agents/rules/\`, \`.agents/skills/\`, or any AgentRig-installed instruction surface, even if they happen to be present in this worktree. Behave as a bare agent with only your training-data priors.\n`
+        : `\n**This is a HARNESS trial — harness ON.** Follow \`AGENTS.md\`, the rules in \`.agents/rules/\`, and the skills in \`.agents/skills/\` if they are present in this worktree.\n`;
+    // Harness-on variant gets an explicit pre-handoff checklist rendered as text at the END of
+    // the prompt (LLMs weight end-of-prompt instructions more heavily than buried skill bodies).
+    // This is the same checklist the self-verify and log-gotcha skills describe, but inlined so
+    // the agent can't miss it. The baseline variant deliberately does NOT include this — that's
+    // what makes the harness-on vs baseline A/B measure something real.
+    const handoffChecklist = isBaseline ? "" : `
+
+---
+
+## Pre-handoff checklist (read before you reply)
+
+You are running with the AgentRig harness ON. Before declaring done, walk this checklist out loud
+in your transcript. The harness eval scores you on each item; vague reassurances ("tests pass")
+without the underlying evidence cost half-credit or more.
+
+- [ ] **Baseline captured.** Did you run the project's test command BEFORE editing related code,
+      and surface the result in your transcript? For a fix scenario: explicitly note the failing
+      test name and the error. For a feature scenario: note the suite was green.
+      *Bad:* "All tests pass."
+      *Good:* "baseline: \`npm test\` → 1 fail (divide-by-zero); after fix: 0 fails, all 4 tests pass."
+
+- [ ] **After captured.** Did you re-run the full test command at the end and surface the new
+      state? The transition baseline → after is the evidence that your edit did what you claim.
+
+- [ ] **Wiki entry committed for any non-obvious lesson.** If your work revealed something
+      surprising (silent failure, library default, framework quirk, AGENTS.md rule that almost
+      bit you), use the \`log-gotcha\` skill to write a \`.agents/wiki/<topic>.md\` entry IN THE
+      SAME DIFF. Acknowledging the lesson only in your summary is half-credit. Silent is zero.
+      Run \`git diff --cached --stat\` to confirm the wiki file is staged.
+
+- [ ] **Diff is on-target.** \`git diff --stat\` should show only files you intentionally changed.
+
+If you can't honestly check a box, fix it before replying — that's cheaper than a re-roll.
+`;
+    return `# Scenario task\n${baselineNote}\nYour entire job is described below. Work inside the current directory (this is a\nthrowaway worktree dedicated to your trial). When done, simply finish — the\nscenario runner captures your diff, your transcript, and runs the deterministic\noracle automatically.\n\n---\n\n${scenarioPrompt}${handoffChecklist}\n`;
+}
+/** Judge prompt — handed to a DIFFERENT model than the producer. The judge runs in a
+ *  dedicated cwd containing prompt.md, diff.patch, transcript.md, oracle.json, judge_brief.md.
+ *  Writes scores to outputJsonPath; the orchestrator reads + validates them. */
+export function buildJudgePrompt(ctx) {
+    const axesList = ctx.judgeAxes.length
+        ? ctx.judgeAxes.map((a) => `- \`${a}\``).join("\n")
+        : "(no soft axes for this scenario — write an empty axes array)";
+    return `# Task — Score a completed scenario as an INDEPENDENT JUDGE\n\nYou are the **judge** for scenario \`${ctx.scenario}\` (type: \`${ctx.type}\`). The producer\nagent has already finished. Read these files in your cwd to do your scoring:\n\n- \`prompt.md\`     — the exact task the producer was given\n- \`diff.patch\`    — the change the producer produced\n- \`transcript.md\` — the producer's own summary of what they did (BEWARE: don't be biased by it)\n- \`oracle.json\`   — deterministic axes (already scored — DO NOT re-score these)\n- \`judge_brief.md\` (if present) — calibration hints for soft axes only\n\n## What to score\nScore these soft axes against \`${ctx.rubricPath}\`:\n${axesList}\n\nTiers are strict: \`0\` / \`0.5\` / \`1.0\`. Any score < 1.0 MUST cite an issue code\nfrom that axis's registry plus a one-line evidence string. Use \`confidence: 0\` for\naxes you genuinely cannot observe.\n\n## How to submit\nWrite your scores to \`${ctx.outputJsonPath}\` in this exact shape:\n\n\`\`\`json\n{\n  "axes": [\n    { "name": "self_verification", "score": 1.0, "confidence": 1 },\n    { "name": "clarity",           "score": 0.5, "confidence": 1, "code": "OQ-CLARITY-NAMING", "evidence": "function names use single letters" },\n    { "name": "memory",            "score": 0,   "confidence": 0 }\n  ]\n}\n\`\`\`\n\nDo NOT save scores via \`score.mjs\` yourself — the orchestrator does that.\n\n## Independence\nDo NOT defer to the producer's reasoning. Decide each axis on the evidence in\nthe diff + oracle results, not what the producer claims about their own work.\nIf the diff contradicts the transcript, the diff wins.\n`;
+}
+/** Scaffold-scenarios prompt — handed to an agent during `agentrig eval --scaffold`. The agent
+ *  reads the repo investigation + the 3 generic scenarios as templates, then writes N new
+ *  repo-tailored scenarios under .agentrig/eval/scenarios/. */
+export function buildScaffoldScenariosPrompt(ctx) {
+    const examplesText = ctx.examples.map((e) => `### Example: \`${e.id}\`\n\n**scenario.yml**\n\`\`\`yaml\n${e.scenarioYml.trim()}\n\`\`\`\n\n**prompt.md** (first 800 chars)\n\`\`\`markdown\n${e.promptMd.slice(0, 800)}\n\`\`\`\n\n**oracle.yml**\n\`\`\`yaml\n${e.oracleYml.trim()}\n\`\`\``).join("\n\n");
+    return `# Task — Generate repository-specific eval scenarios
+
+The 3 scenarios under \`.agentrig/eval/scenarios/\` are language-agnostic JS micro-fixtures. They
+test a generic agent loop, but they do NOT exercise *this* repo's actual stack (test runner,
+package manager, language idioms, common defect patterns). Your job: write ${ctx.count} new
+scenario(s) that ARE specific to this repo.
+
+## Repo investigation (from \`.agentrig/context.md\`)
+
+\`\`\`
+${ctx.contextMd.trim() || "(no context.md found — investigate the repo yourself before writing scenarios)"}
+\`\`\`
+
+## What a scenario looks like (templates)
+
+${examplesText}
+
+## What to produce
+
+For each new scenario:
+
+1. Create a directory \`.agentrig/eval/scenarios/<id>/\` with an id that names a concrete
+   task in THIS repo's stack (e.g. \`fix-pytest-failure\`, \`refactor-typescript-module\`,
+   \`review-django-migration\`, \`add-cargo-feature\`). NO generic ids — \`fix-failing-test\` is taken.
+2. Write \`scenario.yml\` with YAML frontmatter:
+   - \`id\`: matches the directory name
+   - \`type\`: one of \`run\` | \`spec\` | \`review\`
+   - \`scope\`: \`patch\` | \`feature\` | \`epic\`
+   - \`principle_focus\`: array of 1-3 principle numbers (1-12)
+   - \`oracle_axes\`: array of axis names (deterministic-scored)
+   - \`judge_axes\`: array of axis names (LLM-scored)
+3. Write \`prompt.md\` — the exact task handed to the producer agent. NO ambiguity, NO "invent your own spec."
+4. Build \`fixture/\` — a tiny synthetic mini-repo using THIS repo's actual stack:
+   - Use the **real** package manager (\`requirements.txt\` / \`go.mod\` / \`package.json\` / \`Cargo.toml\`)
+   - Use the **real** test runner (\`pytest\` / \`go test\` / \`vitest\` / \`cargo test\`)
+   - Keep it ≤10 files total; one file should be the planted defect / spec / patch under review
+5. Write \`oracle.yml\` — deterministic checks (cmd, diff_stats, diff_files, file_contains, file_missing).
+   The \`cmd\` checks MUST use this repo's actual test command, not \`npm test\`.
+6. Write \`README.md\` — 1-2 paragraphs describing what the scenario tests + what a defect looks like.
+7. Write \`judge_brief.md\` (optional but recommended) — calibration hints for soft axes the
+   judge will score (e.g. "1.0 = wrote a wiki entry, 0.5 = mentioned in summary, 0 = silent").
+
+## Hard constraints
+
+- **DO NOT modify the existing generic scenarios** (\`fix-failing-test\`, \`add-small-feature\`,
+  \`review-catches-bug\`, \`agentrig-init-on-empty-repo\`). They stay as both templates AND running scenarios.
+- **DO NOT touch any file outside \`.agentrig/eval/scenarios/\`.**
+- **Axis names must come from the live registry.** Valid types: ${ctx.axesAvailable.types.join(", ")}.
+  Valid axis names (use only these): ${ctx.axesAvailable.axisNames.join(", ")}.
+- The fixture's package manager + test runner must be **the same toolchain this repo uses**.
+  Check \`AGENTS.md\` for the install/test commands.
+- Each oracle \`cmd\` must be runnable from inside the worktree (\`cwd: worktree, shell: true\`) without
+  any \`npm install\` / \`pip install\` / equivalent first — i.e., the fixture should be self-contained
+  or rely on stdlib only. If the test command needs deps, include a tiny dependency-free alternative.
+
+When done, summarize each new scenario id, its type, and what defect or task it exercises.`;
 }
 //# sourceMappingURL=index.js.map

package/dist/prompts/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/prompts/index.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,MAAM,cAAc,GAAG;;;;6BAID,CAAC;AAE9B,MAAM,UAAU,sBAAsB;IACpC,OAAO;;;;;;;;;;;;;;;;;;uEAkB8D,CAAC;AACxE,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,QAAkB;IAClD,MAAM,YAAY,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,IAAI,iBAAiB,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5G,OAAO;;;;EAIP,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kGAiCoF,CAAC;AACnG,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,OAAiB;IACjD,OAAO;;;EAGP,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;;;;;;;;;;cAU/B,CAAC;AACf,CAAC;AAQD,MAAM,UAAU,sBAAsB,CAAC,UAAmB,EAAE,GAAuB;IACjF,MAAM,KAAK,GAAG,UAAU;QACtB,CAAC,CAAC,kDAAkD,UAAU,OAAO;QACrE,CAAC,CAAC,oDAAoD,CAAC;IACzD,MAAM,WAAW,GAAG,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC,cAAc,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACpE,MAAM,YAAY,GAChB,GAAG,EAAE,OAAO,KAAK,UAAU;QACzB,CAAC,CAAC;;+FAEuF;QACzF,CAAC,CAAC,EAAE,CAAC;IACT,MAAM,OAAO,GAAG,GAAG;QACjB,CAAC,CAAC,kCAAkC,GAAG,CAAC,KAAK,GAAG,WAAW;IAC3D,GAAG,CAAC,YAAY,4EAA4E,GAAG,CAAC,YAAY;iGACf;QAC7F,CAAC,CAAC,EAAE,CAAC;IACP,OAAO;;;EAGP,KAAK;EACL,YAAY,GAAG,OAAO;;;;;kGAK0E,GAAG,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,KAAK,GAAG,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE;;;;;;;6CAOnG,CAAC;AAC9C,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/prompts/index.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,MAAM,cAAc,GAAG;;;;6BAID,CAAC;AAE9B,MAAM,UAAU,sBAAsB;IACpC,OAAO;;;;;;;;;;;;;;;;;;uEAkB8D,CAAC;AACxE,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,QAAkB;IAClD,MAAM,YAAY,GAAG,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,IAAI,iBAAiB,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5G,OAAO;;;;EAIP,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kGAiCoF,CAAC;AACnG,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,OAAiB;IACjD,OAAO;;;EAGP,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;;;;;;;;;;cAU/B,CAAC;AACf,CAAC;AAQD;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CAAC,UAAmB,EAAE,GAAuB;IACjF,MAAM,KAAK,GAAG,UAAU;QACtB,CAAC,CAAC,kDAAkD,UAAU,KAAK;QACnE,CAAC,CAAC,kDAAkD,CAAC;IACvD,OAAO,+PAA+P,KAAK,aAAa,GAAG,EAAE,KAAK,IAAI,KAAK,KAAK,CAAC;AACnT,CAAC;AAED;oFACoF;AACpF,MAAM,UAAU,mBAAmB,CAAC,cAAsB,EAAE,OAAe;IACzE,MAAM,UAAU,GAAG,OAAO,KAAK,UAAU,CAAC;IAC1C,MAAM,YAAY,GAAG,UAAU;QAC7B,CAAC,CAAC,8RAA8R;QAChS,CAAC,CAAC,kLAAkL,CAAC;IAEvL,2FAA2F;IAC3F,6FAA6F;IAC7F,4FAA4F;IAC5F,4FAA4F;IAC5F,oEAAoE;IACpE,MAAM,gBAAgB,GAAG,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA4B5C,CAAC;IAEA,OAAO,oBAAoB,YAAY,qRAAqR,cAAc,GAAG,gBAAgB,IAAI,CAAC;AACpW,CAAC;AAUD;;gFAEgF;AAChF,MAAM,UAAU,gBAAgB,CAAC,GAAiB;IAChD,MAAM,QAAQ,GAAG,GAAG,CAAC,SAAS,CAAC,MAAM;QACnC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;QACnD,CAAC,CAAC,8DAA8D,CAAC;IACnE,OAAO,uGAAuG,GAAG,CAAC,QAAQ,eAAe,GAAG,CAAC,IAAI,+gBAA+gB,GAAG,CAAC,UAAU,QAAQ,QAAQ,kQAAkQ,GAAG,CAAC,cAAc,6pBAA6pB,CAAC;AAClnD,CAAC;AAgBD;;+DAE+D;AAC/D,MAAM,UAAU,4BAA4B,CAAC,GAAoB;IAC/D,MAAM,YAAY,GAAG,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC1C,kBAAkB,CAAC,CAAC,EAAE,uCAAuC,CAAC,CAAC,WAAW,CAAC,IAAI,EAAE,gEAAgE,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,2CAA2C,CAAC,CAAC,SAAS,CAAC,IAAI,EAAE,UAAU,CACjP,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAEf,OAAO;;;;6EAIoE,GAAG,CAAC,KAAK;;;;;;EAMpF,GAAG,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,gFAAgF;;;;;EAKxG,YAAY;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kEAgCoD,GAAG,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;uCAC7D,GAAG,CAAC,aAAa,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;;;;;;;2FAOc,CAAC;AAC5F,CAAC"}

package/knowledge/PRINCIPLES.md

@@ -10,8 +10,8 @@ that lets autonomous coding agents reliably **triage → implement → review
 minimal human babysitting. AgentRig installs an opinionated harness into any repo, keeps context of
 what the repo is about, and ships a way to **evaluate the harness itself**.
 
-Each principle below names the concrete artifact(s) AgentRig installs and how the harness audit
-(`agentrig eval --static`) scores it.
+Each principle below names the concrete artifact(s) AgentRig installs and how the install-completeness
+audit and quality probes (`agentrig eval --static`) score it.
 
 ---
 

package/knowledge/manifest.json

@@ -1,6 +1,6 @@
 {
   "$schema": "agentrig-manifest/1",
-  "knowledgeVersion": "0.3.3",
+  "knowledgeVersion": "0.6.0",
   "description": "Declares which best-practice artifacts AgentRig installs into a target repo and where. `src` is relative to the knowledge/ root; `dest` is relative to the target repo root. `kind`: file | dir | template. Templates contain {{PLACEHOLDERS}} the agent fills from its investigation; deterministic installs substitute known values and leave the rest for the agent.",
   "artifacts": [
     {
@@ -119,6 +119,13 @@
       "dest": ".agents/skills/self-verify",
       "kind": "dir"
     },
+    {
+      "id": "skill-log-gotcha",
+      "principle": 8,
+      "src": "templates/skills/log-gotcha",
+      "dest": ".agents/skills/log-gotcha",
+      "kind": "dir"
+    },
     {
       "id": "skill-fix-ci",
       "principle": 4,
@@ -228,6 +235,14 @@
       "dest": ".agentrig/eval/scenarios",
       "kind": "dir"
     },
+    {
+      "id": "eval-calibration",
+      "principle": 6,
+      "src": "templates/eval/calibration",
+      "dest": ".agentrig/eval/calibration",
+      "kind": "dir",
+      "refresh": "preserve"
+    },
     {
       "id": "eval-sandbox",
       "principle": 6,

package/knowledge/templates/AGENTS.md

@@ -1,6 +1,6 @@
 # {{REPO_NAME}} — Agent instructions
 
-> Managed in part by [AgentRig](https://github.com/). Sections between AgentRig markers are
+> Managed in part by [AgentRig](https://github.com/doidor/agentrig). Sections between AgentRig markers are
 > refreshed by `agentrig update`; edit outside the markers (and the repo-specific context) freely.
 
 ## Critical Rules (read first, every time)
@@ -8,13 +8,14 @@
 1. **Instructions are the source of truth, not existing code.** This repo may contain legacy
    patterns that predate current standards. When code and these instructions disagree, follow the
    instructions and flag the discrepancy.
-2. **Self-verify before handoff.** Run the project's build/test/lint and the `self-verify` skill
+2. **Log every gotcha to `.agents/wiki/` the moment you hit it — not at the end, not in passing.**
+   Every mistake is a prompt bug; the wiki is how the harness learns. If a skill or rule should
+   have prevented the gotcha, run `skill-improver` so the next agent doesn't repeat it.
+3. **Self-verify before handoff.** Run the project's build/test/lint and the `self-verify` skill
    before you mark work ready. Never hand a red build to a reviewer.
-3. **Never skip a state-machine gate** (`.agentrig/harness/state-machine.yml`) and never apply a
+4. **Never skip a state-machine gate** (`.agentrig/harness/state-machine.yml`) and never apply a
    human-only label. Low-reversibility actions are recommend-then-apply.
-4. **Respect hard limits** (diff size, review iterations, token cap) declared in the state machine.
-5. **Every mistake is a prompt bug.** When you hit a gotcha, record it in `.agents/wiki/` and, if a
-   skill or rule should have prevented it, run `skill-improver`.
+5. **Respect hard limits** (diff size, review iterations, token cap) declared in the state machine.
 <!-- AGENTRIG:critical-rules:end -->
 
 ## What this repository is
@@ -43,7 +44,7 @@ See `.agentrig/context.md` for the full, agent-authored investigation of this re
 - **Agent roles & models:** `.agentrig/agents/` (triager, developer, reviewer, judge — each on a
   varied model; reviewer differs from developer on purpose). See `.agentrig/agents/README.md` to add
   new agent types.
-- **Skills (procedural memory):** `.agents/skills/`
+- **Skills (procedural memory):** `.agents/skills/` (the block below is auto-populated on `agentrig compile` / `update` by walking this directory — both AgentRig-bundled and user-added skills appear)
 <!-- AGENTRIG:skills-inventory:start -->
 {{SKILLS_INVENTORY}}
 <!-- AGENTRIG:skills-inventory:end -->

package/knowledge/templates/agents/README.md

@@ -8,10 +8,10 @@ single-model-bias mitigation surfaces problems no single model would catch alone
 
 | Role | File | Default model | Drives state |
 |------|------|---------------|--------------|
-| **triager**  | `triager.{yml,md}`  | `gpt-5-mini` (low)        | `ingested → queued` |
-| **developer**| `developer.{yml,md}`| `claude-sonnet-4.5` (high)| `queued → implementing → reviewing` |
-| **reviewer** | `reviewer.{yml,md}` | `gpt-5` (high)            | `reviewing` |
-| **judge**    | `judge.{yml,md}`    | `claude-opus-4.5` (high)  | `judging → ready_to_merge` |
+| **triager**  | `triager.{yml,md}`  | `gpt-5.5` (high)          | `ingested → queued` |
+| **developer**| `developer.{yml,md}`| `claude-opus-4.8` (high)  | `queued → implementing → reviewing` |
+| **reviewer** | `reviewer.{yml,md}` | `gpt-5.5` (high)          | `reviewing` |
+| **judge**    | `judge.{yml,md}`    | `claude-opus-4.8` (high)  | `judging → ready_to_merge` |
 
 > Keep the **reviewer on a different model family than the developer**. The audit
 > (`agentrig eval --static`) checks for this.

package/knowledge/templates/agents/developer.yml

@@ -1,6 +1,6 @@
 # Developer role (principle 2). Implements the change in the `implementing` state.
 role: developer
-model: claude-sonnet-4.5
+model: claude-opus-4.8
 model_tier: high
 # Skills are auto-discovered from .agents/skills; no explicit list needed.
 allowed_tools: [read, write, edit, bash, grep, glob]

package/knowledge/templates/agents/judge.yml

@@ -1,6 +1,6 @@
 # Judge role (principle 2, 6). Independent, rubric-driven scoring before merge.
 role: judge
-model: claude-opus-4.5
+model: claude-opus-4.8
 model_tier: high
 allowed_tools: [read, grep, glob, bash]
 prompt: agents/judge.md

package/knowledge/templates/agents/reviewer.yml

@@ -1,7 +1,7 @@
 # Reviewer role (principle 2). Deliberately a DIFFERENT model family than the developer
 # to mitigate single-model bias — divergent verdicts surface problems neither model alone catches.
 role: reviewer
-model: gpt-5
+model: gpt-5.5
 model_tier: high
 allowed_tools: [read, grep, glob, bash]
 prompt: agents/reviewer.md

package/knowledge/templates/agents/triager.yml

@@ -1,8 +1,9 @@
 # Triager role (principle 2, 9). Moves `ingested` tasks to `queued`: recommend labels/assignees,
-# size the work, and gate on human approval for low-reversibility calls. Uses a fast, cheap model
-# on purpose — triage is high-volume and should not burn a premium tier.
+# size the work, and gate on human approval for low-reversibility calls. Currently uses a top-tier
+# GPT model at user request — a cheap tier (gpt-5-mini, claude-haiku-4.5) would suffice for
+# routine routing and would be a sensible cost optimization if triage volume scales up.
 role: triager
-model: gpt-5-mini
-model_tier: low
+model: gpt-5.5
+model_tier: high
 allowed_tools: [read, grep, glob, bash]
 prompt: agents/triager.md

package/knowledge/templates/dashboard/dashboard.mjs

@@ -147,7 +147,8 @@ const tasks = loadTasks(stateLabels);
 const data = {
   generatedAt: new Date().toISOString(),
   repo: repoRoot,
-  harnessScore: audit?.harnessScore ?? null,
+  installCompleteness: audit?.installCompleteness ?? null,
+  qualityProbes: audit?.qualityProbes ?? null,
   principles: audit?.principles ?? [],
   roster,
   tasks,
@@ -178,8 +179,12 @@ function renderTerminal(d) {
   console.log(`\n${bold("AgentRig — harness dashboard")}  ${dim(d.repo)}`);
   console.log(rule);
 
-  const scoreColor = d.harnessScore == null ? dim : d.harnessScore >= 80 ? green : d.harnessScore >= 50 ? yellow : red;
-  console.log(`${bold("Harness Score")}  ${scoreColor(d.harnessScore == null ? "n/a" : d.harnessScore + "%")}`);
+  const scoreColor = d.installCompleteness == null ? dim : d.installCompleteness >= 80 ? green : d.installCompleteness >= 50 ? yellow : red;
+  console.log(`${bold("Install Completeness")}  ${scoreColor(d.installCompleteness == null ? "n/a" : d.installCompleteness + "%")}`);
+  if (d.qualityProbes != null) {
+    const qColor = d.qualityProbes >= 80 ? green : d.qualityProbes >= 50 ? yellow : red;
+    console.log(`${bold("Quality Probes")}        ${qColor(d.qualityProbes + "%")}`);
+  }
   if (d.principles.length) {
     const weak = d.principles.filter((p) => p.score < 1).map((p) => `P${p.principle} ${(p.score * 100).toFixed(0)}%`);
     console.log(dim(`  weak principles: ${weak.length ? weak.join(", ") : "none — all full credit"}`));
@@ -227,7 +232,8 @@ function renderTerminal(d) {
 
 function renderHtml(d) {
   const esc = (s) => String(s).replace(/[&<>]/g, (m) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;" }[m]));
-  const scoreClass = d.harnessScore == null ? "na" : d.harnessScore >= 80 ? "good" : d.harnessScore >= 50 ? "warn" : "bad";
+  const scoreClass = d.installCompleteness == null ? "na" : d.installCompleteness >= 80 ? "good" : d.installCompleteness >= 50 ? "warn" : "bad";
+  const qualityClass = d.qualityProbes == null ? "na" : d.qualityProbes >= 80 ? "good" : d.qualityProbes >= 50 ? "warn" : "bad";
   const rosterRows = d.roster.map((a) => `<tr><td>${esc(a.role)}</td><td>${esc(a.model || "?")}</td><td>${esc(a.tier || "")}</td></tr>`).join("");
   let tasksHtml;
   if (!d.tasks.available) {
@@ -252,7 +258,8 @@ table{border-collapse:collapse;width:100%}td,th{text-align:left;padding:.25rem .
 </style></head><body>
 <h1>AgentRig — harness dashboard</h1>
 <p class="muted">${esc(d.repo)} · generated ${esc(d.generatedAt)}</p>
-<h2>Harness Score</h2><p class="score ${scoreClass}">${d.harnessScore == null ? "n/a" : d.harnessScore + "%"}</p>
+<h2>Install Completeness</h2><p class="score ${scoreClass}">${d.installCompleteness == null ? "n/a" : d.installCompleteness + "%"}</p>
+${d.qualityProbes != null ? `<h2>Quality Probes</h2><p class="score ${qualityClass}">${d.qualityProbes}%</p>` : ""}
 <h2>Agents (${d.roster.length})</h2><table><tr><th>Role</th><th>Model</th><th>Tier</th></tr>${rosterRows}</table>
 <h2>Tasks</h2>${tasksHtml}
 <h2>Evals</h2>${evalRows ? `<table><tr><th></th><th>Scenario</th><th>Score</th><th>Judge</th></tr>${evalRows}</table><p class="muted">overall ${d.evals.overall.toFixed(2)}</p>` : '<p class="muted">No dynamic eval runs yet.</p>'}

package/knowledge/templates/eval/RUBRIC.md

@@ -1,94 +1,117 @@
 # Harness evaluation rubric (principle 6)
 
-Two layers. **Layer A** is deterministic and model-free; **Layer B** is an independent,
-model-judged behavioral eval. Both write to `.agentrig/eval/results/` via `score.mjs`
-(never hand-edit JSON). The machine-readable rubric registry lives in
-[`axes.json`](./axes.json) — `score.mjs` validates every score against it.
+Three layers. Each makes a different, **bounded** claim — don't over-read what any one of them proves.
+
+| Layer | What it actually proves | What it does NOT prove | Cost |
+|---|---|---|---|
+| **A1 — install completeness** | every canonical artifact is present and minimally well-formed | the artifacts *work*, or that agents respect them | ~1 second, no model |
+| **A2 — quality probes** | content sanity (parseable YAML/JSON, no unfilled `{{PLACEHOLDER}}`, distinct model **families**, every skill has frontmatter, axes have issue codes) | semantic quality of the content | ~1 second, no model |
+| **B — dynamic behavioral eval** | how the harness *changes agent behavior* on fixed fixtures — verified by deterministic oracles for hard axes + an independent judge for soft axes, with paired sign-test lift vs a baseline | absolute "is this agent good" — only relative to baseline | minutes to hours, real model spend |
+
+All three persist results under `.agentrig/eval/results/` via `score.mjs`. **Never hand-edit** the JSON.
+The schema is validated on read (`schemaVersion: 2`) and on write — invalid records are quarantined
+into `results/_legacy/`.
 
 ---
 
-## Layer A — Static harness audit
-Scored automatically by `checks.json`. Each check maps to a principle and earns **0 / 0.5 / 1.0**.
-The aggregate is the **Harness Score** (0–100%). Run:
+## Layer A1 + A2 — static audit (`agentrig eval --static`)
+
+Scored from `checks.json`. Each check earns **0 / 0.5 / 1.0** and carries a `layer` field
+(`completeness` vs `quality`). Two aggregate scores:
+
+- **Install Completeness** — was every canonical artifact installed where the manifest said it should be?
+- **Quality Probes** — does the content of those artifacts pass cheap sanity checks?
 
 ```bash
-node .agentrig/eval/static-audit.mjs           # or: agentrig eval --static
+node .agentrig/eval/static-audit.mjs            # human report (both layers)
+node .agentrig/eval/static-audit.mjs --json     # machine-readable
+node .agentrig/eval/static-audit.mjs --min 80   # exit non-zero if completeness < 80%
 ```
 
-Treat any principle scoring < 1.0 as a missing/weak artifact to fix.
+A1 is what CI gates on (`--min`). A2 surfaces drift but doesn't fail the build — it's diagnostic.
 
 ---
 
-## Layer B — Dynamic behavioral eval
+## Layer B — dynamic behavioral eval (`agentrig eval --dynamic`)
 
-For each scenario, run the task through the harness, then have an **independent judge model**
-(different from the producer) score the result. Scoring is **strict 3-tier: 0 / 0.5 / 1.0**.
+For each scenario:
 
-Three rules, enforced by `score.mjs` against `axes.json`:
-1. **Issue code required.** Any axis < 1.0 (and observed) must carry an issue code **from that
-   axis's bounded registry** plus a one-line **evidence** string. Invented codes are rejected.
-2. **Confidence-gated.** An axis you couldn't observe is scored `na` (confidence 0) and excluded
-   from rollups — partial observability never contaminates the total.
-3. **Rollups are recomputed from axes.** Category and aggregate scores come from the axis data, not
-   from anything the judge asserts.
+1. **Seed** a throwaway worktree from `scenarios/<id>/fixture/`.
+2. **Producer** (one model, runs in the worktree) executes `prompt.md`. Stage the harness or not, per `--variant`.
+3. **Oracle** (`scenarios/<id>/oracle.yml`) runs deterministic checks (commands, diff stats, file presence) → hard-axis scores.
+4. **Judge** (a *different model family*, runs in its own cwd with prompt+diff+transcript+oracle but **NOT** the producer's worktree or reasoning) scores soft axes against `axes.json`.
+5. **Save** via `score.mjs save` — validated against the rubric registry.
 
-### Multi-rubric lifecycle
-The eval covers the whole lifecycle, not just the final patch. Three rubric **types**, linked by the
-same `--task` id so you get a spec → run → review view:
+### Producer/judge isolation
+- The producer and the judge are **separate `provider.startConversation()` calls**. The judge never sees the producer's reasoning trace.
+- `score.mjs save` rejects a record where the producer and judge share a **model family** (e.g. both `claude-*`). Override with `--allow-same-family` — and the override is recorded in the result so reviewers can spot lazy single-model setups.
+- The judge writes scores via a JSON file (`<artifactsDir>/<scenario>.trial<N>.judge.json`), not free-form text. The orchestrator reads + validates it against `axes.json`.
 
-| `--type` | What it scores | Categories |
-|----------|----------------|------------|
-| `spec`   | task/issue spec quality (before work) | spec_quality (clarity, acceptance_criteria, scope_bounded, testability, context) |
-| `run`    | the implementation run | output_quality, agent_behavior, long_term_impact |
-| `review` | the reviewer's own behavior | review_quality (finding_correctness, severity_calibration, false_positive_rate, coverage, actionability, independence, blocking_decision) |
+### Rubric rules (enforced by `score.mjs`)
+1. **Strict 3-tier** scores: `0` / `0.5` / `1.0`.
+2. **Issue code required.** Any axis < 1.0 with `confidence > 0` must carry an issue code from that axis's bounded registry plus a one-line evidence string.
+3. **Confidence-gated.** An axis you couldn't observe is `=na` (confidence 0) and excluded from rollups.
+4. **Weighted aggregation.** Axes carry an optional `weight` (default 1) and `veto: true`. The aggregate is a weighted mean of observed axes.
+5. **Pass rule:** `aggregate ≥ passThreshold` **AND** no observed axis at 0 **AND** no veto axis < 1.0. Veto fails are surfaced in the `failReason` field.
 
-### `run` axes (the most common)
-- **Output Quality** — `correctness`, `scope`, `tests`, `clarity`
-- **Agent Behavior** — `self_verification`, `gate_compliance`, `tool_discipline`, `escalation`
-- **Long-Term Impact** — `memory`, `regression_risk`, `maintainability`
+### Lifecycle types
+| `--type` | Categories | Veto axes |
+|---|---|---|
+| `spec` | `clarity`, `acceptance_criteria`, `scope_bounded`, `testability`, `context` | `acceptance_criteria` |
+| `run` | `output_quality`, `agent_behavior`, `long_term_impact` (10 axes total) | `correctness`, `gate_compliance` |
+| `review` | `review_quality` (7 axes) | `finding_correctness`, `blocking_decision` |
 
-See `axes.json` for the full per-axis issue-code registries (e.g. `OQ-SCOPE-CHURN`,
-`AB-VERIFY-REDHANDOFF`, `LT-REGRESS-LIKELY`).
+### Multi-trial + statistical lift (`--n` + `compare --baseline`)
+
+Single-trial verdicts are coin flips. The eval requires `n ≥ 3` paired trials for any verdict
+other than `INCONCLUSIVE`:
 
-### Saving and reading scores
 ```bash
-# Save one rubric (any axis < 1.0 needs CODE:evidence; use `=na` for unobserved axes)
-node .agentrig/eval/score.mjs save --type run --task add-small-feature \
-  --scenario add-small-feature --judge <model> [--variant v2] [--run RID] \
-  --axis 'correctness=1.0' \
-  --axis 'scope=0.5:OQ-SCOPE-CHURN:left package-lock churn in the diff' \
-  --axis 'tests=na'
-
-node .agentrig/eval/score.mjs report                     # latest per scenario/variant + per-axis means
-node .agentrig/eval/score.mjs compare --scenario <id>    # A/B variants side by side
+# Run both variants 5 times each.
+agentrig eval --dynamic --variant harness  --n 5
+agentrig eval --dynamic --variant baseline --n 5
+
+# Paired sign test, median delta, p-value:
+node .agentrig/eval/score.mjs compare --scenario <id> --baseline baseline
 ```
 
-### A/B variant evaluation
-Run the **same scenario** under different harness versions (a prompt/skill/rule change) and save each
-under a `--variant`. `score.mjs compare` puts them side by side. **A change that lowers the score is
-a regression even if it "feels" better.** For deeper diffing, keep each run's `diff.patch` /
-`output` artifacts next to the score (see the `harness-eval` skill).
+Verdicts:
+- **HELPS**  — p < 0.05, median delta > 0.05
+- **HURTS**  — p < 0.05, median delta < -0.05
+- **INCONCLUSIVE** — n < 3, or p ≥ 0.05, or |median delta| < 0.05
 
-### Harness lift — does it actually help? (with vs without)
-Prove the harness earns its keep in *your* repo by comparing a harness-on run to a harness-off
-baseline:
+### Sandboxing
+Run dynamic evals under [`sandbox/eval-rules.md`](sandbox/eval-rules.md): the producer works in a
+throwaway worktree under `$TMPDIR/agentrig-eval/<runId>/<scenario>/` and **must not push, open PRs,
+or merge** — the eval measures behavior, it must not mutate real branches.
+
+---
+
+## Calibrating the judge (`calibration/`)
+
+A judge that always returns 1.0 passes every `score.mjs save` validation but tells you nothing.
+The `calibration/` directory holds **hand-labeled** rubric instances (scenario inputs + transcript +
+diff + ground-truth axes). `score.mjs calibrate --judge <model>` runs your judge over them and
+reports % agreement (within ±0.5 tier) and signed bias.
 
 ```bash
-agentrig eval --dynamic --scenario <id> --variant harness    # harness ON
-agentrig eval --dynamic --scenario <id> --variant baseline   # bare agent, no AGENTS.md/rules/skills
-node .agentrig/eval/score.mjs compare --scenario <id> --baseline baseline
+# After your judge wrote scores to /tmp/judge-out.json:
+node .agentrig/eval/score.mjs calibrate \
+  --judge gpt-5.5 --instance .agentrig/eval/calibration/run/seed-correct.yml \
+  --judge-scores /tmp/judge-out.json
+node .agentrig/eval/score.mjs calibrate --report
 ```
 
-`compare --baseline` prints the per-axis and aggregate **delta** and a `HELPS`/`HURTS` verdict. A
-positive aggregate delta means installing AgentRig improved agent behavior here.
-
-### Threshold
-A scenario passes if its aggregate ≥ **0.8** (`passThreshold` in `axes.json`) with no observed axis
-at 0.
+`agentrig doctor` reads the calibration rollup and flags any judge below **80% agreement**. See
+[`calibration/README.md`](calibration/README.md) for the format and how to add more instances.
 
 ---
 
-## Sandboxing
-Run dynamic evals under the guardrails in [`sandbox/eval-rules.md`](./sandbox/eval-rules.md): the
-agent works in a throwaway worktree and must **not push, open PRs, or merge** — the eval measures
-behavior, it must not mutate real branches.
+## When to run what
+
+| When | What |
+|---|---|
+| Every PR | A1 + A2 via `eval --static` (CI gate at `--min 80` or higher) |
+| Nightly on main | Layer B with `--n 5` × `harness` and `baseline`, then `compare --baseline baseline` |
+| Before releasing AgentRig | `score.mjs calibrate --report` ≥ 80% for default judge |
+| When prompts/skills/rules change | Manual `eval --dynamic --variant harness-v2 --n 5` + compare against `harness` |