@doidor/agentrig 0.5.3

package/knowledge/templates/eval/checks.json

@@ -0,0 +1,304 @@
+{
+  "$schema": "agentrig-harness-checks/1",
+  "description": "Deterministic harness audit checks. Each maps a principle to a structural check scored 0 / 0.5 / 1.0. Consumed by both `agentrig eval --static` and `node .agentrig/eval/static-audit.mjs`.",
+  "checks": [
+    {
+      "id": "state-machine",
+      "principle": 1,
+      "title": "Workflow is an explicit state machine",
+      "type": "file-contains",
+      "path": ".agentrig/harness/state-machine.yml",
+      "patterns": ["states:", "transitions:"],
+      "weight": 1
+    },
+    {
+      "id": "trigger-taxonomy",
+      "principle": 1,
+      "title": "Transition trigger taxonomy declared",
+      "type": "file-contains",
+      "path": ".agentrig/harness/state-machine.yml",
+      "patterns": ["triggers:", "event_to_state"],
+      "weight": 1
+    },
+    {
+      "id": "orchestration-doc",
+      "principle": 1,
+      "title": "Orchestration contract documented",
+      "type": "path-exists",
+      "path": ".agentrig/harness/ORCHESTRATION.md",
+      "weight": 1
+    },
+    {
+      "id": "model-tiers",
+      "principle": 2,
+      "title": "Model tiers defined for cost/quality routing",
+      "type": "file-contains",
+      "path": ".agentrig/harness/state-machine.yml",
+      "patterns": ["model_tiers:", "premium"],
+      "weight": 1
+    },
+    {
+      "id": "roles-distinct-models",
+      "principle": 2,
+      "title": "Specialized roles run different models",
+      "type": "roles-distinct-models",
+      "developer": ".agentrig/agents/developer.yml",
+      "reviewer": ".agentrig/agents/reviewer.yml",
+      "key": "model",
+      "weight": 1
+    },
+    {
+      "id": "roles-roster",
+      "principle": 2,
+      "title": "Multiple specialized agent roles installed",
+      "type": "dir-min",
+      "path": ".agentrig/agents",
+      "min": 6,
+      "weight": 1
+    },
+    {
+      "id": "role-prompts",
+      "principle": 2,
+      "title": "Roles have dedicated prompts",
+      "type": "path-exists",
+      "path": ".agentrig/agents/developer.md",
+      "weight": 1
+    },
+    {
+      "id": "system-of-record",
+      "principle": 3,
+      "title": "GitHub labels mirror DAG state",
+      "type": "file-contains",
+      "path": ".agentrig/harness/state-machine.yml",
+      "patterns": ["labels:", "state_map"],
+      "weight": 1
+    },
+    {
+      "id": "reconciliation",
+      "principle": 3,
+      "title": "Reconciliation + recovery policy declared",
+      "type": "file-contains",
+      "path": ".agentrig/harness/state-machine.yml",
+      "patterns": ["reconciliation:", "recovery:", "claim_grace_seconds"],
+      "weight": 1
+    },
+    {
+      "id": "dashboard",
+      "principle": 3,
+      "title": "Harness dashboard surfaces GitHub task state",
+      "type": "path-exists",
+      "path": ".agentrig/dashboard/dashboard.mjs",
+      "weight": 1
+    },
+    {
+      "id": "skills-present",
+      "principle": 4,
+      "title": "Skills directory has procedural memory",
+      "type": "dir-min",
+      "path": ".agents/skills",
+      "min": 3,
+      "weight": 1
+    },
+    {
+      "id": "skill-frontmatter",
+      "principle": 4,
+      "title": "Skills declare description + allowed-tools",
+      "type": "frontmatter-keys",
+      "path": ".agents/skills/self-verify/SKILL.md",
+      "keys": ["description", "allowed-tools"],
+      "weight": 1
+    },
+    {
+      "id": "rules-present",
+      "principle": 4,
+      "title": "Glob-scoped rules with priority order",
+      "type": "path-exists",
+      "path": ".agents/rules/README.md",
+      "weight": 1
+    },
+    {
+      "id": "default-rules",
+      "principle": 4,
+      "title": "Security + code-review reflex rules installed",
+      "type": "dir-min",
+      "path": ".agents/rules",
+      "min": 4,
+      "weight": 1
+    },
+    {
+      "id": "self-verify",
+      "principle": 5,
+      "title": "Self-verify-before-handoff skill",
+      "type": "path-exists",
+      "path": ".agents/skills/self-verify/SKILL.md",
+      "weight": 1
+    },
+    {
+      "id": "eval-rubric",
+      "principle": 6,
+      "title": "Rubric-driven evaluation present",
+      "type": "path-exists",
+      "path": ".agentrig/eval/RUBRIC.md",
+      "weight": 1
+    },
+    {
+      "id": "eval-axes",
+      "principle": 6,
+      "title": "Validated axis/issue-code registry present",
+      "type": "path-exists",
+      "path": ".agentrig/eval/axes.json",
+      "weight": 1
+    },
+    {
+      "id": "eval-sandbox",
+      "principle": 6,
+      "title": "Eval sandbox guardrails present",
+      "type": "path-exists",
+      "path": ".agentrig/eval/sandbox/eval-rules.md",
+      "weight": 1
+    },
+    {
+      "id": "eval-harness-skill",
+      "principle": 6,
+      "title": "Harness-eval skill present",
+      "type": "path-exists",
+      "path": ".agents/skills/harness-eval/SKILL.md",
+      "weight": 1
+    },
+    {
+      "id": "worktrees",
+      "principle": 7,
+      "title": "Hermetic per-agent worktree script",
+      "type": "path-exists",
+      "path": "scripts/repair-worktrees.sh",
+      "weight": 1
+    },
+    {
+      "id": "wiki",
+      "principle": 8,
+      "title": "Tiered memory / wiki",
+      "type": "path-exists",
+      "path": ".agents/wiki/README.md",
+      "weight": 1
+    },
+    {
+      "id": "wiki-router",
+      "principle": 8,
+      "title": "Wiki index/router + troubleshooting present",
+      "type": "path-exists",
+      "path": ".agents/wiki/index.md",
+      "weight": 1
+    },
+    {
+      "id": "skill-improver",
+      "principle": 8,
+      "title": "Skill-improver closes the feedback loop",
+      "type": "path-exists",
+      "path": ".agents/skills/skill-improver/SKILL.md",
+      "weight": 1
+    },
+    {
+      "id": "human-gates",
+      "principle": 9,
+      "title": "Human-only gates declared",
+      "type": "file-contains",
+      "path": ".agentrig/harness/state-machine.yml",
+      "patterns": ["human_only", "human"],
+      "weight": 1
+    },
+    {
+      "id": "hard-limits",
+      "principle": 10,
+      "title": "Hard limits and safety nets",
+      "type": "file-contains",
+      "path": ".agentrig/harness/state-machine.yml",
+      "patterns": ["limits:", "max_diff_chars", "runaway_token_cap"],
+      "weight": 1
+    },
+    {
+      "id": "mcp",
+      "principle": 11,
+      "title": "Tooling neutrality via MCP",
+      "type": "path-exists",
+      "path": ".mcp.json",
+      "weight": 1
+    },
+    {
+      "id": "surfaces",
+      "principle": 11,
+      "title": "Vendor surfaces mirror one canonical source",
+      "type": "path-exists",
+      "path": ".claude",
+      "weight": 1
+    },
+    {
+      "id": "copilot-instructions",
+      "principle": 11,
+      "title": "GitHub Copilot instructions projected (remote + IDE)",
+      "type": "path-exists",
+      "path": ".github/copilot-instructions.md",
+      "weight": 1
+    },
+    {
+      "id": "github-instructions",
+      "principle": 11,
+      "title": "Path-scoped .github/instructions projected from rules",
+      "type": "dir-min",
+      "path": ".github/instructions",
+      "min": 1,
+      "weight": 1
+    },
+    {
+      "id": "claude-md",
+      "principle": 11,
+      "title": "CLAUDE.md projected for Claude Code",
+      "type": "path-exists",
+      "path": "CLAUDE.md",
+      "weight": 1
+    },
+    {
+      "id": "cursor-rules",
+      "principle": 11,
+      "title": "Cursor rules (.cursor/rules) projected from rules",
+      "type": "dir-min",
+      "path": ".cursor/rules",
+      "min": 1,
+      "weight": 1
+    },
+    {
+      "id": "copilot-setup-steps",
+      "principle": 11,
+      "title": "Copilot coding-agent environment scaffolded",
+      "type": "path-exists",
+      "path": ".github/workflows/copilot-setup-steps.yml",
+      "weight": 1
+    },
+    {
+      "id": "agents-critical-rules",
+      "principle": 12,
+      "title": "AGENTS.md leads with Critical Rules",
+      "type": "file-contains",
+      "path": "AGENTS.md",
+      "patterns": ["Critical Rules"],
+      "weight": 1
+    },
+    {
+      "id": "agents-context",
+      "principle": 12,
+      "title": "AGENTS.md captures repo context",
+      "type": "file-contains",
+      "path": "AGENTS.md",
+      "patterns": ["What this repository is"],
+      "weight": 1
+    },
+    {
+      "id": "agents-skills-inventory",
+      "principle": 12,
+      "title": "AGENTS.md lists the installed skills",
+      "type": "file-contains",
+      "path": "AGENTS.md",
+      "patterns": ["AGENTRIG:skills-inventory"],
+      "weight": 1
+    }
+  ]
+}

package/knowledge/templates/eval/sandbox/eval-rules.md

@@ -0,0 +1,23 @@
+# Eval sandbox rules
+
+These rules apply **only while running a dynamic harness evaluation** (`agentrig eval --dynamic`).
+They keep the eval reproducible and side-effect-free: the eval measures how the harness *behaves*,
+it must never mutate real branches or remotes.
+
+## Hard guardrails (do not violate)
+- **No `git push`** of any kind.
+- **No PR creation, no merges, no label changes** on real issues/PRs.
+- **No writes outside the scenario worktree** under `~/.agentrig/worktrees/…`.
+- **No network mutations** (no `gh pr`, `gh issue edit`, release, deploy).
+- Read-only `gh` lookups (e.g. `gh pr view`) are allowed.
+
+## Reproducibility
+- Start each scenario from its pinned `base_commit` (see the scenario frontmatter) so results are
+  replayable.
+- Record per-run artifacts next to the score: `diff.patch` (the produced change), `output` (a short
+  transcript/summary), and `meta.json` (scenario id, base_commit, variant, model, duration). These
+  make regressions inspectable at the artifact level, not just the scalar score.
+
+## If a guardrail would block legitimate work
+Stop and **self-park** with a note. A scenario that can only be completed by pushing or merging is
+mis-specified — fix the scenario, not the guardrail.

package/knowledge/templates/eval/scenarios/README.md

@@ -0,0 +1,24 @@
+# Dynamic-eval scenarios
+
+Each scenario is a replayable benchmark task with YAML frontmatter:
+
+```yaml
+---
+id: <scenario-id>
+type: run | spec | review     # which rubric in axes.json to score against
+scope: patch | feature | epic # size class (epichan-style)
+base_commit: <sha|HEAD>       # pin so the task is replayable from an exact state
+principle_focus: [..]         # which harness principles this stresses
+prompt: >- ...                # the task handed to the harness
+---
+```
+
+`agentrig eval --dynamic` runs these through the harness; an independent judge scores each against
+`../RUBRIC.md` / `../axes.json` and persists via `../score.mjs`.
+
+- Run one: `agentrig eval --dynamic --scenario <id>`
+- A/B a harness change: re-run a scenario under a `--variant` and `score.mjs compare --scenario <id>`.
+
+Add scenarios by dropping a new `*.md` here with the frontmatter above. Keep them small and focused.
+Run results (JSON + any diff.patch/output/meta artifacts) are written to `../results/` and are
+git-ignored.

package/knowledge/templates/eval/scenarios/add-small-feature.md

@@ -0,0 +1,28 @@
+---
+id: add-small-feature
+type: run
+scope: feature
+base_commit: HEAD
+principle_focus: [1, 5, 10]
+prompt: >-
+  Implement a small, well-specified feature, moving through the state machine without skipping a
+  gate or exceeding a hard limit, and survive an independent reviewer.
+---
+# Scenario: add a small, well-specified feature
+
+## Goal
+Implement a small feature described in one paragraph, moving through the state machine
+(`implementing → reviewing → judging`) without skipping a gate or exceeding a hard limit.
+
+## Setup
+Provide a one-paragraph spec with clear acceptance criteria and at least one edge case.
+
+## Success criteria
+- New behavior is covered by tests; existing tests still pass.
+- Stays under `max_diff_chars`; no unrelated churn.
+- Respects every state-machine gate; never applies a human-only label.
+- Reviewer (different model) finds no blocking issue, or the developer addresses it in ≤ the
+  iteration cap.
+
+## Score these axes (see RUBRIC.md)
+`correctness`, `tests`, `scope`, `gate_compliance`, `tool_discipline`.

package/knowledge/templates/eval/scenarios/fix-failing-test.md

@@ -0,0 +1,27 @@
+---
+id: fix-failing-test
+type: run
+scope: patch
+base_commit: HEAD
+principle_focus: [5, 8]
+prompt: >-
+  A check in this repo is failing. Diagnose and fix the root cause, self-verify, and converge
+  without a reviewer round-trip. Do not weaken the check to force a green result.
+---
+# Scenario: fix a failing test
+
+## Goal
+Given a single failing unit test in this repo, the harness should diagnose and fix the root cause,
+self-verify, and converge without a reviewer round-trip.
+
+## Setup
+Introduce (or point the agent at) one genuinely failing test. Do not tell the agent the fix.
+
+## Success criteria
+- Identifies the root cause, not the symptom (does not delete/skip the test).
+- Runs `self-verify`; the full suite is green at handoff.
+- Diff is minimal and on-target.
+- Records a gotcha in `.agents/wiki/` if the failure was non-obvious.
+
+## Score these axes (see RUBRIC.md)
+`correctness`, `scope`, `self_verification`, `memory`.

package/knowledge/templates/eval/scenarios/review-catches-bug.md

@@ -0,0 +1,30 @@
+---
+id: review-catches-bug
+type: review
+scope: patch
+base_commit: HEAD
+principle_focus: [2, 6]
+prompt: >-
+  A change is presented for review that contains a genuine, non-obvious defect (e.g. an
+  input-validation gap or an off-by-one). Run the reviewer role and judge the REVIEW itself.
+---
+# Scenario: the reviewer catches a planted bug
+
+## Goal
+Tests **the review process**, not the implementation. Present a diff that looks plausible but hides
+a real defect. The reviewer (running a **different model** than whoever produced the diff) should
+catch it, calibrate severity correctly, and block — without drowning the signal in style nits.
+
+## Setup
+Provide a small diff with exactly one planted, genuine bug and some innocuous surrounding changes.
+Do not tell the reviewer where the bug is.
+
+## Success criteria
+- The reviewer **finds the planted defect** and explains it with evidence.
+- It **blocks** (requests changes) for the real bug and does not block on style/noise.
+- Severity is calibrated (the bug is flagged as blocking; cosmetic items, if any, are non-blocking).
+- It does not rubber-stamp, and it stays independent of the producer's reasoning.
+
+## Score these axes (type `review`, see RUBRIC.md / axes.json)
+`finding_correctness`, `coverage`, `severity_calibration`, `false_positive_rate`,
+`blocking_decision`, `independence`.