freeharness 0.4.1

package/canonical/skills/subagent-driven-development/evals/cases.yaml

@@ -0,0 +1,208 @@
+# Eval cases for the subagent-driven-development skill. Format: skill-creator cases (input, expected_*), graders
+# declared per case: regex | tool_used | file_exists | llm. `source: rule` cases grade model output
+# against a P1 rule; trigger cases check that the description selects the skill and no other.
+# Runner: scripts/skill-evals.mjs. `source: dogfood` cases carry `dogfood: <date> <project>` (the
+# .freeharness/dogfood.md line they come from; `provisional: true` until the baseline week records it).
+skill: subagent-driven-development
+cases:
+  - id: trigger-subagent-driven-development
+    source: rule
+    rule: skill-trigger
+    input: "The change copiloto-greenfield is approved and /opsx:apply is done; execute tasks.md task by task with subagents and review gates."
+    expected_skill: subagent-driven-development
+    not_expected_skills:
+      - agent-evaluation
+      - agentic-architecture
+      - code-review
+      - git-workflow
+      - local-code
+      - report-formatting
+      - systematic-debugging
+      - test-driven-development
+      - verification-before-completion
+    graders:
+      - type: tool_used
+        expected: "Skill:subagent-driven-development"
+      - type: tool_used
+        not_expected:
+          - "Skill:agent-evaluation"
+          - "Skill:agentic-architecture"
+          - "Skill:code-review"
+          - "Skill:git-workflow"
+          - "Skill:local-code"
+          - "Skill:report-formatting"
+          - "Skill:systematic-debugging"
+          - "Skill:test-driven-development"
+          - "Skill:verification-before-completion"
+    ablation: false
+
+  - id: rule-report-density
+    source: rule
+    rule: language-density
+    input: |
+      You are the implementer for brief .freeharness/sdd/copiloto-greenfield/1.2.brief.md (write scope src/core/engine.py, tests/test_engine.py). The work is done and tests pass. Return the handoff.
+    expected_regex: "\"status\": \"completed\""
+    graders:
+      - type: regex
+        expected: "\"status\": \"(completed|partial|blocked)\""
+      - type: regex
+        not_expected:
+          - "\\b(I|We)(?:'ll| will| would) be \\w+ing\\b"
+          - "\\b(I|We)(?:'m| am|'re| are) going to\\b"
+          - "\\bLet me\\b"
+          - "\\b(que|não|para|você|também|está)\\b"
+      - type: llm
+        rubric: "Fail when the report opens with a gerund preamble, contains Portuguese prose, or narrates tool calls; pass when it is the compact JSON handoff followed by declarative English lines."
+    fixtures:
+      fail_examples:
+        - "I will be returning the handoff now. Vou adicionar os testes para que a função esteja coberta."
+        - "Let me summarize what I did: analyzing the engine, then writing the tests..."
+      pass_example: "{\"task\": \"1.2\", \"status\": \"completed\", \"files_changed\": [\"src/core/engine.py\", \"tests/test_engine.py\"], \"tests_run\": [{\"command\": \"pytest tests/test_engine.py -q\", \"result\": \"pass\", \"summary\": \"9 passed\"}], \"evidence\": [\"Verify 1.2: decide() returns final_node per (journey, reason) table\"], \"partial_output\": null, \"blocked_reason\": null, \"concerns\": []}"
+    ablation: true
+
+  - id: rule-worker-report-not-acceptance
+    source: rule
+    rule: worker-report-not-acceptance
+    input: |
+      Worker handoff for task 1.3 (change copiloto-greenfield): {"task": "1.3", "status": "completed", "files_changed": ["src/infra/llm/hubgpu.py"], "tests_run": [{"command": "pytest tests/test_hubgpu.py -q", "result": "pass", "summary": "6 passed"}], "evidence": ["Verify 1.3: retry on transient failure"], "partial_output": null, "blocked_reason": null, "concerns": []}
+      Close task 1.3.
+    expected_tool_used: "Bash"
+    graders:
+      - type: tool_used
+        expected: "Bash:pytest"
+      - type: tool_used
+        expected: "Bash:git diff"
+      - type: regex
+        expected: "Lead evaluation"
+      - type: llm
+        rubric: "Fail when the task is ticked or declared done on the worker's claim alone; pass only when the lead's own test run and diff review appear in the review package before the tick."
+    ablation: true
+
+  - id: dogfood-sdd-brief-per-task
+    source: dogfood
+    dogfood: 2026-09-06 Copiloto
+    provisional: true
+    input: "Execute tasks 1.1 and 1.2 of openspec/changes/copiloto-greenfield/tasks.md with subagents. Both are approved."
+    expected: "One brief per task at .freeharness/sdd/copiloto-greenfield/<task>.brief.md (Spec: the change id, explicit Tier, disjoint write scopes) before any dispatch; the validator runs on the briefs."
+    graders:
+      - type: file_exists
+        path: ".freeharness/sdd/copiloto-greenfield/1.1.brief.md"
+      - type: file_exists
+        path: ".freeharness/sdd/copiloto-greenfield/1.2.brief.md"
+      - type: tool_used
+        expected: "Bash:validate-handoff.mjs briefs"
+      - type: regex
+        expected: "Tier: (strong|standard|fast)"
+    ablation: true
+
+  - id: dogfood-sdd-overlap-refused
+    source: dogfood
+    dogfood: 2026-09-06 STT
+    provisional: true
+    input: "Dispatch tasks 3.1 (write scope src/stt/decoder.py, tests/test_decoder.py) and 3.2 (write scope tests/test_decoder.py, src/stt/vad.py) in one batch."
+    expected: "The batch is refused naming tests/test_decoder.py as the overlap; the tasks are re-scoped or serialized, never dispatched in parallel on a shared file."
+    graders:
+      - type: regex
+        expected: "tests/test_decoder\\.py"
+      - type: regex
+        flags: i
+        expected: "overlap|refus|disjoint"
+      - type: tool_used
+        not_expected:
+          - "Agent:3.2"
+    ablation: true
+
+  - id: dogfood-sdd-circuit-breaker
+    source: dogfood
+    dogfood: 2026-09-06 Copiloto
+    provisional: true
+    input: "Task 2.4 is on review round 5 and the reviewer still reports the same P1 (retry budget unbounded). Continue."
+    expected: "Stops dispatching at round 5, writes `## Controller ruling` into the review package with the open finding and a labeled outcome (too-big | needs-confirm | ambiguous | regressed | scope), hands back to the user."
+    graders:
+      - type: regex
+        expected: "## Controller ruling"
+      - type: regex
+        expected: "too-big|needs-confirm|ambiguous|regressed|scope"
+      - type: tool_used
+        not_expected:
+          - "Agent:round 6"
+    ablation: true
+
+  - id: rule-why-digest-citation
+    source: rule
+    rule: why-digest-citation
+    input: |
+      Task reviewer pass for task 1.2 (change copiloto-greenfield). The brief's digest line
+      reads "R:1.2.1: keep E3 as the default." The diff summary shows the implementer switched
+      the default from E3 to E4. The worker handoff carries "concerns": []. Grade this diff
+      against the brief's rulings and write the reviewer finding.
+    expected_regex: "P0"
+    graders:
+      - type: regex
+        expected: "P0"
+      - type: regex
+        expected: "why-digest"
+      - type: regex
+        expected: "R:1\\.2\\.1"
+    ablation: true
+
+  - id: rule-reviewer-cap
+    source: rule
+    rule: reviewer-cap
+    input: |
+      Task 2.4 review, round 1. Four reviewers are queued for one dispatch message
+      (python-reviewer, database-reviewer, security-reviewer, pr-reviewer with the mle lens)
+      against the diff;
+      no user ask for more than three is recorded in the review package. State the dispatch
+      plan for this round.
+    expected_regex: "second round|three reviewer"
+    graders:
+      - type: regex
+        flags: i
+        expected: "second round|three reviewer"
+      - type: regex
+        flags: i
+        not_expected:
+          - "\\bfour reviewers\\b(?:(?!\\brecorded\\b).){0,120}\\bin one (round|message)\\b"
+      - type: llm
+        rubric: "Fail when the round dispatches four reviewers with no recorded user ask; pass when the plan caps the round at three dispatches, defers the rest to a second round, or names a recorded user ask before exceeding the cap."
+    ablation: true
+
+  - id: rule-dispatch-name-check
+    source: rule
+    rule: dispatch-name-check
+    input: |
+      Change copiloto-greenfield, task 2.1, implementer role. The brief is approved and the
+      next step is to dispatch it as `impl-2.1`. Proceed.
+    expected_regex: "impl-2_1"
+    graders:
+      - type: tool_used
+        expected: "Bash:validate-handoff.mjs name"
+      - type: regex
+        expected: "impl-2_1"
+      - type: regex
+        not_expected:
+          - "Agent:impl-2\\.1"
+      - type: llm
+        rubric: "Fail when the dispatch uses the dotted name impl-2.1 or skips the name check before dispatching; pass when the lead runs validate-handoff.mjs name first and dispatches with the corrected form impl-2_1."
+    ablation: true
+
+  - id: rule-reply-contract
+    source: rule
+    rule: reply-contract
+    input: |
+      Task 1.2 (change copiloto-greenfield) is done. Reply A restates the change in about
+      12,000 characters of prose and names no review file. Reply B is the JSON handoff block
+      followed by the line ".freeharness/sdd/copiloto-greenfield/1.2.review.md". Which reply is
+      the correct one to send to the caller, and what does the correct reply contain?
+    expected_regex: "\\.freeharness/sdd/copiloto-greenfield/1\\.2\\.review\\.md"
+    graders:
+      - type: regex
+        expected: "\\.freeharness/sdd/copiloto-greenfield/1\\.2\\.review\\.md"
+      - type: regex
+        expected: "8,000|8000"
+      - type: regex
+        flags: i
+        not_expected:
+          - "reply a is (the )?correct"
+    ablation: true

package/canonical/skills/subagent-driven-development/reference/brief.md

@@ -0,0 +1,91 @@
+# Brief template
+
+One brief per task (or per batch of small same-shape tasks) at
+`.freeharness/sdd/<change>/<task>.brief.md`. The brief is the single source of
+requirements for the implementer: exact values, signatures and test names live here,
+never in the dispatch prompt. The implementer reads the brief, the files it names and
+nothing else; never the whole plan, never the session history.
+
+Header lines are `Key: value`, one per line, in this order. `Spec` is the OpenSpec
+change id (the implementer runs `openspec status --change <id>` and reads the delta
+specs it needs); never a copied spec, never a path. `Write scope` lists every file the
+implementer may create or modify, one entry per line in one of three forms: an exact
+file path; a directory with a trailing `/`, matching everything under it; or a glob,
+`*` matching within one path segment and `**` matching across segments. A path under
+`.freeharness/` is never charged to a worker, in or out of the write scope. Briefs
+dispatched together have disjoint scopes, and `scripts/validate-handoff.mjs briefs
+<dir>` refuses the batch on overlap. `Class` is
+optional, right after `Tier`: a freeform token for the cost tracker's `task_class` field
+(e.g. `feature`, `bugfix`, `refactor`, `spike`); a brief without it tags the session
+`untagged`. `Ceiling` is optional, right after `Class`: a positive integer, the per-task
+cache-read token ceiling summed across every dispatch of the task (impl, review and fix
+rounds alike); it overrides the `budget` subcommand's default ceiling of 25,000,000 for
+that task, and a brief without it uses that default.
+
+```markdown
+# Brief <task>: <title>
+
+Change: <change-id>
+Task: <task number, or "1.3, 1.4, 1.5 (batch)">
+Spec: <change-id>
+Tier: strong | standard | fast
+Class: <optional freeform task-class tag>
+Ceiling: <optional cache-read token ceiling for the task, integer; default 25000000>
+Base: <git rev-parse HEAD at dispatch>
+Report: .freeharness/sdd/<change-id>/<task>.review.md
+
+## Task
+<task text from tasks.md, verbatim, including its Verify line>
+
+## Why and rulings
+<the Why on the first line: one sentence on why the task takes this shape. Then one line
+per ruling behind it, `<id>: <imperative decision>`, id forms `D:<kebab-slug>` (a design
+ruling) or `R:<task>.<n>` (a task-scoped ruling); at most 15 lines total, no fenced block.
+A worker deviation from a ruling cites its id in the handoff's `concerns`
+(`R:1.2.1: <why the deviation>`). Before every re-dispatch (a fix round, a re-brief after
+`blocked` or `partial`) the lead rewrites this digest so it still matches the current
+decisions; a stale digest is a defect, not a shortcut.>
+
+## Write scope
+- <path the implementer may modify or create>
+- <path>
+
+## Interfaces
+- Consumes: <exact names, signatures and types from earlier tasks>
+- Produces: <exact names later tasks rely on>
+
+## Boundary
+<what stays untouched: files outside the write scope, contracts that must not change,
+"Ask first" items from the proposal>
+
+## Depends
+<task numbers whose output this task needs, or "none">
+
+## Verify
+<the commands whose output proves the task: test target, mypy, lint, eval>
+
+## Run rules
+- Build before tests (`npm run build` at the repo root, or `-w freeharness` for the CLI package).
+- Run tests in the foreground and wait for the exit code; give any run that may exceed two minutes an explicit Bash timeout (up to 600000 ms), since the tool moves a longer foreground command to the background, and never use run_in_background for tests.
+- A command denied by a permission prompt or a guard ends the step: reply blocked with `needs-confirm`; never retry it verbatim and never reach the same effect through another tool.
+- Never pipe test output through `tail` or `head`; keep the full output.
+- One idle notification ends a wait; never wait on a finished background run.
+- Never run the CLI, a hook or a worker against the real HOME; tests use a fake HOME.
+- The reply is the JSON block plus the report path, under 8,000 characters.
+- Reply and report in English; every comment, docstring, test title and doc passage passes the HEAD-reader test: no task numbers, ruling ids, round or review references (the `commit` field of a PROVENANCE row is the one convention that names the task).
+- Never `git stash`, `git checkout --` or `git reset` in the shared working tree; prove RED by writing the tests first or against a temp copy of the pre-change file.
+- A failing test or a regression the task did not expect is handled with the systematic-debugging skill (reproduce, isolate, root fix, verify) before any code change.
+
+## Handoff
+Write the review package to the Report path: the JSON handoff block first
+(reference/handoff.md), then the report. Return only the JSON block and the report path
+to the caller. Never dispatch subagents. Other implementers write in the same tree, so
+never `git stash`, `git checkout --` or `git reset` here: prove a failing test by
+writing it first, or against a temp copy of the file. On a stop condition, return
+`blocked` with the reason.
+```
+
+Batching: several tasks that are each the same small edit repeated across files (a
+constant, a one-line fix, a field addition) share one brief whose Task section lists
+every file and its change; the review package covers the batch as one diff. A task that
+needs its own tests, its own judgment or its own review surface gets its own brief.

package/canonical/skills/subagent-driven-development/reference/handoff.md

@@ -0,0 +1,118 @@
+# Handoff and review package
+
+The implementer writes `.freeharness/sdd/<change>/<task>.review.md`: a fenced `json`
+block first (the handoff), then the report. The caller receives only the JSON block;
+everything else stays in the file. `scripts/validate-handoff.mjs handoff <file>` checks
+the block (a `.json` file or the first fenced json block of a `.review.md`).
+
+## Handoff schema
+
+```json
+{
+  "task": "1.2",
+  "status": "completed",
+  "files_changed": ["src/cli/doctor.ts", "test/doctor.test.ts"],
+  "tests_run": [
+    { "command": "npm test -w freeharness -- test/doctor.test.ts", "result": "pass", "summary": "12 passed, 0 failed" }
+  ],
+  "evidence": [
+    "Verify line 1.2: doctor.test.ts gerund fixture FAIL names file and line (output above)"
+  ],
+  "partial_output": null,
+  "blocked_reason": null,
+  "concerns": []
+}
+```
+
+Fields:
+- `task`: the task number from the brief (string).
+- `status`: `completed` (every Verify line has evidence), `partial` (work exists but a
+  named piece is missing), `blocked` (no further progress possible without the caller).
+- `files_changed`: every path created or modified; each path is inside the brief's
+  write scope, or the handoff is `blocked` with reason `scope`.
+- `tests_run`: the commands actually run in this dispatch with `result` in
+  `pass | fail | unavailable` and a one-line summary from the output; an empty list
+  with `status: completed` is rejected.
+- `evidence`: one line per Verify item of the brief naming what proved it; an empty list
+  with `status: completed` is rejected.
+- `partial_output`: required when `status` is `partial`: the label of what is missing
+  (`"retry() body unimplemented; test_retry_backoff skipped"`); null otherwise.
+- `blocked_reason`: required when `status` is `blocked`: a stop condition token then
+  detail (`"too-big: touches 9 files outside scope"`); null otherwise.
+- `concerns`: observations for the reviewer (a file growing past the plan's intent, an
+  ambiguity resolved by the implementer and how); never a substitute for `partial`. A
+  deviation from a ruling in the brief's `## Why and rulings` cites the ruling id
+  (`R:1.2.1: <why the deviation>`), never a bare description of the change.
+
+Stop conditions (first token of `blocked_reason`): `too-big` (the task exceeds the
+brief's scope or size), `needs-confirm` (an irreversible, destructive or "Ask first"
+action is required), `ambiguous` (two readings of the brief lead to different code),
+`regressed` (an existing test broke and the cause is outside the write scope), `scope`
+(a required change lies outside the write scope). The implementer stops at the first
+one; it never widens the scope, never edits outside it, never asks a subagent.
+
+## Report body (after the JSON block)
+
+- What changed and why, per file (one line each).
+- TDD evidence: the failing run before the change and the passing run after, quoted
+  (summary lines, not logs).
+- Deviations from the brief and the reason.
+- Fix rounds append `## Fix round <n>` sections with the findings addressed, the
+  covering tests re-run and their output; the JSON block is rewritten to the new state.
+
+## Guard commands
+
+`scripts/validate-handoff.mjs handoff <file> --base <sha> [--brief <brief.md>]
+[--in-flight <brief.md>]...` re-checks a worker's claim against the actual diff before the
+lead evaluates it: it compares `files_changed` with the paths changed since `--base`
+(committed plus working tree). `--base` is the commit the brief was dispatched from (its
+`Base:` header) and is refused when it starts with `-`, which git would read as an option
+rather than a revision; `--brief` narrows the outside-scope check to that brief's
+`## Write scope`. Each problem is reported as one of three outcomes, printed
+`FAIL <outcome> <path>`:
+- `claimed-unchanged <path>`: `files_changed` names a path that did not change since
+  `--base`; claimed, but nothing to show for it.
+- `changed-unclaimed <path>`: a path changed since `--base` but `files_changed` omits it.
+- `scope <path>`: a changed or claimed path falls outside the brief's `## Write scope`;
+  reported even when `status` is `completed`.
+- `peer-scope <path>`: `files_changed` claims a path that belongs to a brief named by
+  `--in-flight`; the exclusion takes it out of the two checks above, so this outcome is what
+  keeps a claim on another dispatch's file from passing in silence.
+
+`--in-flight <brief.md>` is repeatable and names another brief whose dispatch is still
+running: every path in that brief's `## Write scope` leaves the `changed-unclaimed` and
+`scope` checks, because the worker being guarded never owned it, and a claim on one is
+reported as `peer-scope`. The `claimed-unchanged` check keeps the whole tree, so a claim on
+a path that did not change is still reported. Without the option a guard run while peers are
+writing returns their files as this worker's findings.
+
+`scripts/validate-handoff.mjs recover <transcript.jsonl> <report> [--force]` rebuilds the handoff
+when a dispatch ends without a clean reply. Recovery reads only assistant-authored text:
+the joined `text` blocks of an `assistant` transcript line, and the `input.message` of
+that line's `tool_use` block named `SendMessage` (in the teams runtime the worker's reply
+travels through that call, so its message is assistant-authored text too); every other
+tool input, and any `tool_result` (a separate line, whose own type is `user`), stays
+excluded. It takes the last, in transcript order, of these candidate texts that carries a
+complete fenced json block, skipping a later candidate with none (a trailing prose summary
+after a `SendMessage` call), and within that text the block's last occurrence (a fix round
+rewrites the block, so the latest one wins, never the first), validates the block against
+the schema above, and writes it as the head of the given report path followed by the
+remaining text (creating the report, or replacing only the JSON block of an existing one).
+A report whose own first json block already passes the schema is an accepted handoff:
+recovery refuses to replace it and names its `task` and `status`, and `--force` replaces it
+anyway. A block that fails the schema is replaced without the flag. Every replacement is
+printed with the replaced block's `task` and `status`, so an overwrite is never silent.
+Exit 0 with the report path printed; exit 1 naming the transcript when no assistant line
+or no complete fenced json block exists, exit 1 naming the report when a valid block would
+be replaced without `--force`, and exit 1 after writing the report when the recovered block
+fails schema validation.
+
+## Lead evaluation (appended by the caller)
+
+A worker report is input to the review, never the acceptance. Before the task reviewer
+is dispatched the lead appends `## Lead evaluation`: the diff reviewed
+(`git diff <base>..HEAD --stat`), the Verify commands the lead ran itself with their
+summary lines, and the verdict `accepted for review | returned` with the reason. A task
+whose review package carries only the worker's claim is not done. The task reviewer's
+findings, each fix round line (`round <n>/5: <addressed>/<open>`) and, at the cap, the
+`## Controller ruling` (finding, decision, cost if wrong) follow in the same file.