claude-dev-env 1.38.1 → 1.39.0

package/scripts/test_groq_bugteam_spec.py

@@ -1,338 +0,0 @@
-"""Coherence tests for groq_bugteam_spec module import surface.
-
-The behavioral contract for apply_fix_from_spec lives in
-test_groq_bugteam_apply_fix_from_spec.py; those tests pass whether the
-function is defined in groq_bugteam.py directly or re-exported from the
-spec module. This file exists solely so the spec module has a
-same-named test companion for filename-based test pairing.
-"""
-
-from __future__ import annotations
-
-import importlib.util
-import io
-import json
-import pathlib
-import sys
-import types
-
-import pytest
-
-
-def _load_spec_module():
-    scripts_directory = pathlib.Path(__file__).parent
-    sys.path.insert(0, str(scripts_directory))
-    sys.modules.pop("groq_bugteam_spec", None)
-    module_path = scripts_directory / "groq_bugteam_spec.py"
-    module_spec = importlib.util.spec_from_file_location(
-        "groq_bugteam_spec", module_path
-    )
-    loaded_module = importlib.util.module_from_spec(module_spec)
-    sys.modules["groq_bugteam_spec"] = loaded_module
-    module_spec.loader.exec_module(loaded_module)
-    return loaded_module
-
-
-groq_bugteam_spec = _load_spec_module()
-
-
-def test_is_spec_mode_invocation_detects_flag_value_pair():
-    assert groq_bugteam_spec.is_spec_mode_invocation(["--mode", "spec"]) is True
-    assert groq_bugteam_spec.is_spec_mode_invocation(["--mode", "pipeline"]) is False
-    assert groq_bugteam_spec.is_spec_mode_invocation([]) is False
-
-
-def _attach_required_groq_attributes(target_module: types.ModuleType) -> None:
-    target_module.call_groq_with_fallback = lambda *args, **kwargs: None
-    target_module.parse_json_object = lambda text: {}
-    target_module.preserve_trailing_newline = lambda original, updated: updated
-
-
-def test_resolver_prefers_registered_groq_bugteam_over_main(monkeypatch):
-    fake_groq_bugteam = types.ModuleType("groq_bugteam")
-    _attach_required_groq_attributes(fake_groq_bugteam)
-    monkeypatch.setitem(sys.modules, "groq_bugteam", fake_groq_bugteam)
-
-    resolved_module = groq_bugteam_spec.resolve_groq_bugteam_module()
-
-    assert resolved_module is fake_groq_bugteam
-
-
-def test_resolver_falls_back_to_main_when_groq_bugteam_absent(monkeypatch):
-    monkeypatch.delitem(sys.modules, "groq_bugteam", raising=False)
-    fake_main = types.ModuleType("__main__")
-    _attach_required_groq_attributes(fake_main)
-    monkeypatch.setitem(sys.modules, "__main__", fake_main)
-
-    resolved_module = groq_bugteam_spec.resolve_groq_bugteam_module()
-
-    assert resolved_module is fake_main
-
-
-def test_resolver_falls_back_to_main_when_registered_module_is_stub(monkeypatch):
-    stub_groq_bugteam = types.ModuleType("groq_bugteam")
-    stub_groq_bugteam.call_groq_with_fallback = lambda *args, **kwargs: None
-    monkeypatch.setitem(sys.modules, "groq_bugteam", stub_groq_bugteam)
-    complete_main = types.ModuleType("__main__")
-    _attach_required_groq_attributes(complete_main)
-    monkeypatch.setitem(sys.modules, "__main__", complete_main)
-
-    resolved_module = groq_bugteam_spec.resolve_groq_bugteam_module()
-
-    assert resolved_module is complete_main
-
-
-def test_resolver_raises_when_registered_module_missing_required_attributes(
-    monkeypatch,
-):
-    stub_groq_bugteam = types.ModuleType("groq_bugteam")
-    stub_groq_bugteam.call_groq_with_fallback = lambda *args, **kwargs: None
-    monkeypatch.setitem(sys.modules, "groq_bugteam", stub_groq_bugteam)
-    monkeypatch.delitem(sys.modules, "__main__", raising=False)
-
-    try:
-        groq_bugteam_spec.resolve_groq_bugteam_module()
-    except RuntimeError as resolver_error:
-        resolver_error_text = str(resolver_error)
-        assert "parse_json_object" in resolver_error_text
-        assert "preserve_trailing_newline" in resolver_error_text
-    else:
-        raise AssertionError("resolver should have raised RuntimeError")
-
-
-def test_resolver_raises_when_neither_module_available(monkeypatch):
-    monkeypatch.delitem(sys.modules, "groq_bugteam", raising=False)
-    placeholder_main = types.ModuleType("__main__")
-    monkeypatch.setitem(sys.modules, "__main__", placeholder_main)
-
-    try:
-        groq_bugteam_spec.resolve_groq_bugteam_module()
-    except RuntimeError as resolver_error:
-        resolver_error_text = str(resolver_error)
-        assert "groq_bugteam" in resolver_error_text
-    else:
-        raise AssertionError("resolver should have raised RuntimeError")
-
-
-FAKE_API_KEY = "gsk_test_placeholder_value"
-
-
-def _install_fake_groq_bugteam_module(monkeypatch, response_object):
-    """Register a minimal fake groq_bugteam module for resolver lookup."""
-
-    fake_module = types.ModuleType("groq_bugteam")
-
-    def fake_call(api_key, messages, temperature, max_completion_tokens):
-        return types.SimpleNamespace(
-            content=json.dumps(response_object),
-            model="fake-model",
-        )
-
-    def fake_parse_json_object(text):
-        return json.loads(text)
-
-    def fake_preserve_trailing_newline(original, updated):
-        if original.endswith("\n") and not updated.endswith("\n"):
-            return updated + "\n"
-        if not original.endswith("\n") and updated.endswith("\n"):
-            return updated[:-1]
-        return updated
-
-    fake_module.call_groq_with_fallback = fake_call
-    fake_module.parse_json_object = fake_parse_json_object
-    fake_module.preserve_trailing_newline = fake_preserve_trailing_newline
-    monkeypatch.setitem(sys.modules, "groq_bugteam", fake_module)
-    monkeypatch.setenv("GROQ_API_KEY", FAKE_API_KEY)
-
-
-def test_skipped_entry_missing_finding_index_does_not_crash(monkeypatch):
-    original_file = "alpha\nbeta\n"
-    spec_list = [
-        {
-            "finding_index": 4,
-            "severity": "P1",
-            "category": "J",
-            "file": "sample.py",
-            "target_line_start": 1,
-            "target_line_end": 1,
-            "intended_change": "rename alpha",
-            "replacement_code": "alpha_fixed",
-            "acceptance_criteria": ["alpha_fixed appears on line 1"],
-        }
-    ]
-    patched_file = "alpha_fixed\nbeta\n"
-    fake_response = {
-        "updated_content": patched_file,
-        "applied_finding_indexes": [4],
-        "skipped": [{"reason": "malformed entry without finding_index"}],
-        "acceptance_checks": [
-            {
-                "finding_index": 4,
-                "criterion": "alpha_fixed appears on line 1",
-                "met": True,
-            }
-        ],
-    }
-    _install_fake_groq_bugteam_module(monkeypatch, fake_response)
-
-    outcome = groq_bugteam_spec.apply_fix_from_spec(spec_list, original_file)
-
-    assert outcome["updated_content"] == patched_file
-    assert outcome["applied_finding_indexes"] == [4]
-
-
-def test_null_updated_content_falls_back_to_current_content(monkeypatch):
-    original_file = "alpha\nbeta\n"
-    spec_list = [
-        {
-            "finding_index": 0,
-            "severity": "P2",
-            "category": "E",
-            "file": "sample.py",
-            "target_line_start": 1,
-            "target_line_end": 1,
-            "intended_change": "no-op fallback",
-            "replacement_code": "alpha",
-            "acceptance_criteria": ["alpha remains on line 1"],
-        }
-    ]
-    fake_response = {
-        "updated_content": None,
-        "applied_finding_indexes": [],
-        "skipped": [
-            {
-                "finding_index": 0,
-                "reason": "Groq returned null updated_content",
-            }
-        ],
-        "acceptance_checks": [],
-    }
-    _install_fake_groq_bugteam_module(monkeypatch, fake_response)
-
-    outcome = groq_bugteam_spec.apply_fix_from_spec(spec_list, original_file)
-
-    assert outcome["updated_content"] == original_file
-
-
-def test_null_collection_fields_coerce_to_empty_lists(monkeypatch):
-    original_file = "alpha\n"
-    spec_list = [
-        {
-            "finding_index": 1,
-            "severity": "P2",
-            "category": "E",
-            "file": "sample.py",
-            "target_line_start": 1,
-            "target_line_end": 1,
-            "intended_change": "no-op",
-            "replacement_code": "alpha",
-            "acceptance_criteria": ["alpha remains"],
-        }
-    ]
-    fake_response = {
-        "updated_content": original_file,
-        "applied_finding_indexes": None,
-        "skipped": None,
-        "acceptance_checks": None,
-    }
-    _install_fake_groq_bugteam_module(monkeypatch, fake_response)
-
-    outcome = groq_bugteam_spec.apply_fix_from_spec(spec_list, original_file)
-
-    assert outcome["applied_finding_indexes"] == []
-    assert outcome["skipped"] == []
-    assert outcome["acceptance_checks"] == []
-
-
-def test_dict_collection_fields_coerce_to_empty_lists(monkeypatch):
-    original_file = "alpha\n"
-    spec_list = [
-        {
-            "finding_index": 2,
-            "severity": "P2",
-            "category": "E",
-            "file": "sample.py",
-            "target_line_start": 1,
-            "target_line_end": 1,
-            "intended_change": "no-op",
-            "replacement_code": "alpha",
-            "acceptance_criteria": ["alpha remains"],
-        }
-    ]
-    fake_response = {
-        "updated_content": original_file,
-        "applied_finding_indexes": {"not": "a list"},
-        "skipped": {"0": "not a list either"},
-        "acceptance_checks": {"also": "a dict"},
-    }
-    _install_fake_groq_bugteam_module(monkeypatch, fake_response)
-
-    outcome = groq_bugteam_spec.apply_fix_from_spec(spec_list, original_file)
-
-    assert outcome["applied_finding_indexes"] == []
-    assert outcome["skipped"] == []
-    assert outcome["acceptance_checks"] == []
-
-
-def test_non_string_updated_content_falls_back_to_current_content(monkeypatch):
-    original_file = "alpha\nbeta\n"
-    spec_list = [
-        {
-            "finding_index": 0,
-            "severity": "P2",
-            "category": "E",
-            "file": "sample.py",
-            "target_line_start": 1,
-            "target_line_end": 1,
-            "intended_change": "no-op fallback",
-            "replacement_code": "alpha",
-            "acceptance_criteria": ["alpha remains on line 1"],
-        }
-    ]
-    fake_response = {
-        "updated_content": {"unexpected": "dict instead of str"},
-        "applied_finding_indexes": [],
-        "skipped": [],
-        "acceptance_checks": [],
-    }
-    _install_fake_groq_bugteam_module(monkeypatch, fake_response)
-
-    outcome = groq_bugteam_spec.apply_fix_from_spec(spec_list, original_file)
-
-    assert outcome["updated_content"] == original_file
-
-
-def test_run_spec_mode_main_emits_error_json_on_missing_api_key(
-    monkeypatch, capsys
-):
-    monkeypatch.delenv("GROQ_API_KEY", raising=False)
-    monkeypatch.setattr(
-        "groq_bugteam_dotenv.load_claude_dev_env_dotenv_file",
-        lambda: None,
-    )
-    spec_payload = {
-        "spec": [
-            {
-                "finding_index": 0,
-                "severity": "P1",
-                "category": "J",
-                "file": "sample.py",
-                "target_line_start": 1,
-                "target_line_end": 1,
-                "intended_change": "noop",
-                "replacement_code": "noop",
-                "acceptance_criteria": ["noop"],
-            }
-        ],
-        "current_content": "noop\n",
-    }
-    monkeypatch.setattr("sys.stdin", io.StringIO(json.dumps(spec_payload)))
-
-    with pytest.raises(SystemExit) as exit_info:
-        groq_bugteam_spec.run_spec_mode_main()
-
-    captured = capsys.readouterr()
-    emitted_outcome = json.loads(captured.out)
-    assert "error" in emitted_outcome
-    assert "GROQ_API_KEY" in emitted_outcome["error"]
-    assert exit_info.value.code != 0

package/skills/bugteam/SKILL_EVALS.md

@@ -1,309 +0,0 @@
-# Bugteam — Evaluation Suite
-
-Evaluation-driven iteration set for the `bugteam` skill, following [Anthropic — Agent Skills best practices: evaluation and iteration](https://platform.claude.com/docs/en/agents-and-tools/agent-skills/best-practices#evaluation-and-iteration).
-
-## Methodology
-
-Evals are split into two layers. Both layers run against the same trace but carry different failure semantics.
-
-**Layer A — Ironclad invariants.** Order-and-presence rules that MUST hold on every run regardless of fixture, regardless of model choice, regardless of the exact number of loops taken. Citations use **section headings and companion files** (`SKILL.md`, `CONSTRAINTS.md`, `reference/*.md`) — not fragile line numbers — so layout edits to `SKILL.md` do not invalidate the contract. If an assertion fails, either the run diverged from the skill or the cited text is ambiguous and needs patching.
-
-**Layer B — Fixture-dependent expectations.** The concrete tool trace predicted for a specific fixture (fixed PR state, canned audit XML, canned fix XML). Layer B is prediction — reality may diverge in small ways (extra `Bash("git rev-parse HEAD")` checkpoints the lead inserts for sanity; retry loops on transient failures; consolidated cleanup calls) without indicating a skill defect. Layer B failures trigger reconciliation, not auto-failure.
-
-**Process note.** This document was drafted before running a real trace. Layer B predictions are labeled *predicted*, not *observed*. On the first real run, every Layer B prediction is reconciled against the observed trace and the diffs written back here — that reconciliation is Cycle 0 of the iteration protocol below.
-
-## Ironclad invariants (Layer A, apply to every eval)
-
-Each invariant cites the normative section or companion file it derives from. All spawns use `Agent(..., run_in_background=true)`. Invariants apply uniformly across all eval fixtures.
-
-| # | Invariant | Citation |
-|---|---|---|
-| I-1 | `Bash` invoking `scripts/grant_project_claude_permissions.py` precedes the first audit `Agent` spawn. | `SKILL.md` § Step 0 |
-| I-2 | `Bash` invoking `scripts/revoke_project_claude_permissions.py` runs exactly once per invocation on every exit path, after teardown. | `SKILL.md` § Step 5 |
-| I-3 | Orchestration uses `Agent(..., run_in_background=true)` only — no `TeamCreate`, `TeamDelete`, `SendMessage`, or `Task` tool calls. | `SKILL.md` § Step 2; § Step 4 |
-| I-4 | `Agent` calls are fresh per loop (`run_in_background=true`; new `name` each loop). | `CONSTRAINTS.md` — **Fresh subagent per loop** |
-| I-5 | Audit sibling spawns pass `model="haiku"`; validator and fix spawns pass `model="opus"`. | `SKILL.md` § AUDIT action (parallel auditors); § FIX action; `CONSTRAINTS.md` — **Opus 4.7 at xhigh effort for validator and fix subagents** |
-| I-6 | Loop count ≤ 10 audits. 11th audit never fires. | `SKILL.md` YAML `description` (10-loop cap); § Step 3 (**Pre-audit** / **FIX** increment rules) |
-| I-7 | From loop 4 onward without convergence, eleven parallel `Agent(..., run_in_background=true)` calls in one message for audit. | `SKILL.md` § AUDIT action (**Parallel auditors**) |
-| I-8 | Lead reads `.bugteam-pr<N>-loop<L>.outcomes.xml` with the `Read` tool after each audit, before the next action. | `SKILL.md` § AUDIT action |
-| I-9 | Teardown sequence: `git worktree remove` each PR → `rmtree` `<run_temp_dir>` → Step 4.5 → revoke. | `SKILL.md` § Step 4; § Step 4.5; § Step 5 |
-| I-10 | The bugfind subagent posts ONE per-loop review; the bugfix subagent posts fix replies. The lead's only PR-write action is the Step 4.5 description rewrite. | `CONSTRAINTS.md` — **Audit/fix comment posting** |
-
-Any eval failing one or more Layer A invariants fails the run.
-
-## Observation strategy
-
-Evals run in a harness that intercepts the tool layer:
-
-- A **mock tool layer** records each tool call with its arguments and returns synthetic responses matching the real tool's response shape. Nothing hits GitHub; no real teammates spawn.
-- A **fixture repo** supplies deterministic git state and a mocked `gh` CLI that returns canned JSON for `pr view`, `pr diff`, and `api` calls.
-- **Assertions** run against the recorded call list, not against real PR state.
-
-The harness does not yet exist; this document defines its contract.
-
----
-
-## Eval 1 — Smoke: background subagent spawns fire correctly
-
-**Scenario.** PR exists; PR is a clean target with no unusual pre-conditions.
-
-**Trigger.** `/bugteam`
-
-**Layer A invariants.** I-1, I-2, I-3, I-4, I-5, I-8, I-9, I-10.
-
-**Layer B predicted trace (smoke).**
-1. `Bash("python .../grant_project_claude_permissions.py")` runs (Step 0).
-2. `Agent(subagent_type="code-quality-agent", name="bugfind-pr...-loop1", run_in_background=true, model="opus", ...)` spawned for AUDIT.
-3. Lead awaits background-completion notification, then `Read(".bugteam-pr42-loop1.outcomes.xml")`.
-4. `Agent(subagent_type="clean-coder", name="bugfix-pr...-loop1", run_in_background=true, model="opus", ...)` spawned for FIX (if findings).
-5. `Bash("python .../revoke_project_claude_permissions.py")` on exit.
-
-**Pass criteria.**
-- Non-zero `Agent(subagent_type="code-quality-agent", run_in_background=true)` and `Agent(subagent_type="clean-coder", run_in_background=true)` calls.
-
----
-
-## Eval 2 — Refusal: missing PR, no upstream diff
-
-**Scenario.** Current branch is `main` with no PR and no upstream difference.
-
-**Layer B predicted trace.**
-1. `pull_request_read(method="get", pullNumber=N, owner=O, repo=R)` → fails / no matching PR.
-2. `Bash("git merge-base HEAD origin/main")` → empty.
-3. No grant script.
-
-**Pass criteria.** Assistant message matches `No PR or upstream diff. /bugteam needs a target.`. Zero downstream tool calls.
-
----
-
-## Eval 3 — Refusal: uncommitted changes in working tree
-
-**Scenario.** Clean PR exists but `git status --porcelain` shows unstaged changes.
-
-**Pass criteria.** Assistant message matches `Uncommitted changes detected. Stash, commit, or revert before /bugteam.`. Zero downstream tool calls.
-
----
-
-## Eval 4 — Refusal: required subagent missing
-
-**Scenario.** `code-quality-agent` is present in the available-agents list; `clean-coder` is not.
-
-**Pass criteria.** Assistant message contains `Required subagent type clean-coder not installed.`. Zero grant script call, zero `Agent` spawns.
-
----
-
-## Eval 5 — Happy path: converges in 2 loops
-
-**Scenario.** PR #42 contains three P1 bugs all addressable by the mock fix subagent. Loop 1 audit returns 3 findings; loop 1 fix commits cleanly; loop 2 audit returns zero findings.
-
-**Layer A invariants.** I-1, I-2, I-3, I-4, I-5, I-6, I-8, I-9, I-10.
-
-**Layer B predicted trace.**
-
-| # | Tool call | Source |
-|---|---|---|
-| 1 | `Bash("python .../scripts/grant_project_claude_permissions.py")` | `SKILL.md` § Step 0 |
-| 2 | `pull_request_read(method="get", pullNumber=42, owner=..., repo=...)` | `SKILL.md` § Step 1 |
-| 3 | `Bash("git -C \"<run_temp_dir>/pr-42/worktree\" rev-parse HEAD")` → captures `starting_sha` | `SKILL.md` § Step 2 — **Loop state** block |
-| 4 | `Bash("mkdir -p <run_temp_dir>/pr-42")` | `SKILL.md` § AUDIT action |
-| 5 | `pull_request_read(method="get_diff", pullNumber=42, owner=..., repo=...)` → write to `<run_temp_dir>/pr-42/loop-1.patch` | `SKILL.md` § AUDIT action |
-| 6 | `Agent(subagent_type="code-quality-agent", name="bugfind-pr42-loop1", run_in_background=true, model="opus", description=..., prompt=<audit XML loop 1>)` | `SKILL.md` § AUDIT action |
-| 7 | Lead awaits background-completion notification | `SKILL.md` § AUDIT action |
-| 8 | `Read(".bugteam-pr42-loop1.outcomes.xml")` | `SKILL.md` § AUDIT action |
-| 9 | `Agent(subagent_type="clean-coder", name="bugfix-pr42-loop1", run_in_background=true, model="opus", description=..., prompt=<fix XML loop 1>)` | `SKILL.md` § FIX action |
-| 10 | Lead awaits background-completion notification | `SKILL.md` § FIX action |
-| 11 | `Read(".bugteam-pr42-loop1.outcomes.xml")` — bugfix outcome XML | `SKILL.md` § FIX action |
-| 12 | `Bash("git -C \"<run_temp_dir>/pr-42/worktree\" rev-parse HEAD")` → verify HEAD advanced | `SKILL.md` § FIX action (**Verify**) |
-| 13 | `Bash("git -C \"<run_temp_dir>/pr-42/worktree\" fetch origin <branch>")` → fetch remote state | `SKILL.md` § FIX action (**Verify**) |
-| 14 | `Bash("git -C \"<run_temp_dir>/pr-42/worktree\" rev-parse origin/<branch>")` → confirm matches HEAD | `SKILL.md` § FIX action (**Verify**) |
-| 15 | `pull_request_read(method="get_diff", pullNumber=42, owner=..., repo=...)` → write to `<run_temp_dir>/pr-42/loop-2.patch` | `SKILL.md` § AUDIT action |
-| 16 | `Agent(subagent_type="code-quality-agent", name="bugfind-pr42-loop2", run_in_background=true, ...)` (loop 2) | `SKILL.md` § AUDIT action |
-| 17 | Lead awaits background-completion notification | `SKILL.md` § AUDIT action |
-| 18 | `Read(".bugteam-pr42-loop2.outcomes.xml")` — zero findings | `SKILL.md` § AUDIT action |
-| 19 | `Bash("git worktree remove \"<run_temp_dir>/pr-42/worktree\"")` | `SKILL.md` § Step 4 step 1 |
-| 20 | `Bash("python -c \"...shutil.rmtree(r'<run_temp_dir>', ...)\"")` | `SKILL.md` § Step 4 step 2 (Windows-safe teardown) |
-| 21 | `pull_request_read(method="get_diff", pullNumber=42, owner=..., repo=...)` → write to `.bugteam-final.diff` | `SKILL.md` § Step 4.5 step 1 |
-| 22 | `pull_request_read(method="get", pullNumber=42, owner=..., repo=...)` → extract `.body`, write to `.bugteam-original-body.md` | `SKILL.md` § Step 4.5 step 2 |
-| 23 | `Agent(subagent_type="pr-description-writer", description=..., prompt=<brief>)` | `SKILL.md` § Step 4.5 |
-| 24 | `Write(".bugteam-final-body.md", <returned body>)` | `SKILL.md` § Step 4.5 step 4 |
-| 25 | `update_pull_request(pullNumber=42, owner=..., repo=..., body=...)` | `SKILL.md` § Step 4.5 step 4 |
-| 26 | `Bash("rm .bugteam-final.diff .bugteam-original-body.md .bugteam-final-body.md")` | `SKILL.md` § Step 4.5 step 5 |
-| 27 | `Bash("python .../scripts/revoke_project_claude_permissions.py")` | `SKILL.md` § Step 5 |
-
-**Pass criteria.**
-- All Layer A invariants hold.
-- Exactly 2 `Agent(name="bugfind-pr42-loop...")` calls, exactly 1 `Agent(name="bugfix-pr42-loop...")` call.
-- Final report contains `/bugteam exit: converged` and `Loops: 2`.
-
-**Process check after first real run.** Compare the observed trace against steps 1–27. Common expected divergences that should not fail the eval:
-- Extra `Bash("git rev-parse HEAD")` calls the lead inserts for bookkeeping.
-- Consolidated `Bash` calls (step 25 may split into two or three calls).
-- Extra `Read` calls when the lead re-reads an outcome XML to quote specific findings.
-- Reordered but still-Layer-A-compliant cleanup sequencing.
-
-Patch this table to match observation and annotate each correction.
-
----
-
-## Eval 6 — Stuck path: fix subagent produces no commit
-
-**Scenario.** Loop 1 audit finds 2 P1 bugs; the mock fix subagent reports both as `could_not_address` (no commit created).
-
-**Layer A invariants.** I-1, I-2, I-3, I-4, I-5, I-6, I-8, I-9, I-10. I-6 trivially holds.
-
-**Layer B predicted trace.** Identical to Eval 5 steps 1–12 with this divergence:
-- Step 11 bugfix outcome XML marks every finding `status="could_not_address"`.
-- Step 12 `Bash("git rev-parse HEAD")` returns the pre-fix SHA unchanged.
-- Skill sets exit reason = `stuck`, skips loop 2, and falls through to `rmtree`.
-
-**Pass criteria.**
-- Loop count stops at 1.
-- Final report contains `/bugteam exit: stuck` and names the two unresolved findings.
-- Steps 19–26 fire despite the stuck exit — I-2 and I-9 enforce this.
-
----
-
-## Eval 7 — Cap reached: 10 loops, no convergence
-
-**Scenario.** Mock audit returns one P2 finding every loop. Mock fix subagent always commits but never clears the finding.
-
-**Layer A invariants.** All of I-1 through I-10.
-
-**Layer B predicted behavior.**
-- Loops 1–3: single `Agent(name="bugfind-pr<N>-loop<L>", run_in_background=true)` per loop.
-- Loops 4–10: eleven parallel `Agent(name="bugfind-pr<N>-loop<L>-[a..k]", run_in_background=true)` in a single assistant message per loop (10 haiku + 1 opus validator); lead awaits the validator notification.
-- Each loop produces one `Agent(name="bugfix-pr<N>-loop<L>", run_in_background=true)`.
-- Exactly 10 audit phases, exactly 10 fix phases.
-- Steps 19–26 from Eval 5 fire at teardown.
-
-**Pass criteria.**
-- I-6 holds: exactly 10 audit phases.
-- I-7 holds: loops 4–10 each emit eleven audit `Agent` calls in a single assistant message.
-- Final report contains `/bugteam exit: cap reached` and the remaining bug count.
-
-**Process check.** The distinct `Agent(name=...)` audit-call count is a prediction. On the first real run, record the exact count and rewrite the formula here.
-
----
-
-## Eval 8 — Clean on first audit
-
-**Scenario.** Loop 1 audit returns zero findings.
-
-**Layer A invariants.** I-1, I-2, I-3, I-4, I-5, I-6, I-8, I-9, I-10.
-
-**Layer B predicted trace.** Eval 5 steps 1–8 and 19–26 only — no FIX phase because zero findings means the skill exits the loop at `last_action == "audited"` and `last_findings.total == 0`.
-
-**Pass criteria.**
-- Exactly 1 `Agent(subagent_type="code-quality-agent", run_in_background=true)` call, 0 fix agent spawns.
-- Bugfind's outcome XML records zero findings; the per-loop review POST carries body `## /bugteam loop 1 audit: 0P0 / 0P1 / 0P2 → clean`.
-- Step 4.5 and Step 5 still fire.
-
----
-
-## Eval 9 — Anchor fallback: finding outside diff
-
-**Scenario.** Loop 1 audit returns 3 findings; 1 anchors to a line outside the captured diff.
-
-**Layer A invariants.** Same as Eval 5.
-
-**Layer B predicted subagent-side behavior** (observed via the recorded `gh api ... /reviews` POST payload in the bugfind subagent fixture).
-- `comments[]` length in the POST body = 2 (anchored findings only).
-- Review body contains a `### Findings without a diff anchor` section listing the third finding.
-- Bugfix outcome XML marks all 3 findings with a `reply_comment_url`; the unanchored finding's `used_fallback="true"` and `finding_comment_url` equals the parent review URL.
-
-**Pass criteria.** Confirmed in the fixture's canned teammate outcome XML; Layer A invariants hold on the lead side.
-
----
-
-## Eval 10 — Review POST failure fallback
-
-**Scenario.** The first `POST /pulls/42/reviews` call from the bugfind teammate returns HTTP 422.
-
-**Layer B predicted teammate-side behavior.**
-- Bugfind teammate retries via the issue-comments endpoint `POST /repos/.../issues/42/comments` with a single body carrying the review header and every finding inline.
-- Every finding's outcome XML carries `used_fallback="true"` and the issue-comment URL as `finding_comment_url`.
-- Cycle continues to the FIX action without aborting.
-
-**Open item for the real run.** The issue-comments fallback uses `add_issue_comment(owner=..., repo=..., issueNumber=42, body=...)` (`SKILL.md` § Step 2.5 **Review POST fails**; full narrative in `reference/github-pr-reviews.md` § **Review POST failure fallback**). Before running Eval 10 for real, confirm the teammate obeys this shape — the fixture must assert the `add_issue_comment` tool call.
-
----
-
-## Eval 11 — Hook-blocked fix commit
-
-**Scenario.** Bugfix stages edits but `git commit` fails because a `pre-commit` hook returns non-zero.
-
-**Layer B predicted behavior.**
-- Bugfix teammate outcome XML marks every finding `status="hook_blocked"` with populated `<hook_output>`.
-- Bugfix teammate posts `Hook blocked the fix commit: <one-line summary>` to each finding comment.
-- Lead's `Bash("git rev-parse HEAD")` after fix detects no SHA change → exit reason `stuck`.
-- Steps 19–26 from Eval 5 fire at teardown.
-
-**Pass criteria.** Layer A I-2 and I-9 hold. Final report contains `/bugteam exit: stuck` and surfaces the hook_output summary.
-
----
-
-## Eval 12 — `pr-description-writer` unavailable, `general-purpose` available
-
-**Scenario.** The available-agents list does not include `pr-description-writer` but does include `general-purpose`.
-
-**Layer B predicted trace.** Eval 5 steps 1–21 identical; step 22 becomes:
-
-```
-Agent(subagent_type="general-purpose", description="Rewrite PR 42 body from cumulative diff", prompt=<same brief>)
-```
-
-Steps 23–26 follow normally.
-
-**Pass criteria.** Exactly 1 `Agent(subagent_type="general-purpose", ...)` call for the description rewrite. `gh pr edit` fires. Final report carries no Step 4.5 skip warning.
-
----
-
-## Eval 13 — Neither PR-description agent available
-
-**Scenario.** Neither `pr-description-writer` nor `general-purpose` appear in the available-agents list.
-
-**Layer B predicted trace.** Eval 5 steps 1–21, then skip steps 22–24. Steps 25–26 still fire.
-
-**Pass criteria.**
-- Zero `Agent` calls for PR description rewriting.
-- Zero `gh pr edit` calls.
-- Final report carries the Step 4.5 skip warning.
-- Layer A I-2 holds: revoke still fires.
-
----
-
-## Eval 14 — Permissions revoke on error path
-
-**Scenario.** Bugfind subagent completes but writes no outcomes XML (background subagent completes notification arrives with no file at the expected path).
-
-**Layer B predicted trace.** Eval 5 steps 1–7, then:
-- Lead awaits notification and calls `Read(".bugteam-pr42-loop1.outcomes.xml")` → file missing.
-- Skill sets exit reason = `error: outcomes XML missing after bugfind loop 1`.
-- Teardown (steps 19–26 from Eval 5) all fire.
-
-**Pass criteria.** Final report surfaces the error and the loop number. Revoke fires despite the error.
-
----
-
-## Iteration protocol
-
-1. **Cycle 0 — Reconcile predictions with reality.** On the first real run, diff every Layer B predicted trace against the observed trace. Patch this file to match reality and annotate each correction with a reason.
-2. **Baseline.** Run every eval with the skill unloaded. Record which cases the base model handles from memory versus which it gets wrong.
-3. **Treatment.** Run every eval with the skill loaded. Layer A invariants must pass on every case. Layer B mismatches trigger Cycle 0 reconciliation.
-4. **Regress on change.** Every edit to normative text in `SKILL.md`, `CONSTRAINTS.md`, `PROMPTS.md`, or `reference/*.md` sections that Layer A cites re-runs the full suite. A passing→failing transition on any Layer A invariant blocks the change. A Layer B mismatch after such an edit triggers a patch to the affected eval trace in the same commit.
-5. **Extend on gotcha.** When the skill misfires in real use, add a new eval that reproduces the miss before patching the orchestration or companion files.
-
-## Harness sketch (future work)
-
-A minimal Python harness under `packages/claude-dev-env/skills/bugteam/evals/`:
-
-- `harness.py` — loads a fixture, injects a mock tool layer that records calls and returns canned responses, invokes the lead with the trigger, collects the recorded trace, evaluates pass criteria.
-- `fixtures/` — one subdirectory per eval with canned `gh` responses, canned audit XML, canned fix XML, and the expected trace JSON.
-- `run_evals.py` — discovery + pass/fail reporting, exits non-zero on any failure for CI.
-- `invariants.py` — the Layer A assertion bank, imported by every fixture.
-
-## Open research items flagged during this pass
-
-1. **GitHub REST review-POST payload shape.** Eval 9 and Eval 10 depend on the exact body shape of `POST /pulls/<number>/reviews`. The `jq -n --rawfile ... --argjson ... | gh api ... --input -` fence lives in `SKILL.md` § Step 2.5 (**Review POST**); expanded copy in `reference/github-pr-reviews.md` § **Per-loop review**. Before running Eval 9/10 for real, fetch the current GitHub REST reference to confirm the request schema (fields `commit_id`, `event`, `body`, `comments[]`) and the multi-line anchor `{path, start_line, start_side, line, side, body}` shape still apply. Record the confirmed version and URL here.
-2. **Background subagent completion signal.** Real-run observation (loop 1 of eval run 2026-04-18) confirmed: background subagents self-terminate when their task is complete — the background-completion notification arrives and the lead reads the outcomes XML. No shutdown handshake required. `SKILL.md` § AUDIT / FIX actions document this flow. Layer A **I-4** encodes “fresh subagent per loop.”
-3. **Model override redundancy.** `clean-coder` pins `model: opus` in its agent definition, while `code-quality-agent` currently uses `model: inherit`. The explicit `model="opus"` in every spawn is insurance against frontmatter drift; on the first real run, confirm the resolved model is `claude-opus-4-7` and that effort defaults to `xhigh` (Claude Code shows the active effort next to the spinner per the model-config docs). If a teammate's frontmatter ever pins a non-default `effort:` value, that frontmatter overrides the model default for that subagent (https://code.claude.com/docs/en/model-config — *"Frontmatter effort applies when that skill or subagent is active, overriding the session level but not the environment variable."*).