claude-dev-env 8.83.0 → 8.83.2

package/hooks/blocking/test_build_eval_before_eval_write_gate.py

@@ -0,0 +1,162 @@
+"""Behavior tests for the eval write build-eval gate."""
+
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+HOOKS_DIRECTORY = Path(__file__).resolve().parent.parent
+if str(HOOKS_DIRECTORY) not in sys.path:
+    sys.path.insert(0, str(HOOKS_DIRECTORY))
+
+from blocking import build_eval_before_eval_write_gate as gate
+from hooks_constants.build_eval_before_eval_write_gate_constants import (
+    MISSING_BUILD_EVAL_BEFORE_EVAL_WRITE_REASON,
+    UNREADABLE_TRANSCRIPT_BEFORE_EVAL_WRITE_REASON,
+)
+
+GATE_SCRIPT = Path(gate.__file__)
+
+
+def _transcript(tmp_path: Path, build_eval: bool) -> Path:
+    lines = [json.dumps({"type": "user", "message": {"content": "Add a search flag."}})]
+    if build_eval:
+        skill_call = {
+            "type": "tool_use",
+            "name": "Skill",
+            "input": {"skill": "claude-api", "args": "build-eval"},
+        }
+        lines.append(json.dumps({"type": "assistant", "message": {"content": [skill_call]}}))
+    transcript_path = tmp_path / "session.jsonl"
+    transcript_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+    return transcript_path
+
+
+def _payload(file_path: str, transcript_path: Path, tool_name: str = "Write") -> dict[str, object]:
+    return {
+        "tool_name": tool_name,
+        "tool_input": {"file_path": file_path, "content": "[]"},
+        "transcript_path": str(transcript_path),
+    }
+
+
+def _reason(decision: dict[str, object] | None) -> object:
+    assert decision is not None
+    return decision["hookSpecificOutput"]["permissionDecisionReason"]
+
+
+def test_eval_case_write_without_build_eval_should_be_denied(tmp_path: Path) -> None:
+    decision = gate.decision_for(
+        _payload("/repo/evals/cases.json", _transcript(tmp_path, build_eval=False))
+    )
+    assert _reason(decision) == MISSING_BUILD_EVAL_BEFORE_EVAL_WRITE_REASON
+
+
+def test_eval_write_after_build_eval_should_pass(tmp_path: Path) -> None:
+    assert (
+        gate.decision_for(
+            _payload("/repo/evals/cases.json", _transcript(tmp_path, build_eval=True))
+        )
+        is None
+    )
+
+
+def test_hillclimb_edit_without_build_eval_should_be_denied(tmp_path: Path) -> None:
+    payload = _payload(
+        "/repo/.claude/hillclimb/search-flag/run.py",
+        _transcript(tmp_path, build_eval=False),
+        "Edit",
+    )
+    assert _reason(gate.decision_for(payload)) == MISSING_BUILD_EVAL_BEFORE_EVAL_WRITE_REASON
+
+
+def test_source_write_without_build_eval_should_pass(tmp_path: Path) -> None:
+    assert (
+        gate.decision_for(_payload("/repo/src/search.py", _transcript(tmp_path, build_eval=False)))
+        is None
+    )
+
+
+def test_eval_write_with_unreadable_transcript_should_be_denied(tmp_path: Path) -> None:
+    decision = gate.decision_for(_payload("/repo/evals/cases.json", tmp_path / "missing.jsonl"))
+    assert _reason(decision) == UNREADABLE_TRANSCRIPT_BEFORE_EVAL_WRITE_REASON
+
+
+def test_eval_paths_should_be_recognized() -> None:
+    assert gate.is_eval_path("evals/cases.json")
+    assert gate.is_eval_path("C:\\repo\\eval\\grader.py")
+    assert gate.is_eval_path("tools/eval_runner.py")
+    assert gate.is_eval_path("tests/search.eval.json")
+
+
+def test_notebook_edit_of_an_eval_notebook_should_be_denied(tmp_path: Path) -> None:
+    payload = {
+        "tool_name": "NotebookEdit",
+        "tool_input": {"notebook_path": "/repo/evals/grading.ipynb", "new_source": "x = 1"},
+        "transcript_path": str(_transcript(tmp_path, build_eval=False)),
+    }
+    assert _reason(gate.decision_for(payload)) == MISSING_BUILD_EVAL_BEFORE_EVAL_WRITE_REASON
+
+
+def test_multi_edit_of_an_eval_file_should_be_denied(tmp_path: Path) -> None:
+    payload = {
+        "tool_name": "MultiEdit",
+        "tool_input": {
+            "file_path": "/repo/evals/cases.json",
+            "edits": [{"old_string": "[]", "new_string": "[1]"}],
+        },
+        "transcript_path": str(_transcript(tmp_path, build_eval=False)),
+    }
+    assert _reason(gate.decision_for(payload)) == MISSING_BUILD_EVAL_BEFORE_EVAL_WRITE_REASON
+
+
+def test_gate_source_files_should_not_be_recognized() -> None:
+    assert not gate.is_eval_path("hooks/blocking/build_eval_before_eval_write_gate.py")
+    assert not gate.is_eval_path("hooks/blocking/test_build_eval_before_eval_write_gate.py")
+    assert not gate.is_eval_path(
+        "hooks/hooks_constants/build_eval_before_eval_write_gate_constants.py"
+    )
+
+
+def test_non_eval_paths_should_not_be_recognized() -> None:
+    assert not gate.is_eval_path("src/evaluate_score.py")
+    assert not gate.is_eval_path("evals/../src/app.py")
+    assert not gate.is_eval_path("docs/rule-guides/features-start-with-an-eval.md")
+    assert not gate.is_eval_path("src/retrieval.py")
+    assert not gate.is_eval_path("medieval/notes.md")
+    assert not gate.is_eval_path("")
+
+
+def test_script_should_print_a_deny_for_an_eval_write(tmp_path: Path) -> None:
+    payload = _payload("/repo/evals/cases.json", _transcript(tmp_path, build_eval=False))
+    completed = subprocess.run(
+        [sys.executable, str(GATE_SCRIPT)],
+        input=json.dumps(payload),
+        capture_output=True,
+        text=True,
+        check=False,
+    )
+    assert completed.returncode == 0
+    assert json.loads(completed.stdout)["hookSpecificOutput"]["permissionDecision"] == "deny"
+
+
+def test_apply_patch_to_an_eval_file_should_be_denied(tmp_path: Path) -> None:
+    patch = "*** Begin Patch\n*** Add File: evals/cases.json\n+[]\n*** End Patch\n"
+    payload = {
+        "tool_name": "apply_patch",
+        "tool_input": {"command": patch},
+        "cwd": str(tmp_path),
+        "transcript_path": str(_transcript(tmp_path, build_eval=False)),
+    }
+    assert _reason(gate.decision_for(payload)) == MISSING_BUILD_EVAL_BEFORE_EVAL_WRITE_REASON
+
+
+def test_unparseable_apply_patch_naming_an_eval_file_should_be_denied(tmp_path: Path) -> None:
+    patch = "*** Begin Patch\n*** Add File: evals/cases.json\n+[]\n"
+    payload = {
+        "tool_name": "apply_patch",
+        "tool_input": {"command": patch},
+        "cwd": str(tmp_path),
+        "transcript_path": str(_transcript(tmp_path, build_eval=False)),
+    }
+    assert _reason(gate.decision_for(payload)) == MISSING_BUILD_EVAL_BEFORE_EVAL_WRITE_REASON

package/hooks/blocking/test_pr_lifecycle_skill_gate.py

@@ -354,6 +354,14 @@ def test_feature_pull_request_with_build_eval_invocation_should_pass(tmp_path: P
     assert gate.decision_for(_create_payload(tmp_path, "feat: add a gate", transcript_path)) is None
 
 
+def test_is_build_eval_missing_should_follow_the_transcript(tmp_path: Path) -> None:
+    without_call = _transcript_with_lines(tmp_path, "without.jsonl", [SKILL_LOAD_LINE])
+    with_call = _transcript_with_lines(tmp_path, "with.jsonl", [BUILD_EVAL_LINE])
+    assert gate.is_build_eval_missing({"transcript_path": str(without_call)})
+    assert not gate.is_build_eval_missing({"transcript_path": str(with_call)})
+    assert not gate.is_build_eval_missing({"transcript_path": str(tmp_path / "missing.jsonl")})
+
+
 def test_non_feature_pull_request_without_build_eval_invocation_should_pass(tmp_path: Path) -> None:
     transcript_path = _transcript_with_lines(tmp_path, "session.jsonl", [SKILL_LOAD_LINE])
     assert gate.decision_for(_create_payload(tmp_path, "fix: repair a gate", transcript_path)) is None
@@ -416,3 +424,20 @@ def test_pull_request_dispatch_without_the_skill_loaded_should_be_denied(tmp_pat
 def test_dispatch_that_opens_no_pull_request_should_pass_without_the_skill(tmp_path: Path) -> None:
     transcript_path = _transcript_with_lines(tmp_path, "session.jsonl", [BUILD_EVAL_LINE])
     assert gate.decision_for(_dispatch_payload(tmp_path, {"work_order": "wo-1"}, transcript_path)) is None
+
+
+def test_eval_skill_call_status_should_separate_called_missing_and_unreadable(tmp_path: Path) -> None:
+    called_path = tmp_path / "called.jsonl"
+    skill_call = {"type": "tool_use", "name": "Skill", "input": {"skill": "claude-api", "args": "build-eval"}}
+    called_path.write_text(
+        json.dumps({"type": "assistant", "message": {"content": [skill_call]}}) + "\n", encoding="utf-8"
+    )
+    assert gate.eval_skill_call_status(_payload("git status", called_path)) is True
+    assert gate.eval_skill_call_status(_payload("git status", _transcript(tmp_path))) is False
+    assert gate.eval_skill_call_status(_payload("git status", tmp_path / "missing.jsonl")) is None
+
+
+def test_deny_decision_should_carry_the_reason_as_a_deny() -> None:
+    output = gate.deny_decision("Load the skill first.")["hookSpecificOutput"]
+    assert output["permissionDecision"] == "deny"
+    assert output["permissionDecisionReason"] == "Load the skill first."

package/hooks/blocking/test_pull_request_proof.py

@@ -14,6 +14,7 @@ from hooks_constants.pull_request_proof_constants import (
     MISSING_EVAL_SECTION_REASON,
     MISSING_EXISTING_WORK_REASON,
     MISSING_PROOF_REASON,
+    TEST_ONLY_PROOF_REASON,
     UNREADABLE_CHANGES_REASON,
 )
 
@@ -79,6 +80,120 @@ def test_gh_create_body_file_without_proof_should_be_denied(tmp_path: Path) -> N
     assert gate.missing_proof_reason(payload) == MISSING_PROOF_REASON
 
 
+def test_proof_whose_only_command_is_a_test_run_should_be_denied() -> None:
+    body = "## Proof in practice\nRan `pytest tests/test_x.py`.\n> 4 passed\n"
+    assert gate.missing_proof_reason(_mcp_create(body)) == TEST_ONLY_PROOF_REASON
+
+
+def test_proof_with_a_test_run_and_a_live_command_should_pass() -> None:
+    body = "## Proof in practice\nRan `pytest tests/test_x.py` and `python run.py --input a.txt`.\n> denied\n"
+    assert gate.missing_proof_reason(_mcp_create(body)) is None
+
+
+def test_proof_with_only_a_live_command_should_pass() -> None:
+    body = "## Proof in practice\nRan `python run.py --input a.txt`.\n> denied\n"
+    assert gate.missing_proof_reason(_mcp_create(body)) is None
+
+
+def test_proof_with_several_test_runners_should_be_denied() -> None:
+    body = (
+        "## Proof in practice\n`python -m pytest -q`, `python3 -m unittest discover`,"
+        " `uv run pytest`, `npm test`, `npx vitest run`, `jest`, `go test ./...`, `cargo test`\n"
+    )
+    assert gate.missing_proof_reason(_mcp_create(body)) == TEST_ONLY_PROOF_REASON
+
+
+def test_proof_with_a_test_run_and_file_references_should_be_denied() -> None:
+    body = "## Proof in practice\nRan `pytest tests/x.py` against `gate.py:12` and `src/x.py`.\n"
+    assert gate.missing_proof_reason(_mcp_create(body)) == TEST_ONLY_PROOF_REASON
+
+
+def test_proof_with_a_test_run_piped_into_tee_should_be_denied() -> None:
+    body = "## Proof in practice\nRan `pytest x | tee log`.\n"
+    assert gate.missing_proof_reason(_mcp_create(body)) == TEST_ONLY_PROOF_REASON
+
+
+def test_proof_with_a_test_run_piped_into_tail_should_be_denied() -> None:
+    body = "## Proof in practice\nRan `pytest x 2>&1 | tail -5`.\n"
+    assert gate.missing_proof_reason(_mcp_create(body)) == TEST_ONLY_PROOF_REASON
+
+
+def test_proof_with_a_live_script_piped_into_tail_should_pass() -> None:
+    body = "## Proof in practice\nRan `pytest x` and `./run.sh --input a.txt | tail -5`.\n"
+    assert gate.missing_proof_reason(_mcp_create(body)) is None
+
+
+def test_proof_with_a_fenced_live_command_beside_a_test_run_should_pass() -> None:
+    body = "## Proof in practice\nRan `pytest`.\n```\n$ python run.py\ndenied\n```\n"
+    assert gate.missing_proof_reason(_mcp_create(body)) is None
+
+
+def test_proof_with_only_a_fenced_test_run_should_be_denied() -> None:
+    body = "## Proof in practice\n```\n$ npm test\n12 passing\n```\n"
+    assert gate.missing_proof_reason(_mcp_create(body)) == TEST_ONLY_PROOF_REASON
+
+
+def test_gh_create_body_file_with_test_only_proof_should_be_denied(tmp_path: Path) -> None:
+    (tmp_path / "body.md").write_text("## Proof in practice\nRan `go test ./...`.\n", encoding="utf-8")
+    payload = _shell("gh pr create --draft --body-file body.md", tmp_path)
+    assert gate.missing_proof_reason(payload) == TEST_ONLY_PROOF_REASON
+
+
+def test_proof_with_only_py_test_should_be_denied() -> None:
+    body = "## Proof in practice\nRan `py.test`.\n"
+    assert gate.missing_proof_reason(_mcp_create(body)) == TEST_ONLY_PROOF_REASON
+
+
+def test_proof_with_only_a_fenced_pytest_exe_should_be_denied() -> None:
+    body = "## Proof in practice\n```\n$ pytest.exe\n```\n"
+    assert gate.missing_proof_reason(_mcp_create(body)) == TEST_ONLY_PROOF_REASON
+
+
+def test_proof_with_only_a_source_citation_should_pass() -> None:
+    body = "## Proof in practice\nSee `src/x.py`.\n"
+    assert gate.missing_proof_reason(_mcp_create(body)) is None
+
+
+def test_proof_with_unittest_after_an_interpreter_flag_should_be_denied() -> None:
+    body = "## Proof in practice\nRan `python -q -m unittest discover`.\n"
+    assert gate.missing_proof_reason(_mcp_create(body)) == TEST_ONLY_PROOF_REASON
+
+
+def test_proof_with_unittest_after_a_valued_interpreter_flag_should_be_denied() -> None:
+    body = "## Proof in practice\nRan `python -W default -m unittest`.\n"
+    assert gate.missing_proof_reason(_mcp_create(body)) == TEST_ONLY_PROOF_REASON
+
+
+def test_proof_with_a_script_that_mentions_unittest_should_pass() -> None:
+    body = "## Proof in practice\nRan `python script.py -m unittest`.\n"
+    assert gate.missing_proof_reason(_mcp_create(body)) is None
+
+
+def test_proof_with_a_module_other_than_unittest_should_pass() -> None:
+    body = "## Proof in practice\nRan `python -W default -m my_app`.\n"
+    assert gate.missing_proof_reason(_mcp_create(body)) is None
+
+
+def test_proof_with_a_quoted_test_runner_word_should_be_denied() -> None:
+    body = "## Proof in practice\nRan `npm \"test\"` and `cargo \"test\"`.\n"
+    assert gate.missing_proof_reason(_mcp_create(body)) == TEST_ONLY_PROOF_REASON
+
+
+def test_proof_with_a_quoted_unittest_module_should_be_denied() -> None:
+    body = "## Proof in practice\nRan `python -m \"unittest\"`.\n"
+    assert gate.missing_proof_reason(_mcp_create(body)) == TEST_ONLY_PROOF_REASON
+
+
+def test_proof_with_coverage_running_unittest_should_be_denied() -> None:
+    body = "## Proof in practice\nRan `coverage run -m unittest discover`.\n"
+    assert gate.missing_proof_reason(_mcp_create(body)) == TEST_ONLY_PROOF_REASON
+
+
+def test_proof_with_coverage_running_an_application_should_pass() -> None:
+    body = "## Proof in practice\nRan `coverage run -m my_app`.\n"
+    assert gate.missing_proof_reason(_mcp_create(body)) is None
+
+
 def test_gh_create_with_fill_should_ask_for_a_body() -> None:
     assert gate.missing_proof_reason(_shell("gh pr create --fill")) == MISSING_BODY_REASON
 

package/hooks/blocking/test_reply_length_gate.py

@@ -354,26 +354,114 @@ def test_should_deny_a_decision_card_that_hedges_in_an_option(
     assert '"probably"' in stderr_text
 
 
-def test_should_allow_a_long_decision_card_with_no_hedge(
+def short_card(**all_overrides: object) -> dict[str, object]:
+    card: dict[str, object] = {
+        "question": "Exclude the call button?",
+        "context": "Crops show the call button flagged in 7 themes.",
+        "reason": "Seven themes pass with it excluded.",
+        "options": [
+            {"label": "Exclude it", "consequence": "Seven themes pass."},
+            {"label": "Leave it", "consequence": "Seven themes keep a warning."},
+        ],
+        "recommended": 0,
+    }
+    card.update(all_overrides)
+    return card
+
+
+def test_should_allow_a_card_context_of_two_fifteen_word_sentences(
     monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
 ) -> None:
     exit_code, stderr_text = run_gate(
         monkeypatch,
         capsys,
         DECISION_TOOL_NAME,
-        {
-            "question": "Exclude the call button?",
-            "context": " ".join([FIFTEEN_WORD_SENTENCE] * 5),
-            "options": [
-                {"label": "Exclude it", "consequence": SIXTEEN_WORD_SENTENCE},
-                {"label": "Leave it", "consequence": "Three themes keep a warning."},
-            ],
-            "recommended": 0,
-        },
+        short_card(context=" ".join([FIFTEEN_WORD_SENTENCE] * 2)),
     )
     assert (exit_code, stderr_text) == (0, "")
 
 
+def test_should_deny_a_card_context_of_four_sentences(
+    monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
+) -> None:
+    exit_code, stderr_text = run_gate(
+        monkeypatch,
+        capsys,
+        DECISION_TOOL_NAME,
+        short_card(context="One. Two. Three. Four."),
+    )
+    assert exit_code == 2
+    assert "Card context too long: 4 sentences, limit 2." in stderr_text
+    assert "publish an Artifact page with a tap button per item and a Send button" in stderr_text
+
+
+@pytest.mark.parametrize(
+    "context_text",
+    [
+        "Three themes fail.\n- `Cedar`\n- `Birch`",
+        "Open one.\n- https://github.com/owner/repo/pull/12\n- https://github.com/owner/repo/pull/13",
+    ],
+)
+def test_should_deny_a_card_context_list_whose_items_are_code_or_urls(
+    monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], context_text: str
+) -> None:
+    exit_code, stderr_text = run_gate(
+        monkeypatch,
+        capsys,
+        DECISION_TOOL_NAME,
+        short_card(context=context_text),
+    )
+    assert exit_code == 2
+    assert "Card context holds a list of items" in stderr_text
+
+
+def test_should_deny_a_card_context_that_lists_several_items_and_name_the_page_flow(
+    monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
+) -> None:
+    exit_code, stderr_text = run_gate(
+        monkeypatch,
+        capsys,
+        DECISION_TOOL_NAME,
+        short_card(context="Three themes fail.\n- Cedar\n- Birch"),
+    )
+    assert exit_code == 2
+    assert "Card context holds a list of items" in stderr_text
+    assert "publish an Artifact page with a tap button per item and a Send button" in stderr_text
+    assert "send a short reply that links the page" in stderr_text
+
+
+def test_should_deny_a_long_card_consequence(
+    monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]
+) -> None:
+    exit_code, stderr_text = run_gate(
+        monkeypatch,
+        capsys,
+        DECISION_TOOL_NAME,
+        short_card(
+            options=[
+                {"label": "Exclude it", "consequence": SIXTEEN_WORD_SENTENCE},
+                {"label": "Leave it", "consequence": "Seven themes keep a warning."},
+            ]
+        ),
+    )
+    assert exit_code == 2
+    assert "Sentence too long: 16 words, limit 15" in stderr_text
+
+
+@pytest.mark.parametrize("card_field_name", ["question", "reason"])
+def test_should_deny_a_long_sentence_in_the_card_question_and_reason(
+    monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str], card_field_name: str
+) -> None:
+    exit_code, stderr_text = run_gate(
+        monkeypatch,
+        capsys,
+        DECISION_TOOL_NAME,
+        short_card(**{card_field_name: SIXTEEN_WORD_SENTENCE}),
+    )
+    assert exit_code == 2
+    assert "Sentence too long: 16 words" in stderr_text
+
+
 @pytest.mark.parametrize(
     "reply_text",
     ["It maybe passed.", "Perhaps it passed.", "It might be the cache.", "I am not sure it passed."],

package/hooks/context_budget/tests/test_model.py

@@ -0,0 +1,65 @@
+"""Tests for the context budget data shapes."""
+
+from __future__ import annotations
+
+from context_budget.model import (
+    BaselineEntry,
+    BudgetFinding,
+    BudgetPolicy,
+    ContextKind,
+    FileMeasure,
+    HookBudget,
+    SectionScan,
+)
+
+SKILL_ENTRY_KIND = ContextKind("skill entry", ("**/SKILL.md",), 200, True)
+
+
+def test_policy_holds_kinds_hooks_and_baselines() -> None:
+    hook = HookBudget("startup print", ("python", "hook.py"), "{}", 1500)
+    baseline = BaselineEntry(210, ("Plan",))
+
+    policy = BudgetPolicy(
+        section_detail_line_limit=12,
+        all_kinds=(SKILL_ENTRY_KIND,),
+        all_hooks=(hook,),
+        baseline_entry_by_path={"a/SKILL.md": baseline},
+        baseline_characters_by_hook_name={"startup print": 5200},
+        is_baseline_the_prior_text=True,
+    )
+
+    assert policy.section_detail_line_limit == 12
+    assert policy.all_kinds == (SKILL_ENTRY_KIND,)
+    assert policy.all_hooks == (hook,)
+    assert policy.baseline_entry_by_path["a/SKILL.md"] == baseline
+    assert policy.baseline_characters_by_hook_name["startup print"] == 5200
+    assert policy.is_baseline_the_prior_text is True
+
+
+def test_baseline_entry_holds_no_line_count_for_a_section_only_file() -> None:
+    entry = BaselineEntry(None, ("Steps", "Steps"))
+
+    assert entry.lines is None
+    assert entry.all_section_headings == ("Steps", "Steps")
+
+
+def test_policy_defaults_are_empty() -> None:
+    policy = BudgetPolicy(section_detail_line_limit=12, all_kinds=())
+
+    assert policy.all_hooks == ()
+    assert policy.baseline_entry_by_path == {}
+    assert policy.baseline_characters_by_hook_name == {}
+    assert policy.is_baseline_the_prior_text is False
+
+
+def test_measure_and_finding_hold_their_fields() -> None:
+    section = SectionScan("Plan", 15, False)
+    measure = FileMeasure(SKILL_ENTRY_KIND, 210, (section,))
+    finding = BudgetFinding("a/SKILL.md", "Plan", "section exceeds limit")
+
+    assert measure.kind == SKILL_ENTRY_KIND
+    assert measure.line_count == 210
+    assert measure.all_over_limit_sections == (section,)
+    assert finding.path == "a/SKILL.md"
+    assert finding.scope == "Plan"
+    assert finding.message == "section exceeds limit"

package/hooks/features/conduct-gates.md

@@ -13,6 +13,7 @@ This family stops a chat reply, edited message, or tool call when its payload br
 - `blocking/step_note_gate.py` requires a status line before a tool call while its opt-in flag is on.
 - `blocking/verify_before_acting.py` blocks a completed mutating call when the agent's reasoning contains an unchecked claim.
 - `blocking/pr_lifecycle_skill_gate.py` denies a commit, push, pull request action, or merge until the `pr-lifecycle` skill appears in the transcript since the last compaction.
+- `blocking/build_eval_before_eval_write_gate.py` denies a write to an eval file until a Skill call named exactly `claude-api` with args `build-eval`, or the `/claude-api build-eval` command, appears in the transcript.
 - `blocking/merge_review_gate.py` denies an agent's merge until a review pass on the merged head reported zero findings, or ruled out every finding and linked the posted rulings. It reads the receipt `scripts/merge_review_receipt.py` writes, requires the merge, auto-merge or merge-queue entry to be pinned to that head, and skips repositories listed in `CLAUDE_MERGE_REVIEW_EXEMPT_REPOSITORIES`. Its deny reason names the review command `CLAUDE_MERGE_REVIEW_COMMAND` sets.
 - `blocking/merge_over_review_offer_gate.py` denies a decision card or question that offers a merge while it names open review findings, a red check, unaddressed review comments, or an override. It reads the whole card as one text: every question, header, context, reason, and option, whether the option is a string or a label with a consequence or description.
 - `blocking/artifact_dark_mode_gate.py` denies an Artifact page publish when the page, inside the publish wrapper, keeps a light body or shows text under 4.5:1 contrast in dark mode. `blocking/artifact_dark_mode_render.cjs` renders the page in headless Chromium for it.
@@ -28,6 +29,7 @@ This family stops a chat reply, edited message, or tool call when its payload br
 - `blocking/step_note_gate.py` runs on `PreToolUse`, matcher `*`, timeout `15` seconds in `hooks.json`.
 - `blocking/verify_before_acting.py` runs on `PostToolUse`, matcher `Write|Edit|MultiEdit|NotebookEdit|Agent|Task|apply_patch|Bash|PowerShell|mcp__.*`, timeout `10` seconds in `hooks.json`.
 - `blocking/pr_lifecycle_skill_gate.py` runs on `PreToolUse`, matcher `Bash|PowerShell|mcp__.*__(create_pull_request|merge_pull_request|enable_pr_auto_merge|update_pull_request|actions_run_trigger)`, timeout `10` seconds in `hooks.json`.
+- `blocking/build_eval_before_eval_write_gate.py` runs on `PreToolUse`, matcher `Write|Edit|MultiEdit|NotebookEdit|apply_patch`, timeout `10` seconds in `hooks.json`.
 - `blocking/merge_review_gate.py` runs on `PreToolUse`, matcher `Bash|PowerShell|mcp__.*__(merge_pull_request|enable_pr_auto_merge)`, timeout `10` seconds in `hooks.json`.
 - `blocking/merge_over_review_offer_gate.py` runs on `PreToolUse`, matcher `AskUserQuestion|mcp__.*__ask_decision`, timeout `10` seconds in `hooks.json`.
 - `blocking/artifact_dark_mode_gate.py` runs on `PreToolUse`, matcher `Artifact`, timeout `60` seconds in `hooks.json`.
@@ -47,6 +49,7 @@ Preconditions:
 - **Step note.** Input is a tool call after a transcript message without a status line while the flag is on. Run `python -m pytest packages/claude-dev-env/hooks/blocking/test_step_note_gate.py -q`. The adjacent test observes a block until a status line appears.
 - **Checked claim.** Input is a Write after hedged reasoning in the transcript. Run `python -m pytest packages/claude-dev-env/hooks/blocking/test_verify_before_acting.py -q`. The adjacent test observes a block and a logged blocked outcome.
 - **Lifecycle skill.** Input is a `git commit` or `gh pr create` command with no `pr-lifecycle` invocation in the transcript. Run `python -m pytest packages/claude-dev-env/hooks/blocking/test_pr_lifecycle_skill_gate.py -q`. The adjacent test observes a denial, then a silent allow once the skill or its slash command appears.
+- **Eval write.** Input is a Write to `evals/cases.json` with no build-eval call in the transcript. Run `python -m pytest packages/claude-dev-env/hooks/blocking/test_build_eval_before_eval_write_gate.py -q`. The adjacent test observes a denial, then an allow once the call appears; a write outside an eval path passes.
 - **Clean review.** Input is a merge with no receipt, a receipt with findings, a merge pinned to a newer head, an unpinned merge, or auto-merge with no head. Run `python -m pytest packages/claude-dev-env/hooks/blocking/test_merge_review_gate.py -q`. The adjacent test observes a denial for each; a merge or auto-merge pinned to the clean head and a merge in an exempt repository pass.
 - **Merge offer.** Input is a decision card or AskUserQuestion that offers "Merge now", the merge queue, or a yes to "Merge it now?" while any part of the card names open findings or unaddressed review comments. Run `python -m pytest packages/claude-dev-env/hooks/blocking/test_merge_over_review_offer_gate.py -q`. The adjacent test observes a denial for each, including an offer and findings in separate questions; a card with no merge offer, a card about the merge review check itself, a card about shipping a docs change, and a merge offer after a clean review pass.
 - **Dark mode page.** Input is an Artifact publish of a page that themes `html` but leaves `body` to the publish wrapper. Run `python -m pytest packages/claude-dev-env/hooks/blocking/test_artifact_dark_mode_gate.py -q`. The adjacent test observes a denial that names the text and its contrast; a page that themes `body` passes. The render tests skip on a host with no Node Playwright and Chromium.
@@ -59,6 +62,7 @@ Preconditions:
 - The final reply gate blocks the stop and returns its reason to the model, which rewrites the reply. Claude Code ends the turn anyway after eight blocks in a row. The cited number checks in both gates skip when the transcript cannot be read.
 - The edit marker gate scans replacement cards as well as the message text.
 - The handoff close gate reads the `body` and `comment` fields of issue tools only. A `gh issue close --comment` command in Bash and a pull request body that closes the issue with `Closes #N` do not reach it.
+- The eval write gate reads Write, Edit, MultiEdit, NotebookEdit and apply_patch calls. A Bash or PowerShell command that writes an eval file reaches it without the gate. An apply_patch that will not parse counts as an eval write when any of its words is an eval path. A missing or unreadable transcript denies the write.
 - The lifecycle gate counts only invocations after the last compaction, so a compacted session invokes `pr-lifecycle` again. A missing or unreadable transcript allows the call.
 - The dark mode gate denies a page publish when Node, Playwright or Chromium is missing, and its reason names the install command. It checks both dark signals, `prefers-color-scheme: dark` and `data-theme="dark"` on the root element.
 - The clean-review gate trusts the receipt. A receipt the agent writes by hand with zero findings, or with every finding ruled out and a rulings link, passes, so the pr-lifecycle skill names the review loop that writes it. A `gh pr merge` that names no repository is denied, because the gate cannot tell which repository the current branch belongs to.

package/hooks/hooks.json

@@ -102,6 +102,16 @@
           }
         ]
       },
+      {
+        "matcher": "Write|Edit|MultiEdit|NotebookEdit|apply_patch",
+        "hooks": [
+          {
+            "type": "command",
+            "command": "python3 ${CLAUDE_PLUGIN_ROOT}/hooks/blocking/build_eval_before_eval_write_gate.py",
+            "timeout": 10
+          }
+        ]
+      },
       {
         "matcher": "Bash|PowerShell|mcp__.*__(merge_pull_request|enable_pr_auto_merge)",
         "hooks": [

package/hooks/hooks_constants/build_eval_before_eval_write_gate_constants.py

@@ -0,0 +1,19 @@
+"""Constants for the gate that holds eval file writes until /claude-api build-eval runs."""
+
+ALL_FILE_WRITE_TOOL_NAMES = frozenset({"Write", "Edit", "MultiEdit", "NotebookEdit"})
+ALL_FILE_PATH_FIELDS = ("file_path", "notebook_path")
+APPLY_PATCH_TOOL_NAME = "apply_patch"
+ALL_EVAL_DIRECTORY_NAMES = frozenset({"eval", "evals", "hillclimb"})
+EVAL_FILE_NAME_WORD_PATTERN = r"^evals?(?:[._-]|$)|\.evals?\."
+MISSING_BUILD_EVAL_BEFORE_EVAL_WRITE_REASON = (
+    "This file is part of an eval, and this session has not invoked /claude-api"
+    " build-eval. An eval starts with that skill. Invoke the Skill tool with skill"
+    " 'claude-api' and args 'build-eval', follow the guide it loads, then write this"
+    " file again. Read ~/.claude/docs/rule-guides/features-start-with-an-eval.md for the rule."
+)
+UNREADABLE_TRANSCRIPT_BEFORE_EVAL_WRITE_REASON = (
+    "This file is part of an eval, and this hook could not read a session transcript"
+    " from the payload's transcript_path or agent_transcript_path field. The write is"
+    " held because no /claude-api build-eval call can be confirmed. A retry clears only"
+    " once the hook receives a readable transcript."
+)

package/hooks/hooks_constants/pull_request_proof_constants.py

@@ -24,6 +24,44 @@ MISSING_PROOF_REASON = (
     " prove the change, leave the pull request unopened and tell the user what you could"
     " not run. Read " + PROOF_GUIDE_PATH + " for the full rule."
 )
+INLINE_COMMAND_PATTERN = r"(?<!`)`([^`\n]+)`(?!`)"
+CODE_FENCE_MARKER = "```"
+FENCED_COMMAND_PROMPT = "$ "
+UNITTEST_MODULE_NAME = "unittest"
+PATH_WITH_LINE_PATTERN = r"^\S+:\d+(?:[-:]\d+)?$"
+SINGLE_FILE_NAME_PATTERN = r"^\S*[/\\]\S*$|^[\w.-]+\.[A-Za-z0-9]+$"
+ALL_PASSTHROUGH_PROGRAM_NAMES = frozenset({"tee", "tail", "head", "grep", "cat"})
+ALL_TEST_RUNNER_COMMAND_PREFIXES = frozenset(
+    {
+        ("jest",),
+        ("vitest",),
+        ("mocha",),
+        ("npm", "test"),
+        ("npm", "t"),
+        ("npm", "run", "test"),
+        ("yarn", "test"),
+        ("yarn", "run", "test"),
+        ("pnpm", "test"),
+        ("pnpm", "run", "test"),
+        ("bun", "test"),
+        ("npx", "jest"),
+        ("npx", "vitest"),
+        ("npx", "mocha"),
+        ("go", "test"),
+        ("cargo", "test"),
+        ("deno", "test"),
+        ("dotnet", "test"),
+        ("unittest",),
+    }
+)
+UNIT_TESTS_GUIDE_LINE = "Unit tests are welcome as extra evidence. The live run is the proof."
+TEST_ONLY_PROOF_REASON = (
+    "Every command under this pull request's 'Proof in practice' heading runs a test"
+    " suite. The proof rule says: \"" + UNIT_TESTS_GUIDE_LINE + "\" Run the changed thing"
+    " the way its user runs it, once without the change and once with it. List each"
+    " command in backticks and quote the output lines that show the difference. Keep the"
+    " test run in the section as extra evidence. Read " + PROOF_GUIDE_PATH + " for the full rule."
+)
 EXISTING_WORK_HEADING_PATTERN = r"^(#{1,6})[ \t]+existing work[ \t]*#*[ \t]*$"
 EXISTING_WORK_GUIDE_PATH = "~/.claude/docs/rule-guides/explore-thoroughly.md"
 MISSING_EXISTING_WORK_REASON = (