master-skill 0.11.0 → 0.12.0

package/scripts/tests/test_validate_promptfoo_configs.py

@@ -1,386 +0,0 @@
-"""Tests for validate-promptfoo-configs.py.
-
-Exercises the persona-test config validator on synthetic prebuilt/ trees so
-the production tests/persona/ files don't get coupled to the unit tests.
-"""
-from __future__ import annotations
-
-import importlib.util
-import json
-import sys
-import textwrap
-from pathlib import Path
-
-import pytest
-
-
-def _load_module():
-    spec_path = Path(__file__).resolve().parents[1] / "validate-promptfoo-configs.py"
-    spec = importlib.util.spec_from_file_location("vppc", spec_path)
-    mod = importlib.util.module_from_spec(spec)
-    sys.modules["vppc"] = mod
-    spec.loader.exec_module(mod)
-    return mod
-
-
-def _write_master(tmp_root: Path, slug: str, *, signature_phrases=None) -> None:
-    """Create a minimal prebuilt/master-<slug>/meta.json under tmp_root."""
-    if signature_phrases is None:
-        signature_phrases = ["本来无一物", "明心见性", "不立文字", "无念为宗"]
-    d = tmp_root / "prebuilt" / f"master-{slug}"
-    d.mkdir(parents=True, exist_ok=True)
-    (d / "meta.json").write_text(
-        json.dumps(
-            {
-                "slug": slug,
-                "signature_phrases": signature_phrases,
-                "style": {
-                    "all": "x" * 40,
-                    "qa": "x" * 40,
-                    "monologue": "x" * 40,
-                },
-            },
-            ensure_ascii=False,
-        ),
-        encoding="utf-8",
-    )
-
-
-def _persona_dir(tmp_root: Path) -> Path:
-    d = tmp_root / "tests" / "persona"
-    d.mkdir(parents=True, exist_ok=True)
-    return d
-
-
-SHARED_YAML_HUINENG = textwrap.dedent(
-    """\
-    huineng_persona_prompt: |
-      你扮演中国禅宗六祖慧能大师。请回答用户提问。
-
-      用户问题：{{question}}
-    """
-)
-
-VALID_CONFIG_BODY = textwrap.dedent(
-    """\
-    description: "master-huineng persona fidelity — RAW/SPE/CUS"
-    providers:
-      - id: anthropic:messages:claude-opus-4-7
-    defaultTest:
-      options:
-        provider:
-          id: anthropic:messages:claude-opus-4-7
-    prompts:
-      - |
-        你扮演中国禅宗六祖慧能大师。请回答用户提问。
-
-        用户问题：{{question}}
-    tests:
-      - description: "RAW: 拒答政治"
-        vars:
-          question: "当代政治怎么看？"
-        assert:
-          - type: llm-rubric
-            value: "回答应礼貌拒绝评论当代政治议题并引回禅宗本怀。"
-      - description: "SPE: 顿渐之辨"
-        vars:
-          question: "请讲顿悟和渐修。"
-        assert:
-          - type: llm-rubric
-            value: "回答应包含南宗禅核心概念并立顿教立场。"
-          - type: contains-any
-            value:
-              - 本来无一物
-              - 明心见性
-      - description: "SPE: 风幡公案"
-        vars:
-          question: "风动幡动是什么？"
-        assert:
-          - type: llm-rubric
-            value: "回答应正确叙述风幡公案要旨：非风动非幡动仁者心动。"
-      - description: "CUS: 短句直指"
-        vars:
-          question: "学人初参请师指示。"
-        assert:
-          - type: llm-rubric
-            value: "回答应符合慧能短句直指答疑风格。"
-          - type: contains-any
-            value:
-              - 不立文字
-              - 无念为宗
-    """
-)
-
-
-# -----------------------------------------------------------------------------
-# Tests
-# -----------------------------------------------------------------------------
-
-def test_valid_config_passes(tmp_path, monkeypatch):
-    mod = _load_module()
-    _write_master(tmp_path, "huineng")
-    persona = _persona_dir(tmp_path)
-    (persona / "shared.yaml").write_text(SHARED_YAML_HUINENG, encoding="utf-8")
-    (persona / "huineng.promptfooconfig.yaml").write_text(
-        VALID_CONFIG_BODY, encoding="utf-8"
-    )
-    monkeypatch.setattr(mod, "PREBUILT_DIR", tmp_path / "prebuilt")
-    errors = mod.validate(persona)
-    assert errors == [], errors
-
-
-def test_missing_shared_yaml_is_error(tmp_path, monkeypatch):
-    mod = _load_module()
-    _write_master(tmp_path, "huineng")
-    persona = _persona_dir(tmp_path)
-    (persona / "huineng.promptfooconfig.yaml").write_text(
-        VALID_CONFIG_BODY, encoding="utf-8"
-    )
-    monkeypatch.setattr(mod, "PREBUILT_DIR", tmp_path / "prebuilt")
-    errors = mod.validate(persona)
-    assert any("shared.yaml" in e for e in errors), errors
-
-
-def test_no_configs_is_error(tmp_path, monkeypatch):
-    mod = _load_module()
-    persona = _persona_dir(tmp_path)
-    (persona / "shared.yaml").write_text(SHARED_YAML_HUINENG, encoding="utf-8")
-    monkeypatch.setattr(mod, "PREBUILT_DIR", tmp_path / "prebuilt")
-    errors = mod.validate(persona)
-    assert any("no *.promptfooconfig.yaml" in e for e in errors), errors
-
-
-def test_bad_filename_extension(tmp_path, monkeypatch):
-    mod = _load_module()
-    _write_master(tmp_path, "huineng")
-    persona = _persona_dir(tmp_path)
-    (persona / "shared.yaml").write_text(SHARED_YAML_HUINENG, encoding="utf-8")
-    # wrong suffix: this file won't even be picked up by the glob, so we
-    # instead drop a valid-suffix file but rely on the slug check.
-    (persona / "huineng.promptfooconfig.yaml").write_text(
-        VALID_CONFIG_BODY, encoding="utf-8"
-    )
-    (persona / "NoSuchMaster.promptfooconfig.yaml").write_text(
-        VALID_CONFIG_BODY, encoding="utf-8"
-    )
-    monkeypatch.setattr(mod, "PREBUILT_DIR", tmp_path / "prebuilt")
-    errors = mod.validate(persona)
-    assert any("NoSuchMaster" in e for e in errors), errors
-
-
-def test_slug_with_no_master_dir(tmp_path, monkeypatch):
-    mod = _load_module()
-    persona = _persona_dir(tmp_path)
-    (persona / "shared.yaml").write_text(SHARED_YAML_HUINENG, encoding="utf-8")
-    (persona / "ghostmaster.promptfooconfig.yaml").write_text(
-        VALID_CONFIG_BODY, encoding="utf-8"
-    )
-    # Create at least one prebuilt dir so it exists
-    (tmp_path / "prebuilt").mkdir()
-    monkeypatch.setattr(mod, "PREBUILT_DIR", tmp_path / "prebuilt")
-    errors = mod.validate(persona)
-    assert any("ghostmaster" in e and "no matching master" in e for e in errors), errors
-
-
-def test_dimension_coverage_required(tmp_path, monkeypatch):
-    mod = _load_module()
-    _write_master(tmp_path, "huineng")
-    persona = _persona_dir(tmp_path)
-    (persona / "shared.yaml").write_text(SHARED_YAML_HUINENG, encoding="utf-8")
-    body = textwrap.dedent(
-        """\
-        description: x
-        providers:
-          - id: anthropic:messages:claude-opus-4-7
-        defaultTest:
-          options:
-            provider:
-              id: anthropic:messages:claude-opus-4-7
-        prompts:
-          - |
-            你扮演中国禅宗六祖慧能大师。请回答用户提问。
-
-            用户问题：{{question}}
-        tests:
-          - description: "RAW: a"
-            vars: {question: q1}
-            assert:
-              - type: llm-rubric
-                value: "this rubric is long enough to satisfy minimum char check."
-          - description: "RAW: b"
-            vars: {question: q2}
-            assert:
-              - type: llm-rubric
-                value: "this rubric is long enough to satisfy minimum char check."
-          - description: "RAW: c"
-            vars: {question: q3}
-            assert:
-              - type: llm-rubric
-                value: "this rubric is long enough to satisfy minimum char check."
-          - description: "RAW: d"
-            vars: {question: q4}
-            assert:
-              - type: llm-rubric
-                value: "this rubric is long enough to satisfy minimum char check."
-        """
-    )
-    (persona / "huineng.promptfooconfig.yaml").write_text(body, encoding="utf-8")
-    monkeypatch.setattr(mod, "PREBUILT_DIR", tmp_path / "prebuilt")
-    errors = mod.validate(persona)
-    assert any("missing dimension 'SPE'" in e for e in errors), errors
-    assert any("missing dimension 'CUS'" in e for e in errors), errors
-
-
-def test_too_few_tests(tmp_path, monkeypatch):
-    mod = _load_module()
-    _write_master(tmp_path, "huineng")
-    persona = _persona_dir(tmp_path)
-    (persona / "shared.yaml").write_text(SHARED_YAML_HUINENG, encoding="utf-8")
-    body = textwrap.dedent(
-        """\
-        description: x
-        providers:
-          - id: anthropic:messages:claude-opus-4-7
-        defaultTest:
-          options:
-            provider:
-              id: anthropic:messages:claude-opus-4-7
-        prompts:
-          - |
-            你扮演中国禅宗六祖慧能大师。请回答用户提问。
-
-            用户问题：{{question}}
-        tests:
-          - description: "RAW: a"
-            vars: {question: q1}
-            assert:
-              - type: llm-rubric
-                value: "long enough rubric to clear the floor on chars."
-          - description: "SPE: b"
-            vars: {question: q2}
-            assert:
-              - type: llm-rubric
-                value: "long enough rubric to clear the floor on chars."
-          - description: "CUS: c"
-            vars: {question: q3}
-            assert:
-              - type: llm-rubric
-                value: "long enough rubric to clear the floor on chars."
-        """
-    )
-    (persona / "huineng.promptfooconfig.yaml").write_text(body, encoding="utf-8")
-    monkeypatch.setattr(mod, "PREBUILT_DIR", tmp_path / "prebuilt")
-    errors = mod.validate(persona)
-    assert any("at least 4 tests required" in e for e in errors), errors
-
-
-def test_unknown_dimension_prefix_rejected(tmp_path, monkeypatch):
-    mod = _load_module()
-    _write_master(tmp_path, "huineng")
-    persona = _persona_dir(tmp_path)
-    (persona / "shared.yaml").write_text(SHARED_YAML_HUINENG, encoding="utf-8")
-    bad = VALID_CONFIG_BODY.replace("RAW: 拒答政治", "XXX: bogus prefix")
-    (persona / "huineng.promptfooconfig.yaml").write_text(bad, encoding="utf-8")
-    monkeypatch.setattr(mod, "PREBUILT_DIR", tmp_path / "prebuilt")
-    errors = mod.validate(persona)
-    assert any("must start with one of" in e for e in errors), errors
-
-
-def test_missing_llm_rubric_rejected(tmp_path, monkeypatch):
-    mod = _load_module()
-    _write_master(tmp_path, "huineng")
-    persona = _persona_dir(tmp_path)
-    (persona / "shared.yaml").write_text(SHARED_YAML_HUINENG, encoding="utf-8")
-    # Strip every llm-rubric block by replacing the type. Use textwrap.dedent
-    # only on the snippet so indentation matches what VALID_CONFIG_BODY has.
-    body = VALID_CONFIG_BODY.replace("type: llm-rubric", "type: contains")
-    (persona / "huineng.promptfooconfig.yaml").write_text(body, encoding="utf-8")
-    monkeypatch.setattr(mod, "PREBUILT_DIR", tmp_path / "prebuilt")
-    errors = mod.validate(persona)
-    assert any("must have at least one llm-rubric" in e for e in errors), errors
-
-
-def test_contains_any_value_must_be_anchor(tmp_path, monkeypatch):
-    mod = _load_module()
-    _write_master(tmp_path, "huineng", signature_phrases=["本来无一物"])
-    persona = _persona_dir(tmp_path)
-    (persona / "shared.yaml").write_text(SHARED_YAML_HUINENG, encoding="utf-8")
-    body = VALID_CONFIG_BODY.replace(
-        "- 本来无一物\n              - 明心见性",
-        "- 完全不存在的短语\n              - 另一个伪造短语",
-    )
-    (persona / "huineng.promptfooconfig.yaml").write_text(body, encoding="utf-8")
-    monkeypatch.setattr(mod, "PREBUILT_DIR", tmp_path / "prebuilt")
-    errors = mod.validate(persona)
-    assert any("not in huineng's fidelity anchors" in e for e in errors), errors
-
-
-def test_prompt_must_match_shared(tmp_path, monkeypatch):
-    mod = _load_module()
-    _write_master(tmp_path, "huineng")
-    persona = _persona_dir(tmp_path)
-    (persona / "shared.yaml").write_text(SHARED_YAML_HUINENG, encoding="utf-8")
-    bad = VALID_CONFIG_BODY.replace(
-        "你扮演中国禅宗六祖慧能大师。请回答用户提问。",
-        "你是某个完全不同的角色。",
-    )
-    (persona / "huineng.promptfooconfig.yaml").write_text(bad, encoding="utf-8")
-    monkeypatch.setattr(mod, "PREBUILT_DIR", tmp_path / "prebuilt")
-    errors = mod.validate(persona)
-    assert any("does not match shared.yaml" in e for e in errors), errors
-
-
-def test_judge_provider_required(tmp_path, monkeypatch):
-    mod = _load_module()
-    _write_master(tmp_path, "huineng")
-    persona = _persona_dir(tmp_path)
-    (persona / "shared.yaml").write_text(SHARED_YAML_HUINENG, encoding="utf-8")
-    body = VALID_CONFIG_BODY.replace(
-        "defaultTest:\n  options:\n    provider:\n      id: anthropic:messages:claude-opus-4-7\n",
-        "",
-    )
-    (persona / "huineng.promptfooconfig.yaml").write_text(body, encoding="utf-8")
-    monkeypatch.setattr(mod, "PREBUILT_DIR", tmp_path / "prebuilt")
-    errors = mod.validate(persona)
-    assert any("defaultTest.options.provider" in e for e in errors), errors
-
-
-def test_prompt_missing_question_var_rejected(tmp_path, monkeypatch):
-    """A prompt that doesn't reference {{question}} silently no-ops every
-    test case — must be flagged. Also serves as a regression guard against
-    contributors stripping the var when copying templates."""
-    mod = _load_module()
-    _write_master(tmp_path, "huineng")
-    persona = _persona_dir(tmp_path)
-    # Both shared.yaml and the inlined prompt must agree (so the sync check
-    # passes), and neither references {{question}} (so the new check fires).
-    shared_no_var = textwrap.dedent(
-        """\
-        huineng_persona_prompt: |
-          你扮演中国禅宗六祖慧能大师。请回答用户提问。
-
-          这里故意未引用问题变量。
-        """
-    )
-    (persona / "shared.yaml").write_text(shared_no_var, encoding="utf-8")
-    body = VALID_CONFIG_BODY.replace(
-        "用户问题：{{question}}",
-        "这里故意未引用问题变量。",
-    )
-    (persona / "huineng.promptfooconfig.yaml").write_text(body, encoding="utf-8")
-    monkeypatch.setattr(mod, "PREBUILT_DIR", tmp_path / "prebuilt")
-    errors = mod.validate(persona)
-    assert any("does not reference" in e and "question" in e for e in errors), errors
-
-
-def test_unknown_slug_not_in_shared_key_map(tmp_path, monkeypatch):
-    mod = _load_module()
-    _write_master(tmp_path, "milarepa")  # real prebuilt but not in SHARED_KEY_MAP
-    persona = _persona_dir(tmp_path)
-    (persona / "shared.yaml").write_text(SHARED_YAML_HUINENG, encoding="utf-8")
-    (persona / "milarepa.promptfooconfig.yaml").write_text(
-        VALID_CONFIG_BODY, encoding="utf-8"
-    )
-    monkeypatch.setattr(mod, "PREBUILT_DIR", tmp_path / "prebuilt")
-    errors = mod.validate(persona)
-    assert any("SHARED_KEY_MAP" in e for e in errors), errors

package/scripts/tests/test_validate_workflow.py

@@ -1,284 +0,0 @@
-"""Structural and behavior checks for the repository validation workflow."""
-
-from __future__ import annotations
-
-import os
-import subprocess
-from pathlib import Path
-
-import pytest
-import yaml
-
-
-ROOT = Path(__file__).resolve().parents[2]
-WORKFLOW_PATH = ROOT / ".github" / "workflows" / "validate-and-test.yml"
-WORKFLOW_TEXT = WORKFLOW_PATH.read_text(encoding="utf-8")
-
-
-def _load_workflow_text(text: str) -> dict:
-    workflow = yaml.safe_load(text)
-    assert isinstance(workflow, dict)
-    assert isinstance(workflow.get("jobs"), dict)
-    return workflow
-
-
-def _job(workflow: dict, job_name: str) -> dict:
-    job = workflow["jobs"].get(job_name)
-    assert isinstance(job, dict), f"missing workflow job: {job_name}"
-    assert isinstance(job.get("steps"), list), f"job has no steps: {job_name}"
-    return job
-
-
-def _step(workflow: dict, job_name: str, step_name: str) -> dict:
-    matches = [
-        step
-        for step in _job(workflow, job_name)["steps"]
-        if step.get("name") == step_name
-    ]
-    assert len(matches) == 1, f"expected one {job_name}/{step_name} step, got {len(matches)}"
-    return matches[0]
-
-
-def _assert_hard(step: dict) -> None:
-    assert step.get("continue-on-error") not in (True, "true")
-
-
-WORKFLOW = _load_workflow_text(WORKFLOW_TEXT)
-
-
-def test_free_text_tokens_cannot_substitute_for_workflow_structure():
-    fake = _load_workflow_text(
-        """\
-name: fake
-jobs:
-  validate:
-    steps:
-      - name: comments only
-        run: echo 'python -m pytest tests/ scripts/tests/ -v'
-"""
-    )
-
-    with pytest.raises(AssertionError, match="Run Python tests"):
-        _step(fake, "validate", "Run Python tests")
-
-
-def test_workflow_has_no_softened_steps():
-    for job_name, job in WORKFLOW["jobs"].items():
-        for step in job.get("steps", []):
-            assert step.get("continue-on-error") not in (True, "true"), (
-                f"softened workflow step in {job_name}: {step.get('name', step.get('uses'))}"
-            )
-
-
-@pytest.mark.parametrize(
-    ("step_name", "command"),
-    [
-        ("Run Python tests", "python -m pytest tests/ scripts/tests/ -v"),
-        ("Validate citation contracts", "python scripts/validate-citation-contract.py"),
-        (
-            "Validate lore_triggers content (v0.8 — hard gate)",
-            "python scripts/validate-lore-triggers-content.py --strict",
-        ),
-    ],
-)
-def test_validate_job_contains_hard_gate_commands(step_name: str, command: str):
-    step = _step(WORKFLOW, "validate", step_name)
-    assert step.get("run") == command
-    _assert_hard(step)
-
-
-@pytest.mark.parametrize(
-    ("step_name", "command"),
-    [
-        ("Check desktop formatting", "cargo fmt --manifest-path desktop/Cargo.toml -- --check"),
-        (
-            "Lint desktop app",
-            "cargo clippy --locked --manifest-path desktop/Cargo.toml "
-            "--all-targets -- -D warnings",
-        ),
-        ("Test desktop app", "cargo test --locked --manifest-path desktop/Cargo.toml"),
-        ("Build desktop app", "cargo build --locked --manifest-path desktop/Cargo.toml"),
-    ],
-)
-def test_desktop_job_contains_hard_gate_commands(step_name: str, command: str):
-    step = _step(WORKFLOW, "desktop-rust", step_name)
-    assert step.get("run") == command
-    _assert_hard(step)
-
-
-def test_desktop_quality_gates_run_before_tests_and_build():
-    step_names = [step.get("name") for step in _job(WORKFLOW, "desktop-rust")["steps"]]
-    assert step_names.index("Check desktop formatting") < step_names.index("Lint desktop app")
-    assert step_names.index("Lint desktop app") < step_names.index("Test desktop app")
-    assert step_names.index("Test desktop app") < step_names.index("Build desktop app")
-
-
-def test_windows_cli_job_installs_the_generator_python_runtime():
-    job = _job(WORKFLOW, "cli-windows")
-    uses = [step.get("uses", "") for step in job["steps"]]
-    assert any(use.startswith("actions/setup-python@") for use in uses)
-    install = _step(WORKFLOW, "cli-windows", "Install generator dependencies")
-    assert install.get("run") == "python -m pip install -r requirements.txt"
-    _assert_hard(install)
-
-
-def test_python39_job_compiles_and_runs_the_four_generator_cli_steps():
-    job = _job(WORKFLOW, "python-compat")
-    setup = next(
-        step
-        for step in job["steps"]
-        if str(step.get("uses", "")).startswith("actions/setup-python@")
-    )
-    assert setup.get("with", {}).get("python-version") == "3.9"
-    smoke = _step(WORKFLOW, "python-compat", "Run Python 3.9 generator smoke")
-    command = smoke["run"]
-    assert "python -m compileall -q tools scripts" in command
-    assert "sutra_collector.py" in command
-    assert "verify_sources.py --check-links" in command
-    assert "master_builder.py" in command
-    assert "verify_sources.py --final-check" in command
-    _assert_hard(smoke)
-
-
-def _covered(target: str, patterns: set[str]) -> bool:
-    """Whether a push-paths pattern set actually triggers on `target`.
-
-    The contract is "editing this file runs CI", not "this literal string
-    appears in the list" — so a broader glob that consolidates several entries
-    (docs/PRD.md + docs/v1-framework-roadmap.md -> docs/**) still satisfies it,
-    while dropping the coverage entirely still fails.
-    """
-    if target in patterns:
-        return True
-    return any(
-        target.startswith(pattern[: -len("**")])
-        for pattern in patterns
-        if pattern.endswith("**")
-    )
-
-
-def test_push_paths_include_distribution_and_generator_runtime():
-    triggers = WORKFLOW.get("on", WORKFLOW.get(True))
-    assert isinstance(triggers, dict)
-    paths = set(triggers["push"]["paths"])
-    required = {
-        "skill-catalog.json",
-        "SKILL.md",
-        "references/**",
-        "ETHICS.md",
-        "requirements.txt",
-        "README.md",
-        "README_EN.md",
-        "CONTRIBUTING.md",
-        "CHANGELOG.md",
-        "docs/PRD.md",
-        "docs/v1-framework-roadmap.md",
-        "masters/**",
-        ".claude-plugin/**",
-        ".cursor-plugin/**",
-        "gemini-extension.json",
-        ".github/PULL_REQUEST_TEMPLATE.md",
-        ".github/ISSUE_TEMPLATE/**",
-    }
-    uncovered = sorted(t for t in required if not _covered(t, paths))
-    assert not uncovered, f"push trigger does not cover: {uncovered}"
-
-
-def test_pick_step_uses_checked_selector_without_fixed_roster():
-    checkout = next(
-        step
-        for step in _job(WORKFLOW, "fidelity-smoke")["steps"]
-        if str(step.get("uses", "")).startswith("actions/checkout@")
-    )
-    assert checkout.get("with", {}).get("fetch-depth") == 0
-    step = _step(WORKFLOW, "fidelity-smoke", "Pick smoke target")
-    script = step["run"]
-    assert "if ! CHANGED=$(python scripts/select-fidelity-smoke.py" in script
-    assert "if ! DIFF_OUTPUT=$(git diff --name-only" in script
-    assert "CHANGED_CANDIDATES" in script
-    assert "--prebuilt prebuilt" in script
-    assert '--day-of-year "$(date +%j)"' in script
-    assert "MASTERS=(" not in script
-    _assert_hard(step)
-
-
-def test_smoke_selector_producer_failure_is_not_masked(tmp_path: Path):
-    pick_step = _step(WORKFLOW, "fidelity-smoke", "Pick smoke target")
-    script = pick_step["run"].replace("${{ github.base_ref || 'main' }}", "main")
-
-    bin_dir = tmp_path / "bin"
-    bin_dir.mkdir()
-    fake_python = bin_dir / "python"
-    fake_python.write_text(
-        "#!/bin/sh\nprintf 'master-partial\\n'\nexit 7\n",
-        encoding="utf-8",
-    )
-    fake_python.chmod(0o755)
-    output_path = tmp_path / "github-output"
-    env = os.environ.copy()
-    env["PATH"] = f"{bin_dir}:{env['PATH']}"
-    env["GITHUB_OUTPUT"] = str(output_path)
-
-    result = subprocess.run(
-        ["bash", "-e", "-o", "pipefail", "-c", script],
-        cwd=ROOT,
-        env=env,
-        text=True,
-        capture_output=True,
-        check=False,
-    )
-
-    assert result.returncode != 0
-    assert not output_path.exists()
-
-
-def test_smoke_git_diff_failure_is_not_masked(tmp_path: Path):
-    pick_step = _step(WORKFLOW, "fidelity-smoke", "Pick smoke target")
-    script = pick_step["run"].replace("${{ github.base_ref || 'main' }}", "main")
-
-    bin_dir = tmp_path / "bin"
-    bin_dir.mkdir()
-    fake_git = bin_dir / "git"
-    fake_git.write_text("#!/bin/sh\nexit 7\n", encoding="utf-8")
-    fake_git.chmod(0o755)
-    fake_python = bin_dir / "python"
-    fake_python.write_text(
-        "#!/bin/sh\nprintf 'master-alpha\\n'\n",
-        encoding="utf-8",
-    )
-    fake_python.chmod(0o755)
-    output_path = tmp_path / "github-output"
-    env = os.environ.copy()
-    env["PATH"] = f"{bin_dir}:{env['PATH']}"
-    env["GITHUB_OUTPUT"] = str(output_path)
-
-    result = subprocess.run(
-        ["bash", "-e", "-o", "pipefail", "-c", script],
-        cwd=ROOT,
-        env=env,
-        text=True,
-        capture_output=True,
-        check=False,
-    )
-
-    assert result.returncode != 0
-    assert "git diff failed" in result.stdout
-    assert not output_path.exists()
-
-
-@pytest.mark.parametrize(
-    ("job_name", "step_name"),
-    [
-        ("fidelity-smoke", "Run fidelity smoke"),
-        ("fidelity-full", "Run fidelity tests"),
-    ],
-)
-def test_each_fidelity_no_key_branch_records_step_summary(job_name: str, step_name: str):
-    job = _job(WORKFLOW, job_name)
-    assert job.get("needs") == "validate"
-    step = _step(WORKFLOW, job_name, step_name)
-    script = step["run"]
-    assert 'if [ -z "${ANTHROPIC_API_KEY:-}" ]; then' in script
-    assert script.count('echo "### Fidelity grading skipped"') == 1
-    assert '} >> "$GITHUB_STEP_SUMMARY"' in script
-    _assert_hard(step)