@rpamis/comet 0.3.9 → 0.4.0-beta.10

package/eval/local/tests/scaffold/test_treatments.py

@@ -0,0 +1,353 @@
+"""Unit tests for comet eval treatment loading."""
+
+from pathlib import Path
+
+import pytest
+
+from scaffold.python.treatments import (
+    TreatmentConfig,
+    build_treatment_skills,
+    list_treatments,
+    load_treatments,
+    load_treatments_yaml,
+)
+from scaffold.python.paths import EVAL_ROOT, get_skills_dir
+
+
+BASIC_TREATMENT_YAML = """
+_common_section: &common |
+  Shared guidance.
+
+CONTROL:
+  description: "No skills baseline"
+  skills: []
+
+INLINE_COMET:
+  description: "Inline skill treatment"
+  claude_md: "Use the comet workflow."
+  skills:
+    - skill: inline_comet
+      name: inline-comet
+      content: *common
+  noise_tasks:
+    - DISTRACTOR
+"""
+
+
+@pytest.fixture
+def treatment_file(tmp_path: Path) -> Path:
+    path = tmp_path / "treatments.yaml"
+    path.write_text(BASIC_TREATMENT_YAML)
+    return path
+
+
+def test_load_treatments_yaml_skips_anchor_keys(treatment_file: Path):
+    treatments = load_treatments_yaml(treatment_file)
+
+    assert list(treatments) == ["CONTROL", "INLINE_COMET"]
+    assert treatments["INLINE_COMET"].noise_tasks == ["DISTRACTOR"]
+    assert treatments["INLINE_COMET"].claude_md == "Use the comet workflow."
+
+
+def test_build_treatment_skills_accepts_inline_content_and_generated_names():
+    skills = build_treatment_skills(
+        [
+            {"skill": "snake_case_skill", "content": "Inline content"},
+            {"skill": "ignored", "name": "explicit-name", "content": "Explicit content"},
+        ]
+    )
+
+    assert "snake-case-skill" in skills
+    assert "snake_case_skill" not in skills
+    assert skills["snake-case-skill"]["sections"] == ["Inline content"]
+    assert skills["explicit-name"]["sections"] == ["Explicit content"]
+
+
+def test_build_treatment_skills_accepts_path_skill_source(tmp_path: Path):
+    skill_dir = tmp_path / "local-skill"
+    skill_dir.mkdir()
+    (skill_dir / "SKILL.md").write_text(
+        "---\nname: local-skill\ndescription: Local test skill.\n---\n\nUse this skill.",
+        encoding="utf-8",
+    )
+    scripts_dir = skill_dir / "scripts"
+    scripts_dir.mkdir()
+    (scripts_dir / "helper.py").write_text("print('ok')", encoding="utf-8")
+
+    skills = build_treatment_skills(
+        [{"name": "local-skill", "source": "path", "path": str(skill_dir)}]
+    )
+
+    assert skills["local-skill"]["sections"] == [
+        "---\nname: local-skill\ndescription: Local test skill.\n---\n\nUse this skill."
+    ]
+    assert skills["local-skill"]["scripts_dir"] == scripts_dir
+    assert skills["local-skill"]["script_filter"] is None
+    assert skills["local-skill"]["source"]["source_type"] == "path"
+    assert skills["local-skill"]["source"]["hash"].startswith("sha256:")
+
+
+def test_build_treatment_skills_rejects_path_without_skill_md(tmp_path: Path):
+    skill_dir = tmp_path / "broken-skill"
+    skill_dir.mkdir()
+
+    with pytest.raises(FileNotFoundError, match="SKILL.md"):
+        build_treatment_skills([{"name": "broken-skill", "source": "path", "path": str(skill_dir)}])
+
+
+def test_load_treatments_keeps_comet_core_categories_only():
+    treatments = load_treatments()
+
+    assert set(treatments) == {
+        "CONTROL",
+        "COMET_CLASSIC_DOCS_LAYOUT",
+        "COMET_CLASSIC_LEGACY_LAYOUT",
+        "COMET_FULL_040_BETA",
+        "COMET_FULL_039",
+        "COMET_NATIVE_BATCH",
+        "COMET_NATIVE_PHASE1",
+        "COMET_NATIVE_SEQUENTIAL",
+    }
+    assert all(isinstance(treatment, TreatmentConfig) for treatment in treatments.values())
+
+
+def _benchmark_child_names(*parts: str) -> set[str]:
+    root = get_skills_dir() / "benchmarks"
+    return {path.name for path in (root.joinpath(*parts)).iterdir() if path.is_dir()}
+
+
+def test_comet_full_040_beta_includes_openspec_and_superpowers_dependencies():
+    treatment = load_treatments()["COMET_FULL_040_BETA"]
+    names = {skill["name"] for skill in treatment.skills}
+
+    assert names.issuperset(
+        {
+            "comet",
+            "comet-open",
+            "comet-design",
+            "comet-build",
+            "comet-verify",
+            "comet-archive",
+            "comet-hotfix",
+            "comet-tweak",
+        }
+    )
+    assert names.issuperset(_benchmark_child_names("dependency", "openspec"))
+    assert names.issuperset(_benchmark_child_names("dependency", "superpowers"))
+
+
+@pytest.mark.parametrize(
+    ("treatment_name", "layout"),
+    [
+        ("COMET_CLASSIC_DOCS_LAYOUT", "docs"),
+        ("COMET_CLASSIC_LEGACY_LAYOUT", "legacy"),
+    ],
+)
+def test_current_comet_classic_layouts_use_current_skills_and_dependency_benchmarks(
+    treatment_name: str,
+    layout: str,
+):
+    treatment = load_treatments()[treatment_name]
+    names = {skill["name"] for skill in treatment.skills}
+
+    assert names.issuperset(
+        {
+            "comet",
+            "comet-classic",
+            "comet-open",
+            "comet-design",
+            "comet-build",
+            "comet-verify",
+            "comet-archive",
+            "comet-hotfix",
+            "comet-tweak",
+        }
+    )
+    assert names.issuperset(
+        {
+            "openspec-explore",
+            "openspec-new-change",
+            "openspec-continue-change",
+            "openspec-apply-change",
+            "openspec-sync-specs",
+            "openspec-archive-change",
+            "openspec-verify-change",
+            "brainstorming",
+            "writing-plans",
+            "executing-plans",
+            "subagent-driven-development",
+            "test-driven-development",
+            "systematic-debugging",
+            "verification-before-completion",
+            "using-git-worktrees",
+            "requesting-code-review",
+            "receiving-code-review",
+            "finishing-a-development-branch",
+        }
+    )
+    assert f"classic.artifact_layout `{layout}`" in treatment.claude_md
+    assert "comet classic openspec --" in treatment.claude_md
+
+    skills = build_treatment_skills(treatment.skills)
+    assert skills["comet"]["source_dir"] == EVAL_ROOT.parent / "assets" / "skills" / "comet"
+    assert skills["comet-classic"]["source_dir"] == (
+        EVAL_ROOT.parent / "assets" / "skills" / "comet-classic"
+    )
+
+
+def test_comet_full_039_includes_same_dependency_snapshot():
+    treatment = load_treatments()["COMET_FULL_039"]
+    names = {skill["name"] for skill in treatment.skills}
+
+    assert names.issuperset(
+        {
+            "comet",
+            "comet-open",
+            "comet-design",
+            "comet-build",
+            "comet-verify",
+            "comet-archive",
+            "comet-hotfix",
+            "comet-tweak",
+        }
+    )
+    assert names.issuperset(_benchmark_child_names("dependency", "openspec"))
+    assert names.issuperset(_benchmark_child_names("dependency", "superpowers"))
+
+
+def test_comet_native_phase1_is_self_contained():
+    treatment = load_treatments()["COMET_NATIVE_PHASE1"]
+
+    assert {skill["name"] for skill in treatment.skills} == {"comet-native"}
+    assert treatment.skills[0]["source"] == "path"
+    assert "assets/skills/comet-native" in treatment.skills[0]["path"]
+    assert treatment.skills[0]["profile"] == "generic"
+    assert treatment.skills[0]["required_skills"] == ["comet-native"]
+    assert treatment.skills[0]["require_skill_invocation"] is True
+    assert "Shape, Build, Verify, and Archive" in treatment.claude_md
+    assert "OpenSpec" in treatment.claude_md
+    normalized_claude_md = " ".join(treatment.claude_md.split())
+    assert "continue automatically while no user decision is unresolved" in normalized_claude_md
+    assert "explicit execution boundary" in normalized_claude_md
+    assert "takes priority over automatic progression" in normalized_claude_md
+    assert "material user decision" not in normalized_claude_md
+
+    skills = build_treatment_skills(treatment.skills)
+    assert set(skills) == {"comet-native"}
+    assert skills["comet-native"]["scripts_dir"].name == "scripts"
+    assert skills["comet-native"]["source_dir"].name == "comet-native"
+
+
+@pytest.mark.parametrize(
+    ("name", "mode"),
+    [
+        ("COMET_NATIVE_SEQUENTIAL", "sequential"),
+        ("COMET_NATIVE_BATCH", "batch"),
+    ],
+)
+def test_comet_native_clarification_treatments_are_self_contained(name: str, mode: str):
+    treatment = load_treatments()[name]
+
+    assert {skill["name"] for skill in treatment.skills} == {"comet-native"}
+    assert treatment.skills[0]["source"] == "path"
+    assert treatment.skills[0]["profile"] == "generic"
+    assert f"clarification_mode `{mode}`" in treatment.claude_md
+    assert "Do not change the configured clarification mode" in treatment.claude_md
+
+
+def test_comet_treatments_point_at_versioned_comet_snapshots():
+    COMET_FULL_040_BETA = load_treatments()["COMET_FULL_040_BETA"]
+    comet_039 = load_treatments()["COMET_FULL_039"]
+
+    assert {
+        skill["skill"] for skill in COMET_FULL_040_BETA.skills if skill["name"].startswith("comet")
+    } == {
+        "040-beta/comet",
+        "040-beta/comet-open",
+        "040-beta/comet-design",
+        "040-beta/comet-build",
+        "040-beta/comet-verify",
+        "040-beta/comet-archive",
+        "040-beta/comet-hotfix",
+        "040-beta/comet-tweak",
+    }
+    assert {skill["skill"] for skill in comet_039.skills if skill["name"].startswith("comet")} == {
+        "039-release/comet-classic-039",
+        "039-release/comet-classic-039-open",
+        "039-release/comet-classic-039-design",
+        "039-release/comet-classic-039-build",
+        "039-release/comet-classic-039-verify",
+        "039-release/comet-classic-039-archive",
+        "039-release/comet-classic-039-hotfix",
+        "039-release/comet-classic-039-tweak",
+    }
+
+
+def test_comet_treatment_dependency_snapshots_are_loadable():
+    for treatment_name in ["COMET_FULL_040_BETA", "COMET_FULL_039"]:
+        treatment = load_treatments()[treatment_name]
+
+        skills = build_treatment_skills(treatment.skills)
+
+        assert {
+            "openspec-new-change",
+            "openspec-apply-change",
+            "openspec-verify-change",
+            "openspec-archive-change",
+            "brainstorming",
+            "executing-plans",
+            "subagent-driven-development",
+            "test-driven-development",
+            "systematic-debugging",
+            "verification-before-completion",
+            "requesting-code-review",
+            "writing-skills",
+        }.issubset(skills)
+        assert skills["comet"]["scripts_dir"].name == "scripts"
+        assert skills["comet"]["source_dir"].name in {"comet", "comet-classic-039"}
+        assert skills["openspec-new-change"]["sections"]
+        assert skills["executing-plans"]["sections"]
+
+
+def test_comet_treatment_main_skill_sources_include_rules_and_hooks():
+    skills_040 = build_treatment_skills(load_treatments()["COMET_FULL_040_BETA"].skills)
+    comet_040 = skills_040["comet"]
+    comet_040_text = "\n\n".join(comet_040["sections"])
+    assert "Skill 工具加载对应的 Comet 子 Skill" in comet_040_text
+    assert "OpenSpec 或 Superpowers 技能" in comet_040_text
+    assert (comet_040["source_dir"] / "rules" / "comet-phase-guard.md").exists()
+    assert (comet_040["scripts_dir"] / "comet-hook-guard.mjs").exists()
+    assert (comet_040["source_dir"] / "runtime" / "classic" / "skill.yaml").exists()
+
+    skills_039 = build_treatment_skills(load_treatments()["COMET_FULL_039"].skills)
+    comet_039 = skills_039["comet"]
+    assert (comet_039["source_dir"] / "rules" / "comet-phase-guard.md").exists()
+    assert (comet_039["scripts_dir"] / "comet-hook-guard.sh").exists()
+
+
+def test_comet_full_040_beta_dependency_paths_are_loadable():
+    treatment = load_treatments()["COMET_FULL_040_BETA"]
+
+    skills = build_treatment_skills(treatment.skills)
+
+    assert {
+        "comet",
+        "openspec-new-change",
+        "brainstorming",
+        "executing-plans",
+    }.issubset(skills)
+    assert "040-beta" in str(skills["comet"]["scripts_dir"])
+    assert skills["openspec-new-change"]["sections"][0].startswith("---")
+
+
+def test_list_treatments_is_sorted_for_stable_cli_output():
+    assert list_treatments() == [
+        "COMET_CLASSIC_DOCS_LAYOUT",
+        "COMET_CLASSIC_LEGACY_LAYOUT",
+        "COMET_FULL_039",
+        "COMET_FULL_040_BETA",
+        "COMET_NATIVE_BATCH",
+        "COMET_NATIVE_PHASE1",
+        "COMET_NATIVE_SEQUENTIAL",
+        "CONTROL",
+    ]