@rpamis/comet 0.4.0-beta.3 → 0.4.0-beta.5

package/eval/local/tests/scaffold/test_regression_check.py

@@ -0,0 +1,19 @@
+"""Unit tests for the eval regression gate wrapper."""
+
+import subprocess
+import sys
+from pathlib import Path
+
+
+def test_regression_check_help_imports_from_eval_working_directory():
+    eval_root = Path(__file__).resolve().parents[3]
+
+    result = subprocess.run(
+        [sys.executable, "local/scripts/regression_check.py", "--help"],
+        cwd=eval_root,
+        capture_output=True,
+        text=True,
+    )
+
+    assert result.returncode == 0, result.stderr
+    assert "--count" in result.stdout

package/eval/local/tests/scaffold/test_report_style_demo_charts.py

@@ -0,0 +1,45 @@
+from pathlib import Path
+import re
+
+from local.scripts.generate_report_style_demo_charts import generate_charts
+
+
+def test_generate_report_style_demo_charts_writes_bilingual_svgs(tmp_path: Path):
+    written = generate_charts(tmp_path)
+
+    expected = {
+        "rubric_delta.zh.svg",
+        "rubric_delta.en.svg",
+        "quality_cost.zh.svg",
+        "quality_cost.en.svg",
+        "task_outcomes.zh.svg",
+        "task_outcomes.en.svg",
+    }
+    assert {path.name for path in written} == expected
+
+    zh_rubric = (tmp_path / "rubric_delta.zh.svg").read_text(encoding="utf-8")
+    en_rubric = (tmp_path / "rubric_delta.en.svg").read_text(encoding="utf-8")
+    assert "图 1. Rubric 维度相对 0.3.9 的变化" in zh_rubric
+    assert "Figure 1. Rubric dimension deltas relative to 0.3.9" in en_rubric
+    assert "Times New Roman" in zh_rubric
+    assert "SimSun" in zh_rubric
+
+    circle_x_values = [
+        float(match)
+        for match in re.findall(r"<circle cx=\"([0-9.]+)\" cy=\"[0-9.]+\" r=\"5.2\"", zh_rubric)
+    ]
+    assert circle_x_values
+    assert max(circle_x_values) < 760
+    assert '<text class="value" x="840"' in zh_rubric
+
+    zh_outcomes = (tmp_path / "task_outcomes.zh.svg").read_text(encoding="utf-8")
+    subtitle_y = int(re.search(r'<text class="subtitle" x="32" y="([0-9]+)"', zh_outcomes).group(1))
+    column_header_y_values = [
+        int(match)
+        for match in re.findall(
+            r'<text class="label" x="[0-9.]+" y="([0-9]+)" text-anchor="middle">',
+            zh_outcomes,
+        )
+    ]
+    assert column_header_y_values
+    assert min(column_header_y_values) - subtitle_y >= 28

package/eval/local/tests/scaffold/test_sample_quality.py

@@ -0,0 +1,211 @@
+"""Unit tests for eval sample-quality classification."""
+
+import json
+from pathlib import Path
+
+from scaffold.python.sample_quality import (
+    include_in_analysis,
+    infer_sample_quality,
+    quality_from_report,
+    sample_quality_dict,
+)
+
+
+def test_api_timeout_is_excluded_from_analysis():
+    quality = infer_sample_quality(
+        checks_failed=["Validation failed: no result"],
+        stdout='{"type":"assistant","message":{"content":[]}}\n',
+        stderr="Error: API request timed out after 600s",
+        returncode=1,
+    )
+
+    assert quality.status == "excluded"
+    assert quality.reason_code == "api_timeout"
+    assert quality.include_in_analysis is False
+    assert "API request timed out" in " ".join(quality.evidence)
+
+
+def test_rate_limit_is_excluded_from_analysis():
+    quality = infer_sample_quality(stderr="429 rate_limit_error: insufficient quota")
+
+    assert quality.status == "excluded"
+    assert quality.reason_code == "rate_limited"
+    assert quality.include_in_analysis is False
+
+
+def test_container_failure_is_excluded_from_analysis():
+    quality = infer_sample_quality(stderr="ERROR: Docker daemon not running")
+
+    assert quality.status == "excluded"
+    assert quality.reason_code == "container_failure"
+    assert quality.include_in_analysis is False
+
+
+def test_completed_run_with_api_timeout_in_failed_check_stays_in_analysis():
+    quality = infer_sample_quality(
+        events={
+            "duration_seconds": 42,
+            "total_tokens": 1000,
+            "total_cost_usd": 0.12,
+        },
+        checks_failed=["validator failed: expected graceful API timeout handling"],
+        stdout=json.dumps({"type": "result", "duration_ms": 42000}) + "\n",
+        returncode=0,
+    )
+
+    assert quality.status == "flagged"
+    assert quality.reason_code == "completed_run_mentions_outer_failure"
+    assert quality.include_in_analysis is True
+
+
+def test_completed_run_with_network_wording_in_failed_check_stays_in_analysis():
+    quality = infer_sample_quality(
+        events={
+            "duration_seconds": 42,
+            "total_tokens": 1000,
+            "total_cost_usd": 0.12,
+        },
+        checks_failed=["validator failed: summary should mention network retry guidance"],
+        stdout=json.dumps({"type": "result", "duration_ms": 42000}) + "\n",
+        returncode=0,
+    )
+
+    assert quality.status == "flagged"
+    assert quality.reason_code == "completed_run_mentions_outer_failure"
+    assert quality.include_in_analysis is True
+
+
+def test_completed_run_with_container_wording_in_logs_stays_in_analysis():
+    quality = infer_sample_quality(
+        events={
+            "duration_seconds": 42,
+            "total_tokens": 1000,
+            "total_cost_usd": 0.12,
+        },
+        checks_failed=["validator failed: expected report artifact"],
+        stdout=json.dumps({"type": "result", "duration_ms": 42000}) + "\n",
+        stderr="Task output discussed why a Docker daemon not running error should be surfaced.",
+        returncode=0,
+    )
+
+    assert quality.status == "flagged"
+    assert quality.reason_code == "completed_run_mentions_outer_failure"
+    assert quality.include_in_analysis is True
+
+
+def test_validator_failure_with_observable_run_is_included():
+    quality = infer_sample_quality(
+        events={
+            "duration_seconds": 42,
+            "total_tokens": 1000,
+            "total_cost_usd": 0.12,
+            "skills_invoked": ["comet"],
+            "artifact_references": {"report": "reports/demo.json"},
+        },
+        checks_failed=["validator failed: output file missing"],
+        stdout=json.dumps({"type": "result", "duration_ms": 42000}) + "\n",
+        returncode=0,
+    )
+
+    assert quality.status == "included"
+    assert quality.reason_code == "valid_signal"
+    assert quality.include_in_analysis is True
+
+
+def test_harness_attribution_without_hard_noise_is_flagged():
+    quality = infer_sample_quality(
+        events={"duration_seconds": 10, "skills_invoked": []},
+        checks_failed=["Required skill not invoked: comet"],
+        failure_attribution=[
+            {
+                "bucket": "harness",
+                "check": "Required skill not invoked: comet",
+                "reason": "target Skill was never invoked",
+            }
+        ],
+        stdout=json.dumps({"type": "result", "duration_ms": 10000}) + "\n",
+        returncode=0,
+    )
+
+    assert quality.status == "flagged"
+    assert quality.reason_code == "harness_trigger_suspect"
+    assert quality.include_in_analysis is True
+
+
+def test_existing_sample_quality_is_respected():
+    report = {
+        "sample_quality": {
+            "status": "excluded",
+            "reason_code": "api_timeout",
+            "reason": "API timeout",
+            "include_in_analysis": False,
+            "confidence": "high",
+            "evidence": ["stderr timeout"],
+        }
+    }
+
+    quality = quality_from_report(report)
+
+    assert quality.status == "excluded"
+    assert quality.reason_code == "api_timeout"
+    assert include_in_analysis(report) is False
+    assert sample_quality_dict(report)["include_in_analysis"] is False
+
+
+def test_sample_quality_in_events_summary_is_respected():
+    report = {
+        "events_summary": {
+            "sample_quality": {
+                "status": "excluded",
+                "reason_code": "rate_limited",
+                "reason": "quota exceeded",
+                "include_in_analysis": False,
+                "confidence": "high",
+                "evidence": ["events summary"],
+            }
+        }
+    }
+
+    quality = quality_from_report(report)
+
+    assert quality.status == "excluded"
+    assert quality.reason_code == "rate_limited"
+    assert include_in_analysis(report) is False
+    assert sample_quality_dict(report)["status"] == "excluded"
+
+
+def test_plain_build_failed_with_observable_result_is_included():
+    quality = infer_sample_quality(
+        events={
+            "duration_seconds": 10,
+            "total_tokens": 1234,
+            "total_cost_usd": 0.01,
+        },
+        stdout='{"type":"result","duration_ms": 1000}\n',
+        stderr="build failed",
+        checks_failed=[],
+        returncode=0,
+    )
+
+    assert quality.status == "included"
+    assert quality.reason_code == "valid_signal"
+    assert quality.include_in_analysis is True
+
+
+def test_legacy_report_reads_raw_stderr_reference(tmp_path: Path):
+    stderr = tmp_path / "raw" / "workflow_rep1_stderr.txt"
+    stderr.parent.mkdir()
+    stderr.write_text("Timeout after 600s", encoding="utf-8")
+    report = {
+        "checks_passed": [],
+        "checks_failed": ["no result"],
+        "events_summary": {
+            "artifact_references": {"raw_stderr": str(stderr)},
+        },
+    }
+
+    quality = quality_from_report(report)
+
+    assert quality.status == "excluded"
+    assert quality.reason_code == "runner_timeout"
+    assert quality.include_in_analysis is False

package/eval/local/tests/scaffold/test_tasks.py

@@ -0,0 +1,168 @@
+"""Unit tests for the comet eval task loader."""
+
+from pathlib import Path
+
+import pytest
+import yaml
+
+from scaffold.python.paths import get_tasks_dir
+from scaffold.python.tasks import list_tasks, load_task
+
+
+BASIC_TASK_TOML = """
+[metadata]
+name = "test-basic"
+description = "A basic test task"
+difficulty = "easy"
+category = "testing"
+tags = ["test", "basic"]
+default_treatments = ["CONTROL", "COMET_FULL_040_BETA"]
+
+[template]
+required = ["run_id"]
+
+[environment]
+description = "Test environment"
+dockerfile = "environment/Dockerfile"
+timeout_sec = 300
+
+[validation]
+test_scripts = ["test_one.py", "test_two.py"]
+target_artifacts = ["result.json"]
+timeout = 45
+
+[setup.template_vars]
+artifact_name = "result-{run_id}.json"
+"""
+
+
+@pytest.fixture
+def mock_tasks_dir(tmp_path: Path) -> Path:
+    tasks_dir = tmp_path / "tasks"
+    task_dir = tasks_dir / "test-basic"
+    task_dir.mkdir(parents=True)
+    (task_dir / "task.toml").write_text(BASIC_TASK_TOML)
+    (task_dir / "instruction.md").write_text("Run {run_id} and write {artifact_name}.")
+
+    invalid_dir = tasks_dir / "invalid"
+    invalid_dir.mkdir()
+    (invalid_dir / "task.toml").write_text("[metadata]\nname = 'invalid'\n")
+
+    return tasks_dir
+
+
+def test_list_tasks_returns_only_complete_task_dirs(mock_tasks_dir: Path):
+    assert list_tasks(mock_tasks_dir) == ["test-basic"]
+
+
+def test_load_task_parses_validation_and_setup(mock_tasks_dir: Path):
+    task = load_task("test-basic", mock_tasks_dir)
+
+    assert task.name == "test-basic"
+    assert task.config.validation.test_scripts == ["test_one.py", "test_two.py"]
+    assert task.config.validation.target_artifacts == ["result.json"]
+    assert task.config.validation.timeout == 45
+    assert task.config.setup.template_vars == {"artifact_name": "result-{run_id}.json"}
+
+
+def test_load_task_parses_evaluation_and_interaction(mock_tasks_dir: Path):
+    task_dir = mock_tasks_dir / "test-basic"
+    task_dir.joinpath("task.toml").write_text(
+        BASIC_TASK_TOML
+        + """
+
+[evaluation]
+profile = "generic"
+required_skills = ["target-skill"]
+expected_artifacts = ["result.json"]
+require_skill_invocation = true
+
+[interaction]
+mode = "auto_user"
+max_turns = 7
+simulator_prompt = "Answer as a concise developer user."
+decision_patterns = ["confirm", "choose"]
+continue_prompt = "Please continue."
+"""
+    )
+
+    task = load_task("test-basic", mock_tasks_dir)
+
+    assert task.config.evaluation.profile == "generic"
+    assert task.config.evaluation.required_skills == ["target-skill"]
+    assert task.config.evaluation.expected_artifacts == ["result.json"]
+    assert task.config.evaluation.require_skill_invocation is True
+    assert task.config.interaction.mode == "auto_user"
+    assert task.config.interaction.max_turns == 7
+    assert task.config.interaction.simulator_prompt == "Answer as a concise developer user."
+    assert task.config.interaction.decision_patterns == ["confirm", "choose"]
+    assert task.config.interaction.continue_prompt == "Please continue."
+
+
+def test_comet_tasks_default_to_comet_workflow_profile():
+    task = load_task("comet-full-workflow")
+
+    assert task.config.evaluation.profile == "comet-workflow"
+    assert task.config.interaction.mode == "auto_user"
+
+
+def test_comet_task_prompt_requires_real_comet_invocation():
+    task = load_task("comet-fix-median")
+
+    prompt = task.render_prompt()
+
+    assert prompt.startswith("## Eval harness requirement")
+    assert "must begin by invoking the `/comet` Skill/slash command" in prompt
+    assert "Do not simulate the Comet workflow in plain prose" in prompt
+
+
+def test_render_prompt_requires_declared_template_variables(mock_tasks_dir: Path):
+    task = load_task("test-basic", mock_tasks_dir)
+
+    with pytest.raises(KeyError, match="run_id"):
+        task.render_prompt(artifact_name="result-123.json")
+
+    prompt = task.render_prompt(run_id="123", artifact_name="result-123.json")
+    assert prompt == "Run 123 and write result-123.json."
+
+
+def test_load_task_reads_instruction_as_utf8(mock_tasks_dir: Path):
+    task_dir = mock_tasks_dir / "test-basic"
+    (task_dir / "instruction.md").write_text("Fix Bob’s task {run_id}.", encoding="utf-8")
+
+    task = load_task("test-basic", mock_tasks_dir)
+
+    assert task.render_prompt(run_id="123", artifact_name="unused") == "Fix Bob’s task 123."
+
+
+def test_comet_task_index_lists_real_tasks():
+    index_path = get_tasks_dir() / "index.yaml"
+
+    assert index_path.exists()
+
+    index = yaml.safe_load(index_path.read_text(encoding="utf-8"))
+    names = [task["name"] for task in index["tasks"]]
+    assert sorted(names) == sorted(list_tasks())
+    assert len(names) == 20
+    assert set(names) == {
+        "authoring-skill-smoke",
+        "comet-agent-memory-routing",
+        "comet-api-cache-ttl",
+        "comet-cross-file-refactor",
+        "comet-dependency-confusion",
+        "comet-fix-median",
+        "comet-framework-selection",
+        "comet-full-workflow",
+        "comet-graph-execution-review",
+        "comet-human-approval-flow",
+        "comet-layered-streaming-fix",
+        "comet-perf-dedupe",
+        "comet-persistence-threading",
+        "comet-refactor-counter",
+        "comet-robust-config",
+        "comet-noise-distractor",
+        "comet-observability-env-template",
+        "generic-skill-smoke",
+        "workflow-overlay-contract",
+        "workflow-route-conformance",
+    }

package/eval/local/tests/scaffold/test_treatments.py

@@ -0,0 +1,244 @@
+"""Unit tests for comet eval treatment loading."""
+
+from pathlib import Path
+
+import pytest
+
+from scaffold.python.treatments import (
+    TreatmentConfig,
+    build_treatment_skills,
+    list_treatments,
+    load_treatments,
+    load_treatments_yaml,
+)
+from scaffold.python.paths import get_skills_dir
+
+
+BASIC_TREATMENT_YAML = """
+_common_section: &common |
+  Shared guidance.
+
+CONTROL:
+  description: "No skills baseline"
+  skills: []
+
+INLINE_COMET:
+  description: "Inline skill treatment"
+  claude_md: "Use the comet workflow."
+  skills:
+    - skill: inline_comet
+      name: inline-comet
+      content: *common
+  noise_tasks:
+    - DISTRACTOR
+"""
+
+
+@pytest.fixture
+def treatment_file(tmp_path: Path) -> Path:
+    path = tmp_path / "treatments.yaml"
+    path.write_text(BASIC_TREATMENT_YAML)
+    return path
+
+
+def test_load_treatments_yaml_skips_anchor_keys(treatment_file: Path):
+    treatments = load_treatments_yaml(treatment_file)
+
+    assert list(treatments) == ["CONTROL", "INLINE_COMET"]
+    assert treatments["INLINE_COMET"].noise_tasks == ["DISTRACTOR"]
+    assert treatments["INLINE_COMET"].claude_md == "Use the comet workflow."
+
+
+def test_build_treatment_skills_accepts_inline_content_and_generated_names():
+    skills = build_treatment_skills(
+        [
+            {"skill": "snake_case_skill", "content": "Inline content"},
+            {"skill": "ignored", "name": "explicit-name", "content": "Explicit content"},
+        ]
+    )
+
+    assert "snake-case-skill" in skills
+    assert "snake_case_skill" not in skills
+    assert skills["snake-case-skill"]["sections"] == ["Inline content"]
+    assert skills["explicit-name"]["sections"] == ["Explicit content"]
+
+
+def test_build_treatment_skills_accepts_path_skill_source(tmp_path: Path):
+    skill_dir = tmp_path / "local-skill"
+    skill_dir.mkdir()
+    (skill_dir / "SKILL.md").write_text(
+        "---\nname: local-skill\ndescription: Local test skill.\n---\n\nUse this skill.",
+        encoding="utf-8",
+    )
+    scripts_dir = skill_dir / "scripts"
+    scripts_dir.mkdir()
+    (scripts_dir / "helper.py").write_text("print('ok')", encoding="utf-8")
+
+    skills = build_treatment_skills(
+        [{"name": "local-skill", "source": "path", "path": str(skill_dir)}]
+    )
+
+    assert skills["local-skill"]["sections"] == [
+        "---\nname: local-skill\ndescription: Local test skill.\n---\n\nUse this skill."
+    ]
+    assert skills["local-skill"]["scripts_dir"] == scripts_dir
+    assert skills["local-skill"]["script_filter"] is None
+    assert skills["local-skill"]["source"]["source_type"] == "path"
+    assert skills["local-skill"]["source"]["hash"].startswith("sha256:")
+
+
+def test_build_treatment_skills_rejects_path_without_skill_md(tmp_path: Path):
+    skill_dir = tmp_path / "broken-skill"
+    skill_dir.mkdir()
+
+    with pytest.raises(FileNotFoundError, match="SKILL.md"):
+        build_treatment_skills(
+            [{"name": "broken-skill", "source": "path", "path": str(skill_dir)}]
+        )
+
+
+def test_load_treatments_keeps_comet_core_categories_only():
+    treatments = load_treatments()
+
+    assert set(treatments) == {"CONTROL", "COMET_FULL_040_BETA", "COMET_FULL_039"}
+    assert all(isinstance(treatment, TreatmentConfig) for treatment in treatments.values())
+
+
+def _benchmark_child_names(*parts: str) -> set[str]:
+    root = get_skills_dir() / "benchmarks"
+    return {path.name for path in (root.joinpath(*parts)).iterdir() if path.is_dir()}
+
+
+def test_comet_full_040_beta_includes_openspec_and_superpowers_dependencies():
+    treatment = load_treatments()["COMET_FULL_040_BETA"]
+    names = {skill["name"] for skill in treatment.skills}
+
+    assert names.issuperset(
+        {
+            "comet",
+            "comet-open",
+            "comet-design",
+            "comet-build",
+            "comet-verify",
+            "comet-archive",
+            "comet-hotfix",
+            "comet-tweak",
+        }
+    )
+    assert names.issuperset(_benchmark_child_names("dependency", "openspec"))
+    assert names.issuperset(_benchmark_child_names("dependency", "superpowers"))
+
+
+def test_comet_full_039_includes_same_dependency_snapshot():
+    treatment = load_treatments()["COMET_FULL_039"]
+    names = {skill["name"] for skill in treatment.skills}
+
+    assert names.issuperset(
+        {
+            "comet",
+            "comet-open",
+            "comet-design",
+            "comet-build",
+            "comet-verify",
+            "comet-archive",
+            "comet-hotfix",
+            "comet-tweak",
+        }
+    )
+    assert names.issuperset(_benchmark_child_names("dependency", "openspec"))
+    assert names.issuperset(_benchmark_child_names("dependency", "superpowers"))
+
+
+def test_comet_treatments_point_at_versioned_comet_snapshots():
+    COMET_FULL_040_BETA = load_treatments()["COMET_FULL_040_BETA"]
+    comet_039 = load_treatments()["COMET_FULL_039"]
+
+    assert {
+        skill["skill"]
+        for skill in COMET_FULL_040_BETA.skills
+        if skill["name"].startswith("comet")
+    } == {
+        "040-beta/comet",
+        "040-beta/comet-open",
+        "040-beta/comet-design",
+        "040-beta/comet-build",
+        "040-beta/comet-verify",
+        "040-beta/comet-archive",
+        "040-beta/comet-hotfix",
+        "040-beta/comet-tweak",
+    }
+    assert {
+        skill["skill"]
+        for skill in comet_039.skills
+        if skill["name"].startswith("comet")
+    } == {
+        "039-release/comet-classic-039",
+        "039-release/comet-classic-039-open",
+        "039-release/comet-classic-039-design",
+        "039-release/comet-classic-039-build",
+        "039-release/comet-classic-039-verify",
+        "039-release/comet-classic-039-archive",
+        "039-release/comet-classic-039-hotfix",
+        "039-release/comet-classic-039-tweak",
+    }
+
+
+def test_comet_treatment_dependency_snapshots_are_loadable():
+    for treatment_name in ["COMET_FULL_040_BETA", "COMET_FULL_039"]:
+        treatment = load_treatments()[treatment_name]
+
+        skills = build_treatment_skills(treatment.skills)
+
+        assert {
+            "openspec-new-change",
+            "openspec-apply-change",
+            "openspec-verify-change",
+            "openspec-archive-change",
+            "brainstorming",
+            "executing-plans",
+            "subagent-driven-development",
+            "test-driven-development",
+            "systematic-debugging",
+            "verification-before-completion",
+            "requesting-code-review",
+            "writing-skills",
+        }.issubset(skills)
+        assert skills["comet"]["scripts_dir"].name == "scripts"
+        assert skills["comet"]["source_dir"].name in {"comet", "comet-classic-039"}
+        assert skills["openspec-new-change"]["sections"]
+        assert skills["executing-plans"]["sections"]
+
+
+def test_comet_treatment_main_skill_sources_include_rules_and_hooks():
+    skills_040 = build_treatment_skills(load_treatments()["COMET_FULL_040_BETA"].skills)
+    comet_040 = skills_040["comet"]
+    comet_040_text = "\n\n".join(comet_040["sections"])
+    assert "Skill 工具加载对应的 Comet 子 Skill" in comet_040_text
+    assert "OpenSpec 或 Superpowers 技能" in comet_040_text
+    assert (comet_040["source_dir"] / "rules" / "comet-phase-guard.md").exists()
+    assert (comet_040["scripts_dir"] / "comet-hook-guard.mjs").exists()
+    assert (comet_040["source_dir"] / "runtime" / "classic" / "skill.yaml").exists()
+
+    skills_039 = build_treatment_skills(load_treatments()["COMET_FULL_039"].skills)
+    comet_039 = skills_039["comet"]
+    assert (comet_039["source_dir"] / "rules" / "comet-phase-guard.md").exists()
+    assert (comet_039["scripts_dir"] / "comet-hook-guard.sh").exists()
+
+
+def test_comet_full_040_beta_dependency_paths_are_loadable():
+    treatment = load_treatments()["COMET_FULL_040_BETA"]
+
+    skills = build_treatment_skills(treatment.skills)
+
+    assert {
+        "comet",
+        "openspec-new-change",
+        "brainstorming",
+        "executing-plans",
+    }.issubset(skills)
+    assert "040-beta" in str(skills["comet"]["scripts_dir"])
+    assert skills["openspec-new-change"]["sections"][0].startswith("---")
+
+
+def test_list_treatments_is_sorted_for_stable_cli_output():
+    assert list_treatments() == ["COMET_FULL_039", "COMET_FULL_040_BETA", "CONTROL"]