@rpamis/comet 0.4.0-beta.3 → 0.4.0-beta.5

package/eval/local/tests/scaffold/test_conftest_helpers.py

@@ -0,0 +1,534 @@
+"""Tests for pytest fixture helper behavior."""
+
+import importlib.util
+import json
+import os
+from pathlib import Path
+
+import conftest
+from scaffold.python.tasks import load_task
+from scaffold.python.treatments import build_treatment_skills, load_treatments
+
+
+def test_file_lock_context_manager_allows_exclusive_writes(tmp_path: Path):
+    lock_file = tmp_path / "coordination.lock"
+    data_file = tmp_path / "coordination.txt"
+
+    with conftest.file_lock(lock_file):
+        data_file.write_text("held")
+
+    assert data_file.read_text() == "held"
+
+
+def test_unit_test_detection_handles_scaffold_and_script_paths():
+    class Config:
+        args = ["local/tests/scaffold/test_tasks.py", "-q"]
+
+    assert conftest._is_unit_tests_only(Config()) is True
+
+
+def test_unit_test_detection_keeps_task_runs_as_experiments():
+    class Config:
+        args = ["local/tests/tasks/test_tasks.py", "--task=comet-hotfix"]
+
+    assert conftest._is_unit_tests_only(Config()) is False
+
+
+def test_dynamic_treatment_config_from_skill_path(tmp_path: Path):
+    skill_dir = tmp_path / "demo-skill"
+    skill_dir.mkdir()
+    (skill_dir / "SKILL.md").write_text(
+        "---\nname: demo-skill\ndescription: Demo.\n---\n\nDemo body.",
+        encoding="utf-8",
+    )
+
+    class Config:
+        def getoption(self, name):
+            values = {
+                "--skill-path": str(skill_dir),
+                "--skill-name": "demo-skill",
+                "--profile": "generic",
+            }
+            return values.get(name)
+
+    cfg = conftest._get_dynamic_treatment_config(Config())
+
+    assert cfg.name == "DYNAMIC_SKILL"
+    assert cfg.description == "Dynamic Skill target: demo-skill"
+    assert cfg.skills == [
+        {
+            "name": "demo-skill",
+            "source": "path",
+            "path": str(skill_dir),
+            "profile": "generic",
+        }
+    ]
+
+
+def test_resolve_interaction_config_prefers_cli_mode():
+    task = load_task("comet-full-workflow")
+
+    class Config:
+        def getoption(self, name):
+            values = {
+                "--interaction-mode": "none",
+                "--max-turns": "5",
+                "--simulator-prompt": "Use CLI override.",
+            }
+            return values.get(name)
+
+    interaction = conftest._resolve_interaction_config(task, "comet-workflow", Config())
+
+    assert interaction.mode == "none"
+    assert interaction.max_turns == 5
+    assert interaction.simulator_prompt == "Use CLI override."
+
+
+def test_resolve_interaction_config_uses_profile_default_prompt():
+    task = load_task("comet-full-workflow")
+
+    class Config:
+        def getoption(self, name):
+            return {
+                "--interaction-mode": "auto_user",
+                "--max-turns": None,
+                "--simulator-prompt": None,
+            }.get(name)
+
+    interaction = conftest._resolve_interaction_config(task, "generic", Config())
+
+    assert interaction.mode == "auto_user"
+    assert interaction.max_turns == 12
+    assert interaction.simulator_prompt is not None
+
+
+def test_build_eval_claude_md_injects_comet_workflow_contract():
+    claude_md = conftest._build_eval_claude_md("comet-workflow")
+
+    assert claude_md is not None
+    assert "invoking the `/comet` Skill/slash command" in claude_md
+    assert "nested Comet stage Skill" in claude_md
+    assert "OpenSpec or Superpowers dependency Skill" in claude_md
+    assert "Do not simulate the Comet workflow" in claude_md
+    assert conftest.COMET_WORKFLOW_CLAUDE_MD_PATH.exists()
+
+
+def test_build_eval_claude_md_preserves_treatment_guidance():
+    claude_md = conftest._build_eval_claude_md("comet-workflow", "Extra guidance.")
+
+    assert claude_md is not None
+    assert "Extra guidance." in claude_md
+
+
+def test_build_eval_claude_md_skips_generic_profile_without_treatment_guidance():
+    assert conftest._build_eval_claude_md("generic") is None
+
+
+def test_setup_test_context_copies_full_skill_package_and_configures_040_hook(
+    tmp_path: Path, setup_test_context
+):
+    source_dir = tmp_path / "comet-source"
+    source_dir.mkdir()
+    (source_dir / "SKILL.md").write_text("---\nname: comet\n---\n\nOriginal.", encoding="utf-8")
+    (source_dir / "rules").mkdir()
+    (source_dir / "rules" / "comet-phase-guard.md").write_text("rule", encoding="utf-8")
+    (source_dir / "reference").mkdir()
+    (source_dir / "reference" / "scripts.md").write_text("ref", encoding="utf-8")
+    (source_dir / "runtime" / "classic").mkdir(parents=True)
+    (source_dir / "runtime" / "classic" / "skill.yaml").write_text("runtime", encoding="utf-8")
+    scripts_dir = source_dir / "scripts"
+    scripts_dir.mkdir()
+    (scripts_dir / "comet-hook-guard.mjs").write_text("process.exit(0);", encoding="utf-8")
+
+    claude_dir = tmp_path / ".claude"
+    claude_dir.mkdir()
+    (claude_dir / "settings.json").write_text(
+        json.dumps({"hooks": {"Stop": [{"hooks": [{"type": "command", "command": "stop"}]}]}}),
+        encoding="utf-8",
+    )
+
+    setup_test_context(
+        skills={
+            "comet": {
+                "sections": ["---\nname: comet\n---\n\nGenerated."],
+                "scripts_dir": scripts_dir,
+                "source_dir": source_dir,
+            }
+        }
+    )
+
+    installed = tmp_path / ".claude" / "skills" / "comet"
+    assert (installed / "rules" / "comet-phase-guard.md").exists()
+    assert (installed / "reference" / "scripts.md").exists()
+    assert (installed / "runtime" / "classic" / "skill.yaml").exists()
+    assert "Generated." in (installed / "SKILL.md").read_text(encoding="utf-8")
+
+    settings = json.loads((tmp_path / ".claude" / "settings.json").read_text(encoding="utf-8"))
+    assert "Stop" in settings["hooks"]
+    pre_tool_use = settings["hooks"]["PreToolUse"]
+    assert pre_tool_use == [
+        {
+            "matcher": "Write|Edit|MultiEdit",
+            "hooks": [
+                {
+                    "type": "command",
+                    "command": "node /workspace/.claude/skills/comet/scripts/comet-hook-guard.mjs",
+                }
+            ],
+        }
+    ]
+
+
+def test_setup_test_context_configures_039_shell_hook(tmp_path: Path, setup_test_context):
+    source_dir = tmp_path / "comet-source"
+    source_dir.mkdir()
+    (source_dir / "SKILL.md").write_text("---\nname: comet\n---\n\nBody.", encoding="utf-8")
+    scripts_dir = source_dir / "scripts"
+    scripts_dir.mkdir()
+    (scripts_dir / "comet-hook-guard.sh").write_text("#!/usr/bin/env bash\nexit 0\n", encoding="utf-8")
+
+    setup_test_context(
+        skills={
+            "comet": {
+                "sections": ["---\nname: comet\n---\n\nBody."],
+                "scripts_dir": scripts_dir,
+                "source_dir": source_dir,
+            }
+        }
+    )
+
+    settings = json.loads((tmp_path / ".claude" / "settings.json").read_text(encoding="utf-8"))
+    command = settings["hooks"]["PreToolUse"][0]["hooks"][0]["command"]
+    assert command == "bash /workspace/.claude/skills/comet/scripts/comet-hook-guard.sh"
+
+
+def test_setup_test_context_copies_dependency_skill_packages_with_scripts(
+    tmp_path: Path, setup_test_context
+):
+    skills = build_treatment_skills(load_treatments()["COMET_FULL_040_BETA"].skills)
+    setup_test_context(
+        skills={
+            name: skills[name]
+            for name in [
+                "brainstorming",
+                "subagent-driven-development",
+                "writing-skills",
+                "openspec-new-change",
+            ]
+        }
+    )
+
+    installed = tmp_path / ".claude" / "skills"
+    assert (installed / "brainstorming" / "scripts" / "helper.js").exists()
+    assert (installed / "brainstorming" / "visual-companion.md").exists()
+    assert (installed / "subagent-driven-development" / "scripts" / "review-package").exists()
+    assert (
+        installed / "subagent-driven-development" / "task-reviewer-prompt.md"
+    ).exists()
+    assert (installed / "writing-skills" / "examples" / "CLAUDE_MD_TESTING.md").exists()
+    assert (installed / "openspec-new-change" / "SKILL.md").exists()
+
+
+def test_dynamic_treatment_config_from_eval_manifest(tmp_path: Path):
+    package = tmp_path / "manifest-skill"
+    package.mkdir()
+    (package / "SKILL.md").write_text("---\nname: manifest-skill\n---\n\nBody.", encoding="utf-8")
+    stage = tmp_path / "manifest-skill-open"
+    stage.mkdir()
+    (stage / "SKILL.md").write_text("---\nname: manifest-skill-open\n---\n\nStage.", encoding="utf-8")
+    comet_dir = package / "comet"
+    comet_dir.mkdir()
+    manifest = comet_dir / "eval.yaml"
+    manifest.write_text(
+        """
+apiVersion: comet.eval/v1alpha1
+kind: SkillEvalManifest
+metadata:
+  name: manifest-skill
+  draftHash: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa
+skill:
+  name: manifest-skill
+  source: ..
+  profile: generic
+evaluation:
+  recommendedTasks:
+    - generic-skill-smoke
+    - workflow-route-conformance
+    - workflow-overlay-contract
+  baselineTreatments:
+    - CONTROL
+    - COMET_FULL_040_BETA
+  qualityGates:
+    minWeightedScore: 0.8
+    minPassAt1: 0.6
+    maxInstabilityGap: 0.4
+  requiredOutputSchemas:
+    - comet.grill-me.v1
+  expectedEvidence:
+    - node: design
+      check: augmentation:design.grill-me
+  requiredSkills:
+    - manifest-skill
+  generatedNodeSkills:
+    - manifest-skill-open
+  routeConformance:
+    task: workflow-route-conformance
+    expectedNodeOrder:
+      - open
+interaction:
+  mode: none
+""",
+        encoding="utf-8",
+    )
+
+    class Config:
+        def getoption(self, name):
+            return {"--eval-manifest": str(manifest)}.get(name)
+
+    cfg = conftest._get_dynamic_treatment_config(Config())
+
+    assert cfg.name == "DYNAMIC_SKILL"
+    assert cfg.skills[0]["name"] == "manifest-skill"
+    assert cfg.skills[0]["source"] == "path"
+    assert cfg.skills[0]["profile"] == "generic"
+    assert cfg.skills[0]["generated_node_skills"] == ["manifest-skill-open"]
+    assert cfg.skills[0]["route_conformance_expected_node_order"] == ["open"]
+    assert cfg.skills[0]["baseline_treatments"] == ["CONTROL", "COMET_FULL_040_BETA"]
+    assert cfg.skills[0]["quality_gates"] == {
+        "minWeightedScore": 0.8,
+        "minPassAt1": 0.6,
+        "maxInstabilityGap": 0.4,
+    }
+    assert cfg.skills[0]["required_output_schemas"] == ["comet.grill-me.v1"]
+    assert cfg.skills[0]["expected_evidence"] == [
+        {"node": "design", "check": "augmentation:design.grill-me"}
+    ]
+    assert cfg.skills[0]["draft_hash"] == "a" * 64
+    assert cfg.skills[1]["name"] == "manifest-skill-open"
+    assert cfg.skills[1]["path"] == str(stage.resolve())
+
+
+def test_snapshot_dynamic_skill_package_copies_package_and_node_skills(tmp_path: Path):
+    source_root = tmp_path / "source"
+    package = source_root / "manifest-skill"
+    package.mkdir(parents=True)
+    (package / "SKILL.md").write_text("---\nname: manifest-skill\n---\n\nBody.", encoding="utf-8")
+    (package / "reference").mkdir()
+    (package / "reference" / "workflow-protocol.json").write_text("{}", encoding="utf-8")
+    stage = source_root / "manifest-skill-open"
+    stage.mkdir()
+    (stage / "SKILL.md").write_text("---\nname: manifest-skill-open\n---\n\nStage.", encoding="utf-8")
+    test_dir = tmp_path / "workspace"
+    test_dir.mkdir()
+
+    relative_package = conftest._snapshot_dynamic_skill_package(
+        test_dir,
+        {
+            "path": str(package),
+            "generated_node_skills": ["manifest-skill-open"],
+        },
+    )
+
+    assert relative_package == "_eval_target_skills/manifest-skill"
+    assert (test_dir / relative_package / "SKILL.md").exists()
+    assert (test_dir / "_eval_target_skills" / "manifest-skill-open" / "SKILL.md").exists()
+
+
+def _workflow_overlay_validator():
+    validator_path = (
+        Path(__file__).parents[2]
+        / "tasks"
+        / "workflow-overlay-contract"
+        / "validation"
+        / "test_workflow_overlay_contract.py"
+    )
+    spec = importlib.util.spec_from_file_location(
+        "workflow_overlay_contract_validator",
+        validator_path,
+    )
+    validator = importlib.util.module_from_spec(spec)
+    assert spec.loader is not None
+    spec.loader.exec_module(validator)
+    return validator
+
+
+def _run_workflow_overlay_validator(workspace: Path, context: dict) -> dict:
+    (workspace / "_test_context.json").write_text(
+        json.dumps(context),
+        encoding="utf-8",
+    )
+    validator = _workflow_overlay_validator()
+    old_cwd = Path.cwd()
+    try:
+        os.chdir(workspace)
+        validator.main()
+        return json.loads((workspace / "_test_results.json").read_text(encoding="utf-8"))
+    finally:
+        os.chdir(old_cwd)
+
+
+def _write_overlay_eval_manifest(package: Path, draft_hash: str = "c" * 64) -> None:
+    comet_dir = package / "comet"
+    comet_dir.mkdir(parents=True, exist_ok=True)
+    (comet_dir / "eval.yaml").write_text(
+        """
+apiVersion: comet.eval/v1alpha1
+kind: SkillEvalManifest
+metadata:
+  name: overlay-skill
+  draftHash: {draft_hash}
+skill:
+  name: overlay-skill
+  source: ..
+  profile: authoring-skill
+evaluation:
+  recommendedTasks:
+    - workflow-overlay-contract
+    - comet-full-workflow
+    - comet-fix-median
+  baselineTreatments:
+    - CONTROL
+    - COMET_FULL_040_BETA
+  qualityGates:
+    minWeightedScore: 0.8
+    minPassAt1: 0.6
+    maxInstabilityGap: 0.4
+  requiredOutputSchemas:
+    - comet.grill-me.v1
+  expectedEvidence:
+    - node: design
+      check: augmentation:design.grill-me
+      enforcement: guarded
+    - node: design
+      check: output-schema:design.comet.grill-me.v1.challenge-summary
+      schema: comet.grill-me.v1
+      evidence: challenge-summary
+""".format(draft_hash=draft_hash),
+        encoding="utf-8",
+    )
+
+
+def _overlay_validator_context(package: Path, draft_hash: str = "c" * 64) -> dict:
+    return {
+        "skill_package_path": str(package),
+        "baseline_treatments": ["CONTROL", "COMET_FULL_040_BETA"],
+        "quality_gates": {
+            "minWeightedScore": 0.8,
+            "minPassAt1": 0.6,
+            "maxInstabilityGap": 0.4,
+        },
+        "required_output_schemas": ["comet.grill-me.v1"],
+        "expected_evidence": [
+            {
+                "node": "design",
+                "check": "augmentation:design.grill-me",
+                "enforcement": "guarded",
+            },
+            {
+                "node": "design",
+                "check": "output-schema:design.comet.grill-me.v1.challenge-summary",
+                "schema": "comet.grill-me.v1",
+                "evidence": "challenge-summary",
+            },
+        ],
+        "draft_hash": draft_hash,
+    }
+
+
+def test_workflow_overlay_contract_validator_rejects_missing_contract_files(tmp_path: Path):
+    package = tmp_path / "overlay-skill"
+    package.mkdir()
+    _write_overlay_eval_manifest(package)
+
+    workspace = tmp_path / "workspace"
+    workspace.mkdir()
+    results = _run_workflow_overlay_validator(workspace, _overlay_validator_context(package))
+
+    failures = "\n".join(results["failed"])
+    assert "reference/workflow-protocol.json missing" in failures
+    assert "scripts/workflow-state.mjs missing" in failures
+    assert "scripts/workflow-guard.mjs missing" in failures
+    assert "scripts/workflow-handoff.mjs missing" in failures
+
+
+def test_workflow_overlay_contract_validator_accepts_minimal_contract_package(tmp_path: Path):
+    package = tmp_path / "overlay-skill"
+    package.mkdir()
+    (package / "reference").mkdir()
+    (package / "scripts").mkdir()
+    (package / "reference" / "workflow-protocol.json").write_text(
+        json.dumps(
+            {
+                "schemaVersion": 1,
+                "kind": "comet-five-phase-overlay",
+                "name": "overlay-skill",
+                "goal": "Validate overlay contracts.",
+                "nodes": [{"id": "design", "label": "Design"}],
+            }
+        ),
+        encoding="utf-8",
+    )
+    (package / "scripts" / "workflow-state.mjs").write_text(
+        "activeCometChanges resolveCometOverlayChange workflow-evidence "
+        "isCometOverlay(protocol) command === 'init' /comet-open",
+        encoding="utf-8",
+    )
+    (package / "scripts" / "workflow-guard.mjs").write_text(
+        "readOverlayEvidence workflow-evidence missing augmentation evidence "
+        "COMET STATE: unchanged",
+        encoding="utf-8",
+    )
+    (package / "scripts" / "workflow-handoff.mjs").write_text(
+        "workflow-protocol.json workflow: protocol.name protocol.nodes.map "
+        "requiredSkillCalls augmentations outputSchemas",
+        encoding="utf-8",
+    )
+    _write_overlay_eval_manifest(package)
+
+    workspace = tmp_path / "workspace"
+    workspace.mkdir()
+    results = _run_workflow_overlay_validator(workspace, _overlay_validator_context(package))
+
+    assert results["failed"] == []
+
+
+def test_build_report_payload_persists_sample_quality():
+    report = conftest._build_report_payload(
+        treatment_name="COMET_FULL_040_BETA",
+        rep=1,
+        run_id="run-1",
+        events={"duration_seconds": 10, "total_tokens": 100, "total_cost_usd": 0.01},
+        passed=["[RUBRIC] weighted_score: 1.00"],
+        failed=[],
+        scripts_used=[],
+        artifact_references={"report": "reports/demo.json"},
+        failure_attribution=[],
+        returncode=0,
+        stdout=json.dumps({"type": "result", "duration_ms": 10000}) + "\n",
+        stderr="",
+    )
+
+    assert report["sample_quality"]["status"] == "included"
+    assert report["sample_quality"]["reason_code"] == "valid_signal"
+    assert report["sample_quality"]["include_in_analysis"] is True
+
+
+def test_build_report_payload_marks_timeout_as_excluded():
+    report = conftest._build_report_payload(
+        treatment_name="COMET_FULL_039",
+        rep=1,
+        run_id="run-2",
+        events={},
+        passed=[],
+        failed=["no result"],
+        scripts_used=[],
+        artifact_references={"report": "reports/demo.json"},
+        failure_attribution=[],
+        returncode=124,
+        stdout="",
+        stderr="Timeout after 600s",
+    )
+
+    assert report["sample_quality"]["status"] == "excluded"
+    assert report["sample_quality"]["reason_code"] == "runner_timeout"
+    assert report["sample_quality"]["include_in_analysis"] is False

package/eval/local/tests/scaffold/test_evidence.py

@@ -0,0 +1,15 @@
+from pathlib import Path
+
+from scaffold.python.evidence import build_eval_artifact_references
+
+
+def test_build_eval_artifact_references_uses_stable_paths(tmp_path: Path):
+    refs = build_eval_artifact_references(tmp_path, "COMET-FULL", 2)
+
+    assert refs == {
+        "events": str(tmp_path / "events" / "COMET_FULL_040_BETA_rep2.json"),
+        "raw_stdout": str(tmp_path / "raw" / "COMET_FULL_040_BETA_rep2_stdout.json"),
+        "raw_stderr": str(tmp_path / "raw" / "COMET_FULL_040_BETA_rep2_stderr.txt"),
+        "report": str(tmp_path / "reports" / "COMET_FULL_040_BETA_rep2_report.json"),
+        "artifacts": str(tmp_path / "artifacts" / "COMET_FULL_040_BETA_rep2"),
+    }