@rpamis/comet 0.4.0-beta.2 → 0.4.0-beta.4

package/eval/local/tests/tasks/test_validation_scripts.py

@@ -0,0 +1,143 @@
+"""Regression tests for task validation scripts."""
+
+from __future__ import annotations
+
+import importlib.util
+import sys
+import types
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[3]
+
+
+def _load_validator(path: Path, workspace: Path):
+    fake_checks = types.ModuleType("comet_checks")
+    fake_checks.WORKSPACE = workspace
+    fake_checks._passed = lambda check, message: {
+        "check": check,
+        "status": "passed",
+        "message": message,
+    }
+    fake_checks._failed = lambda check, message: {
+        "check": check,
+        "status": "failed",
+        "message": message,
+    }
+    fake_checks.run_comet_checks = lambda: []
+    fake_checks.write_results = lambda results: results
+
+    previous = sys.modules.get("comet_checks")
+    sys.modules["comet_checks"] = fake_checks
+    try:
+        spec = importlib.util.spec_from_file_location(f"validator_{path.stem}", path)
+        assert spec and spec.loader
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+        return module
+    finally:
+        if previous is None:
+            sys.modules.pop("comet_checks", None)
+        else:
+            sys.modules["comet_checks"] = previous
+
+
+def test_refactor_counter_accepts_annotated_wrappers(tmp_path: Path):
+    (tmp_path / "text_processor.py").write_text(
+        """
+def count(text: str, unit: str) -> int:
+    return len(text)
+
+def count_words(text: str) -> int:
+    return count(text, "words")
+
+def count_lines(text: str) -> int:
+    return count(text, "lines")
+
+def count_chars(text: str) -> int:
+    return count(text, "chars")
+""",
+        encoding="utf-8",
+    )
+    module = _load_validator(
+        ROOT / "local/tasks/comet-refactor-counter/validation/test_refactor_counter.py",
+        tmp_path,
+    )
+
+    result = module.check_count_dispatcher()
+
+    assert result["status"] == "passed"
+
+
+def test_fix_median_reports_pytest_stderr(monkeypatch, tmp_path: Path):
+    module = _load_validator(
+        ROOT / "local/tasks/comet-fix-median/validation/test_fix_median.py",
+        tmp_path,
+    )
+    calls = []
+
+    def fake_run(*args, **kwargs):
+        calls.append(args[0])
+        if len(calls) == 1:
+            return types.SimpleNamespace(returncode=0, stdout="2.5\n", stderr="")
+        return types.SimpleNamespace(
+            returncode=1,
+            stdout="",
+            stderr="/usr/local/bin/python: No module named pytest\n",
+        )
+
+    monkeypatch.setattr(module.subprocess, "run", fake_run)
+
+    result = module.check_median_fix()
+
+    assert result["status"] == "failed"
+    assert "No module named pytest" in result["message"]
+
+
+def test_pytest_task_images_install_pytest():
+    task_root = ROOT / "local/tasks"
+    missing = []
+    for dockerfile in task_root.glob("*/environment/Dockerfile"):
+        environment = dockerfile.parent
+        if not any("import pytest" in test.read_text(encoding="utf-8") for test in environment.glob("test_*.py")):
+            continue
+        text = dockerfile.read_text(encoding="utf-8").lower()
+        if "pytest" not in text:
+            missing.append(str(dockerfile.relative_to(ROOT)))
+
+    assert missing == []
+
+
+def test_comet_state_accepts_archived_change_without_active_state(monkeypatch, tmp_path: Path):
+    from scaffold.python.validation import comet_workflow
+
+    archived = tmp_path / "openspec" / "changes" / "archive" / "2026-06-20-fix"
+    archived.mkdir(parents=True)
+    (archived / "proposal.md").write_text("# Proposal\n", encoding="utf-8")
+    (archived / "tasks.md").write_text("- [x] Done\n", encoding="utf-8")
+    monkeypatch.setattr(comet_workflow, "WORKSPACE", tmp_path)
+
+    result = comet_workflow.check_comet_state()
+
+    assert result == {
+        "check": "comet_state",
+        "status": "passed",
+        "message": "phase=archived",
+    }
+
+
+def test_workflow_phases_accepts_verification_report_name(monkeypatch, tmp_path: Path):
+    from scaffold.python.validation import comet_workflow
+
+    archived = tmp_path / "openspec" / "changes" / "archive" / "2026-06-20-refactor"
+    archived.mkdir(parents=True)
+    (archived / "proposal.md").write_text("# Proposal\n", encoding="utf-8")
+    (archived / "design.md").write_text("# Design\n", encoding="utf-8")
+    (archived / "tasks.md").write_text("- [x] Done\n", encoding="utf-8")
+    (archived / "verification-report.md").write_text("# Verification\n", encoding="utf-8")
+    monkeypatch.setattr(comet_workflow, "WORKSPACE", tmp_path)
+
+    result = comet_workflow.check_workflow_phases()
+
+    assert result["status"] == "passed"
+    assert "verify" in result["message"]

package/eval/local/treatments/comet/comet_full_039.yaml

@@ -0,0 +1,141 @@
+# Full comet skill treatment (0.3.9 baseline, .sh runtime)
+# Injects: the 0.3.9 Comet snapshot plus the same benchmark dependency
+# snapshots used by the 0.4.0 treatment. Source dirs are prefixed
+# comet-classic-039* to avoid colliding with the 0.4.0 sources, but the
+# injected skill names are restored to comet/comet-open/etc. so the 0.3.9
+# boilerplate `find ... */comet/scripts/comet-env.sh` resolves.
+COMET_FULL_039:
+  description: "0.3.9 baseline comet skill (shell runtime) with Comet, OpenSpec, and Superpowers dependencies"
+  skills:
+    # Main dispatcher (carries scripts/ with the frozen 0.3.9 .sh launchers)
+    - name: comet
+      skill: 039-release/comet-classic-039
+      variant: all
+      base: benchmarks
+    # Phase sub-skills — source is the 0.3.9 snapshot, injected under canonical names
+    - name: comet-open
+      skill: 039-release/comet-classic-039-open
+      variant: all
+      base: benchmarks
+    - name: comet-design
+      skill: 039-release/comet-classic-039-design
+      variant: all
+      base: benchmarks
+    - name: comet-build
+      skill: 039-release/comet-classic-039-build
+      variant: all
+      base: benchmarks
+    - name: comet-verify
+      skill: 039-release/comet-classic-039-verify
+      variant: all
+      base: benchmarks
+    - name: comet-archive
+      skill: 039-release/comet-classic-039-archive
+      variant: all
+      base: benchmarks
+    - name: comet-hotfix
+      skill: 039-release/comet-classic-039-hotfix
+      variant: all
+      base: benchmarks
+    - name: comet-tweak
+      skill: 039-release/comet-classic-039-tweak
+      variant: all
+      base: benchmarks
+    # OpenSpec workflow dependencies (benchmark dependency snapshots).
+    - name: openspec-apply-change
+      skill: dependency/openspec/openspec-apply-change
+      variant: all
+      base: benchmarks
+    - name: openspec-archive-change
+      skill: dependency/openspec/openspec-archive-change
+      variant: all
+      base: benchmarks
+    - name: openspec-bulk-archive-change
+      skill: dependency/openspec/openspec-bulk-archive-change
+      variant: all
+      base: benchmarks
+    - name: openspec-continue-change
+      skill: dependency/openspec/openspec-continue-change
+      variant: all
+      base: benchmarks
+    - name: openspec-explore
+      skill: dependency/openspec/openspec-explore
+      variant: all
+      base: benchmarks
+    - name: openspec-ff-change
+      skill: dependency/openspec/openspec-ff-change
+      variant: all
+      base: benchmarks
+    - name: openspec-new-change
+      skill: dependency/openspec/openspec-new-change
+      variant: all
+      base: benchmarks
+    - name: openspec-onboard
+      skill: dependency/openspec/openspec-onboard
+      variant: all
+      base: benchmarks
+    - name: openspec-propose
+      skill: dependency/openspec/openspec-propose
+      variant: all
+      base: benchmarks
+    - name: openspec-sync-specs
+      skill: dependency/openspec/openspec-sync-specs
+      variant: all
+      base: benchmarks
+    - name: openspec-verify-change
+      skill: dependency/openspec/openspec-verify-change
+      variant: all
+      base: benchmarks
+    # Superpowers dependencies used by Comet phase guidance and recovery paths.
+    - name: brainstorming
+      skill: dependency/superpowers/brainstorming
+      variant: all
+      base: benchmarks
+    - name: dispatching-parallel-agents
+      skill: dependency/superpowers/dispatching-parallel-agents
+      variant: all
+      base: benchmarks
+    - name: executing-plans
+      skill: dependency/superpowers/executing-plans
+      variant: all
+      base: benchmarks
+    - name: finishing-a-development-branch
+      skill: dependency/superpowers/finishing-a-development-branch
+      variant: all
+      base: benchmarks
+    - name: receiving-code-review
+      skill: dependency/superpowers/receiving-code-review
+      variant: all
+      base: benchmarks
+    - name: requesting-code-review
+      skill: dependency/superpowers/requesting-code-review
+      variant: all
+      base: benchmarks
+    - name: subagent-driven-development
+      skill: dependency/superpowers/subagent-driven-development
+      variant: all
+      base: benchmarks
+    - name: systematic-debugging
+      skill: dependency/superpowers/systematic-debugging
+      variant: all
+      base: benchmarks
+    - name: test-driven-development
+      skill: dependency/superpowers/test-driven-development
+      variant: all
+      base: benchmarks
+    - name: using-git-worktrees
+      skill: dependency/superpowers/using-git-worktrees
+      variant: all
+      base: benchmarks
+    - name: verification-before-completion
+      skill: dependency/superpowers/verification-before-completion
+      variant: all
+      base: benchmarks
+    - name: writing-plans
+      skill: dependency/superpowers/writing-plans
+      variant: all
+      base: benchmarks
+    - name: writing-skills
+      skill: dependency/superpowers/writing-skills
+      variant: all
+      base: benchmarks

package/eval/local/treatments/comet/comet_full_040_beta.yaml

@@ -0,0 +1,140 @@
+# Full comet skill treatment (workflow 0.4.0-beta.1, .mjs runtime)
+# Injects: the 0.4.0 beta Comet snapshot plus the benchmark dependency
+# snapshots under dependency/openspec and dependency/superpowers.
+# OpenSpec CLI is still provided by the task Docker image; OpenSpec Skills are
+# injected so slash/Skill invocation behavior matches a real installation.
+COMET_FULL_040_BETA:
+  description: "Full comet skill (workflow 0.4.0-beta.1) with Comet, OpenSpec, and Superpowers dependencies"
+  skills:
+    # Main dispatcher (carries scripts/ with independent .mjs command bundles and runtime/classic control package)
+    - name: comet
+      skill: 040-beta/comet
+      variant: all
+      base: benchmarks
+    # Phase sub-skills (slash-invoked as /comet-open etc.)
+    - name: comet-open
+      skill: 040-beta/comet-open
+      variant: all
+      base: benchmarks
+    - name: comet-design
+      skill: 040-beta/comet-design
+      variant: all
+      base: benchmarks
+    - name: comet-build
+      skill: 040-beta/comet-build
+      variant: all
+      base: benchmarks
+    - name: comet-verify
+      skill: 040-beta/comet-verify
+      variant: all
+      base: benchmarks
+    - name: comet-archive
+      skill: 040-beta/comet-archive
+      variant: all
+      base: benchmarks
+    - name: comet-hotfix
+      skill: 040-beta/comet-hotfix
+      variant: all
+      base: benchmarks
+    - name: comet-tweak
+      skill: 040-beta/comet-tweak
+      variant: all
+      base: benchmarks
+    # OpenSpec workflow dependencies (benchmark dependency snapshots).
+    - name: openspec-apply-change
+      skill: dependency/openspec/openspec-apply-change
+      variant: all
+      base: benchmarks
+    - name: openspec-archive-change
+      skill: dependency/openspec/openspec-archive-change
+      variant: all
+      base: benchmarks
+    - name: openspec-bulk-archive-change
+      skill: dependency/openspec/openspec-bulk-archive-change
+      variant: all
+      base: benchmarks
+    - name: openspec-continue-change
+      skill: dependency/openspec/openspec-continue-change
+      variant: all
+      base: benchmarks
+    - name: openspec-explore
+      skill: dependency/openspec/openspec-explore
+      variant: all
+      base: benchmarks
+    - name: openspec-ff-change
+      skill: dependency/openspec/openspec-ff-change
+      variant: all
+      base: benchmarks
+    - name: openspec-new-change
+      skill: dependency/openspec/openspec-new-change
+      variant: all
+      base: benchmarks
+    - name: openspec-onboard
+      skill: dependency/openspec/openspec-onboard
+      variant: all
+      base: benchmarks
+    - name: openspec-propose
+      skill: dependency/openspec/openspec-propose
+      variant: all
+      base: benchmarks
+    - name: openspec-sync-specs
+      skill: dependency/openspec/openspec-sync-specs
+      variant: all
+      base: benchmarks
+    - name: openspec-verify-change
+      skill: dependency/openspec/openspec-verify-change
+      variant: all
+      base: benchmarks
+    # Superpowers dependencies used by Comet phase guidance and recovery paths.
+    - name: brainstorming
+      skill: dependency/superpowers/brainstorming
+      variant: all
+      base: benchmarks
+    - name: dispatching-parallel-agents
+      skill: dependency/superpowers/dispatching-parallel-agents
+      variant: all
+      base: benchmarks
+    - name: executing-plans
+      skill: dependency/superpowers/executing-plans
+      variant: all
+      base: benchmarks
+    - name: finishing-a-development-branch
+      skill: dependency/superpowers/finishing-a-development-branch
+      variant: all
+      base: benchmarks
+    - name: receiving-code-review
+      skill: dependency/superpowers/receiving-code-review
+      variant: all
+      base: benchmarks
+    - name: requesting-code-review
+      skill: dependency/superpowers/requesting-code-review
+      variant: all
+      base: benchmarks
+    - name: subagent-driven-development
+      skill: dependency/superpowers/subagent-driven-development
+      variant: all
+      base: benchmarks
+    - name: systematic-debugging
+      skill: dependency/superpowers/systematic-debugging
+      variant: all
+      base: benchmarks
+    - name: test-driven-development
+      skill: dependency/superpowers/test-driven-development
+      variant: all
+      base: benchmarks
+    - name: using-git-worktrees
+      skill: dependency/superpowers/using-git-worktrees
+      variant: all
+      base: benchmarks
+    - name: verification-before-completion
+      skill: dependency/superpowers/verification-before-completion
+      variant: all
+      base: benchmarks
+    - name: writing-plans
+      skill: dependency/superpowers/writing-plans
+      variant: all
+      base: benchmarks
+    - name: writing-skills
+      skill: dependency/superpowers/writing-skills
+      variant: all
+      base: benchmarks

package/eval/local/treatments/common/control.yaml

@@ -0,0 +1,4 @@
+# Control treatment: no skills injected (baseline)
+CONTROL:
+  description: "No skills baseline - Claude has no comet workflow guidance"
+  skills: []

package/eval/pyproject.toml

@@ -0,0 +1,41 @@
+[project]
+name = "comet-skill-eval"
+version = "0.1.0"
+description = "Benchmark evaluation for Comet skill (OpenSpec + Superpowers workflow)"
+requires-python = ">=3.11"
+dependencies = [
+    "pytest>=8.0",
+    "pytest-timeout>=2.2",
+    "pytest-xdist>=3.5",
+    "python-dotenv>=1.0",
+    "pyyaml>=6.0",
+    "pydantic>=2.0",
+    "requests>=2.31",
+    "tomli>=2.0; python_version < '3.11'",
+]
+
+[project.optional-dependencies]
+# LangSmith tracing + pytest integration; install with: uv sync --extra langsmith
+# The LangSmith suite marks tests with @pytest.mark.langsmith and logs rubric
+# scores + treatment comparison via langsmith.testing (requires the pytest extra).
+langsmith = [
+    "langsmith[pytest]>=0.3.4",
+    "langchain>=0.3",
+    "langchain-openai>=0.2",
+]
+dev = [
+    "ruff>=0.4",
+    "pre-commit>=3.7",
+    "matplotlib>=3.8",
+]
+
+[tool.pytest.ini_options]
+testpaths = ["local/tests"]
+pythonpath = ["."]
+markers = [
+    "langsmith: marks test as using LangSmith tracing",
+]
+
+[tool.ruff]
+line-length = 100
+target-version = "py311"

package/eval/report-html-config.json

@@ -0,0 +1,6 @@
+{
+  "report_outputs": {
+    "markdown": true,
+    "html": true
+  }
+}

package/eval/scaffold/__init__.py

@@ -0,0 +1,67 @@
+"""Testing scaffold for Comet skill benchmarks.
+
+Re-exports from scaffold.python for convenience.
+"""
+
+from .python import (
+    NoiseTask,
+    Treatment,
+    ValidatorFn,
+    build_docker_image,
+    check_claude_available,
+    check_code_execution,
+    check_docker_available,
+    check_file_exists,
+    check_no_pattern,
+    check_pattern,
+    check_skill_invoked,
+    compose_validators,
+    make_execution_validator,
+    run_claude_in_docker,
+    run_eval_in_docker,
+    run_node_in_docker,
+    run_python_in_docker,
+    run_shell,
+    run_validators,
+)
+from .python.tasks import Task, TaskConfig, list_tasks, load_task
+from .python.treatments import (
+    TreatmentConfig,
+    build_treatment_skills,
+    list_treatments,
+    load_treatment,
+    load_treatments,
+    load_treatments_yaml,
+)
+
+__all__ = [
+    "Task",
+    "TaskConfig",
+    "load_task",
+    "list_tasks",
+    "TreatmentConfig",
+    "build_treatment_skills",
+    "list_treatments",
+    "load_treatment",
+    "load_treatments",
+    "load_treatments_yaml",
+    "NoiseTask",
+    "Treatment",
+    "ValidatorFn",
+    "compose_validators",
+    "run_validators",
+    "check_file_exists",
+    "check_pattern",
+    "check_no_pattern",
+    "check_skill_invoked",
+    "check_code_execution",
+    "run_shell",
+    "check_docker_available",
+    "check_claude_available",
+    "build_docker_image",
+    "run_python_in_docker",
+    "make_execution_validator",
+    "run_eval_in_docker",
+    "run_node_in_docker",
+    "run_claude_in_docker",
+]

package/eval/scaffold/python/__init__.py

@@ -0,0 +1,95 @@
+"""Python-specific scaffold components for comet skill benchmarks."""
+
+from .logging import (
+    ExperimentLogger,
+    ReportColumn,
+    TreatmentResult,
+    bool_column,
+    default_columns,
+    extract_events,
+    parse_output,
+    quality_column,
+    save_events,
+    save_raw,
+    save_report,
+    strip_ansi,
+)
+from .manifests import SkillEvalManifest, load_eval_manifest
+from .profiles import (
+    AUTHORING_SKILL_PROFILE,
+    COMET_WORKFLOW_PROFILE,
+    GENERIC_PROFILE,
+    ProfileSpec,
+    all_rubric_dimensions,
+    get_profile,
+    list_profiles,
+    resolve_profile_name,
+    run_profile_rubric,
+)
+from .evidence import EvalArtifactReference, build_eval_artifact_references
+from .attribution import FailureAttribution, classify_failures
+from .sample_quality import (
+    SampleQuality,
+    include_in_analysis,
+    infer_sample_quality,
+    quality_from_report,
+    sample_quality_dict,
+)
+from .report_outputs import (
+    ReportOutputConfig,
+    load_report_output_config,
+    preferred_report_path,
+    render_markdown_html,
+    write_report_outputs,
+)
+from .schema import NoiseTask, Treatment
+from .utils import (
+    build_docker_image,
+    check_claude_available,
+    check_docker_available,
+    get_field,
+    get_nested_field,
+    make_execution_validator,
+    normalize_score,
+    read_json_file,
+    retry_with_backoff,
+    run_claude_in_docker,
+    run_eval_in_docker,
+    run_node_in_docker,
+    run_python_in_docker,
+    run_shell,
+)
+from .validation import (
+    ValidatorFn,
+    check_code_execution,
+    check_file_exists,
+    check_no_pattern,
+    check_pattern,
+    check_python_execution,
+    check_skill_invoked,
+    check_typescript_execution,
+    compose_validators,
+    run_validators,
+)
+
+__all__ = [
+    "NoiseTask", "Treatment", "ValidatorFn", "compose_validators", "run_validators",
+    "check_file_exists", "check_pattern", "check_no_pattern", "check_skill_invoked",
+    "check_code_execution", "check_python_execution", "check_typescript_execution",
+    "run_shell", "check_docker_available", "check_claude_available", "build_docker_image",
+    "run_python_in_docker", "make_execution_validator", "run_eval_in_docker",
+    "run_node_in_docker", "run_claude_in_docker", "retry_with_backoff", "read_json_file",
+    "get_field", "get_nested_field", "normalize_score", "parse_output", "extract_events",
+    "strip_ansi", "ExperimentLogger", "TreatmentResult", "save_events", "save_raw",
+    "save_report", "ReportColumn", "bool_column", "quality_column", "default_columns",
+    "EvalArtifactReference", "build_eval_artifact_references",
+    "FailureAttribution", "classify_failures",
+    "SkillEvalManifest", "load_eval_manifest",
+    "AUTHORING_SKILL_PROFILE", "COMET_WORKFLOW_PROFILE", "GENERIC_PROFILE",
+    "ProfileSpec", "all_rubric_dimensions", "get_profile", "list_profiles",
+    "resolve_profile_name", "run_profile_rubric",
+    "ReportOutputConfig", "load_report_output_config", "preferred_report_path",
+    "render_markdown_html", "write_report_outputs",
+    "SampleQuality", "infer_sample_quality", "quality_from_report",
+    "sample_quality_dict", "include_in_analysis",
+]

package/eval/scaffold/python/attribution.py

@@ -0,0 +1,43 @@
+"""Failure attribution for eval reports."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import asdict, dataclass
+from typing import Literal
+
+FailureBucket = Literal["harness", "workflow", "task", "model"]
+
+
+@dataclass(frozen=True)
+class FailureAttribution:
+    bucket: FailureBucket
+    check: str
+    reason: str
+
+
+def classify_failure(check: str, events: dict, profile: str | None) -> FailureAttribution:
+    skills_invoked = events.get("skills_invoked") or []
+    if re.search(r"Required skill not invoked|skill.*not invoked", check, re.I):
+        if not skills_invoked:
+            return FailureAttribution(
+                "harness",
+                check,
+                "target Skill was never invoked, so workflow quality is not observable",
+            )
+        return FailureAttribution("workflow", check, "Skill invocation contract failed")
+
+    if re.search(r"artifact path|not found in archive|validator|task directory", check, re.I):
+        return FailureAttribution("task", check, "task or validator path assumption failed")
+
+    if re.search(r"\.comet\.yaml|guard|state|transition|archive", check, re.I):
+        return FailureAttribution("workflow", check, "workflow state or guard evidence failed")
+
+    if profile == "generic" and not skills_invoked:
+        return FailureAttribution("harness", check, "generic Skill target did not run")
+
+    return FailureAttribution("model", check, "task failed after observable workflow execution")
+
+
+def classify_failures(failed: list[str], events: dict, profile: str | None) -> list[dict[str, str]]:
+    return [asdict(classify_failure(check, events, profile)) for check in failed]