@rpamis/comet 0.4.0-beta.3 → 0.4.0-beta.5

package/eval/scaffold/python/tasks.py

@@ -0,0 +1,330 @@
+"""Task loader for self-contained benchmark tasks.
+
+Each task is a directory containing:
+- instruction.md: Task prompt with {variable} placeholders
+- task.toml: Task metadata, validation config (test_scripts, target_artifacts)
+- environment/: Docker context (Dockerfile, source code)
+- validation/: Validator implementations
+- data/: Test data and ground truth (optional)
+
+Usage:
+    from scaffold.python.tasks import load_task, list_tasks
+
+    task = load_task("ls-evaluator")
+    prompt = task.render_prompt(py_dataset="ds-py", ts_dataset="ds-ts", run_id="abc123")
+    validators = task.load_validators()
+"""
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+try:
+    import tomllib
+except ImportError:
+    import tomli as tomllib  # Python < 3.11
+
+
+from scaffold.python.paths import get_tasks_dir
+
+
+COMET_WORKFLOW_INVOCATION_CONTRACT = """\
+## Eval harness requirement
+
+You must begin by invoking the `/comet` Skill/slash command with this task request.
+Do not simulate the Comet workflow in plain prose. The run is invalid unless the
+Comet Skill is invoked and leaves real OpenSpec/Comet workflow artifacts.
+"""
+
+
+@dataclass
+class DataHandler:
+    """A data handler triggered by file pattern match."""
+
+    pattern: str  # Glob pattern relative to task data dir (e.g., "trace_*.jsonl")
+    handler: str  # Handler name (e.g., "upload_traces")
+    args: dict = field(default_factory=dict)  # Handler-specific arguments
+
+
+@dataclass
+class SetupConfig:
+    """Setup configuration for a task.
+
+    Defines data handlers and template variables that are computed at test time.
+    """
+
+    # Data handlers triggered by pattern matches
+    data_handlers: list[DataHandler] = field(default_factory=list)
+
+    # Template variables with format strings (can use {run_id})
+    template_vars: dict[str, str] = field(default_factory=dict)
+
+
+@dataclass
+class ValidationConfig:
+    """Validation configuration from [validation] in task.toml.
+
+    Defines what to check and how. The framework auto-builds a validator
+    from this config — no validators.py needed for most tasks.
+    """
+
+    # Test script(s) to run in Docker (e.g., "test_agent.py")
+    test_scripts: str | list[str] = ""
+
+    # File(s) or dir(s) Claude should produce — existence is checked before
+    # running test scripts. Names are available via runner.artifacts.
+    target_artifacts: str | list[str] = field(default_factory=list)
+
+    # Docker execution timeout in seconds
+    timeout: int = 120
+
+
+@dataclass
+class EvaluationConfig:
+    """Skill-agnostic evaluation contract for a task."""
+
+    profile: str | None = None
+    required_skills: list[str] = field(default_factory=list)
+    expected_artifacts: list[str] = field(default_factory=list)
+    require_skill_invocation: bool = False
+    rubric_criteria: list[str] = field(default_factory=list)
+
+
+@dataclass
+class InteractionConfig:
+    """Controls whether the runner should simulate user replies."""
+
+    mode: str = "none"
+    max_turns: int = 12
+    simulator_prompt: str | None = None
+    decision_patterns: list[str] = field(default_factory=list)
+    continue_prompt: str = "Please continue with the next phase of the workflow."
+
+
+@dataclass
+class TaskConfig:
+    """Configuration loaded from task.toml."""
+
+    name: str
+    description: str
+    difficulty: str = "medium"
+    category: str = ""
+    tags: list[str] = field(default_factory=list)
+
+    # Description of what Claude has access to in this task environment
+    environment_description: str = ""
+
+    # Default treatments to test with this task
+    default_treatments: list[str] = field(default_factory=list)
+
+    # Template variables required for instruction.md
+    template_required: list[str] = field(default_factory=list)
+
+    # Environment settings
+    dockerfile: str = "Dockerfile"
+    timeout_sec: int = 900
+
+    # Validation configuration
+    validation: ValidationConfig = field(default_factory=ValidationConfig)
+
+    # Setup configuration
+    setup: SetupConfig = field(default_factory=SetupConfig)
+
+    # Skill-agnostic evaluation contract
+    evaluation: EvaluationConfig = field(default_factory=EvaluationConfig)
+
+    # Interaction / user-simulator configuration
+    interaction: InteractionConfig = field(default_factory=InteractionConfig)
+
+
+@dataclass
+class Task:
+    """A self-contained benchmark task."""
+
+    path: Path
+    config: TaskConfig
+    instruction_template: str
+
+    @property
+    def name(self) -> str:
+        return self.config.name
+
+    @property
+    def environment_dir(self) -> Path:
+        return self.path / "environment"
+
+    @property
+    def validation_dir(self) -> Path:
+        return self.path / "validation"
+
+    @property
+    def data_dir(self) -> Path:
+        return self.path / "data"
+
+    @property
+    def default_treatments(self) -> list[str]:
+        return self.config.default_treatments
+
+    def render_prompt(self, **kwargs: Any) -> str:
+        """Render the instruction template with provided variables.
+
+        Args:
+            **kwargs: Template variables (e.g., run_id, py_dataset, ts_dataset)
+
+        Returns:
+            Rendered prompt string
+
+        Raises:
+            KeyError: If a required template variable is missing
+        """
+        missing = set(self.config.template_required) - set(kwargs.keys())
+        if missing:
+            raise KeyError(f"Missing required template variables: {missing}")
+        prompt = self.instruction_template.format(**kwargs)
+        if self.config.evaluation.profile == "comet-workflow":
+            return f"{COMET_WORKFLOW_INVOCATION_CONTRACT}\n\n{prompt}"
+        return prompt
+
+    def load_validators(self) -> list:
+        """Build validator from task.toml [validation] config.
+
+        The config specifies test_scripts and target_artifacts. The framework
+        auto-builds an execution validator that runs the scripts in Docker.
+        """
+        vc = self.config.validation
+        if vc.test_scripts:
+            from scaffold.python.utils import make_execution_validator
+
+            return [
+                make_execution_validator(
+                    validation_dir=self.validation_dir,
+                    test_scripts=vc.test_scripts,
+                    target_artifacts=vc.target_artifacts,
+                    timeout=vc.timeout,
+                    data_dir=self.data_dir if self.data_dir.exists() else None,
+                )
+            ]
+
+        return []
+
+
+def load_task(name: str, tasks_dir: Path | None = None) -> Task:
+    """Load a task by name.
+
+    Args:
+        name: Task directory name (e.g., "ls-evaluator")
+        tasks_dir: Optional custom tasks directory
+
+    Returns:
+        Task object with config and instruction template
+    """
+    tasks_dir = tasks_dir or get_tasks_dir()
+    task_path = tasks_dir / name
+
+    if not task_path.exists():
+        raise FileNotFoundError(f"Task not found: {name} (looked in {tasks_dir})")
+
+    # Load task.toml
+    toml_path = task_path / "task.toml"
+    if not toml_path.exists():
+        raise FileNotFoundError(f"task.toml not found in {task_path}")
+
+    with open(toml_path, "rb") as f:
+        toml_data = tomllib.load(f)
+
+    metadata = toml_data.get("metadata", {})
+    template = toml_data.get("template", {})
+    environment = toml_data.get("environment", {})
+    validation = toml_data.get("validation", {})
+    evaluation = toml_data.get("evaluation", {})
+    interaction = toml_data.get("interaction", {})
+    setup_data = toml_data.get("setup", {})
+
+    # Parse setup config
+    data_handlers = [
+        DataHandler(pattern=d["pattern"], handler=d["handler"], args=d.get("args", {}))
+        for d in setup_data.get("data", [])
+    ]
+    setup = SetupConfig(
+        data_handlers=data_handlers,
+        template_vars=setup_data.get("template_vars", {}),
+    )
+
+    validation_config = ValidationConfig(
+        test_scripts=validation.get("test_scripts", ""),
+        target_artifacts=validation.get("target_artifacts", []),
+        timeout=validation.get("timeout", 120),
+    )
+
+    task_name = metadata.get("name", name)
+    inferred_profile = evaluation.get("profile")
+    if not inferred_profile and (
+        metadata.get("category") == "comet" or str(task_name).startswith("comet-")
+    ):
+        inferred_profile = "comet-workflow"
+
+    evaluation_config = EvaluationConfig(
+        profile=inferred_profile,
+        required_skills=evaluation.get("required_skills", []),
+        expected_artifacts=evaluation.get("expected_artifacts", []),
+        require_skill_invocation=bool(evaluation.get("require_skill_invocation", False)),
+        rubric_criteria=evaluation.get("rubric_criteria", []),
+    )
+
+    default_interaction_mode = "auto_user" if inferred_profile == "comet-workflow" else "none"
+    interaction_config = InteractionConfig(
+        mode=interaction.get("mode", default_interaction_mode),
+        max_turns=int(interaction.get("max_turns", 12)),
+        simulator_prompt=interaction.get("simulator_prompt"),
+        decision_patterns=interaction.get("decision_patterns", []),
+        continue_prompt=interaction.get(
+            "continue_prompt",
+            "Please continue with the next phase of the workflow.",
+        ),
+    )
+
+    config = TaskConfig(
+        name=task_name,
+        description=metadata.get("description", ""),
+        difficulty=metadata.get("difficulty", "medium"),
+        category=metadata.get("category", ""),
+        tags=metadata.get("tags", []),
+        environment_description=environment.get("description", ""),
+        default_treatments=metadata.get("default_treatments", []),
+        template_required=template.get("required", []),
+        dockerfile=environment.get("dockerfile", "Dockerfile"),
+        timeout_sec=environment.get("timeout_sec", 900),
+        validation=validation_config,
+        setup=setup,
+        evaluation=evaluation_config,
+        interaction=interaction_config,
+    )
+
+    # Load instruction.md
+    instruction_path = task_path / "instruction.md"
+    if not instruction_path.exists():
+        raise FileNotFoundError(f"instruction.md not found in {task_path}")
+
+    instruction_template = instruction_path.read_text(encoding="utf-8")
+
+    return Task(path=task_path, config=config, instruction_template=instruction_template)
+
+
+def list_tasks(tasks_dir: Path | None = None) -> list[str]:
+    """List available task names.
+
+    Args:
+        tasks_dir: Optional custom tasks directory
+
+    Returns:
+        List of task directory names
+    """
+    tasks_dir = tasks_dir or get_tasks_dir()
+    if not tasks_dir.exists():
+        return []
+
+    return sorted(
+        d.name
+        for d in tasks_dir.iterdir()
+        if d.is_dir() and (d / "task.toml").exists() and (d / "instruction.md").exists()
+    )

package/eval/scaffold/python/treatments.py

@@ -0,0 +1,271 @@
+"""Treatment loader for comet skill benchmark experiments."""
+
+import hashlib
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+import yaml
+
+from scaffold.python.skill_parser import load_skill_variant, skill_config
+from scaffold.python.paths import EVAL_ROOT, get_skills_dir, get_treatments_dir
+
+# Only comet-relevant categories
+TREATMENT_CATEGORIES = {
+    "common",
+    "comet",
+}
+
+
+@dataclass
+class TreatmentConfig:
+    name: str
+    description: str
+    claude_md: str = ""
+    skills: list[dict[str, Any]] = field(default_factory=list)
+    noise_tasks: list[str] = field(default_factory=list)
+
+
+@dataclass
+class SkillSource:
+    name: str
+    source_type: str
+    path: str | None = None
+    hash: str = ""
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+def _add_language_suffix(content: str, lang: str) -> str:
+    suffix = "(Python)" if lang == "py" else "(TypeScript)"
+    return re.sub(
+        r'^(description: "?)(.+?)("?)$',
+        rf"\1\2 {suffix}\3",
+        content,
+        count=1,
+        flags=re.MULTILINE,
+    )
+
+
+def _filter_related_skills(sections: list[str]) -> list[str]:
+    return [s for s in sections if s and "<related_skills>" not in s]
+
+
+def _find_skill_md(path: Path) -> Path:
+    if path.is_file():
+        if path.name != "SKILL.md":
+            raise FileNotFoundError(f"Expected SKILL.md file, got: {path}")
+        return path
+    skill_md = path / "SKILL.md"
+    if not skill_md.exists():
+        raise FileNotFoundError(f"SKILL.md not found in {path}")
+    return skill_md
+
+
+def _hash_skill_dir(skill_path: Path) -> str:
+    if skill_path.is_file():
+        data = skill_path.read_bytes()
+        return "sha256:" + hashlib.sha256(data).hexdigest()
+    digest = hashlib.sha256()
+    for item in sorted(skill_path.rglob("*")):
+        if item.is_file():
+            digest.update(str(item.relative_to(skill_path)).replace("\\", "/").encode())
+            digest.update(item.read_bytes())
+    return "sha256:" + digest.hexdigest()
+
+
+def _build_path_skill_config(name: str, path_value: str) -> dict:
+    raw_path = Path(path_value).expanduser()
+    source_path = raw_path.resolve()
+    if not source_path.exists() and not raw_path.is_absolute():
+        source_path = (EVAL_ROOT / raw_path).resolve()
+    skill_md = _find_skill_md(source_path)
+    skill_dir = skill_md.parent
+    scripts_dir = skill_dir / "scripts"
+    content = skill_md.read_text(encoding="utf-8")
+    cfg = skill_config([content], scripts_dir if scripts_dir.exists() else None, None)
+    cfg["source_dir"] = skill_dir
+    cfg["source"] = {
+        "name": name,
+        "source_type": "path",
+        "path": str(source_path),
+        "hash": _hash_skill_dir(skill_dir),
+    }
+    return cfg
+
+
+def _build_skill_config(
+    skill_dir: str,
+    variant: str = "all",
+    suffix: bool = False,
+    include_related: bool = False,
+    noise: bool = False,
+    base: str = "benchmarks",
+    included_sections: list[str] | None = None,
+    extra_sections: list[str] | None = None,
+    section_overrides: dict[str, str] | None = None,
+) -> dict:
+    if base == "main":
+        skill_path = get_skills_dir() / "main" / skill_dir
+    else:
+        skill_path = get_skills_dir() / "benchmarks" / skill_dir
+
+    if noise:
+        noise_path = get_skills_dir() / "noise" / skill_dir
+        for filename in ["SKILL.md", "skill.md"]:
+            skill_md = noise_path / filename
+            if skill_md.exists():
+                return skill_config([skill_md.read_text(encoding="utf-8")], None, None)
+        return None
+
+    variant_path = skill_path / f"skill_{variant}.md" if variant else skill_path / "skill.md"
+    if variant and not variant_path.exists():
+        resolved_md_path = None
+        for filename in ["SKILL.md", "skill.md"]:
+            if (skill_path / filename).exists():
+                resolved_md_path = skill_path / filename
+                break
+        if resolved_md_path is None:
+            raise FileNotFoundError(f"No skill file found in {skill_path}")
+        from scaffold.python.skill_parser import get_section_list, parse_skill_md
+        sections = parse_skill_md(resolved_md_path)
+        all_sections = get_section_list(resolved_md_path)
+        scripts_dir = skill_path / "scripts"
+        skill = {
+            "sections": sections,
+            "all": all_sections,
+            "scripts_dir": scripts_dir if scripts_dir.exists() else None,
+            "script_filter": None,
+        }
+    else:
+        resolved_md_path = variant_path
+        skill = load_skill_variant(skill_path, variant)
+
+    if included_sections:
+        sections = []
+        for section_name in included_sections:
+            if section_overrides and section_name in section_overrides:
+                sections.append(section_overrides[section_name])
+            elif section_name in skill["sections"]:
+                sections.append(skill["sections"][section_name])
+        if not include_related:
+            sections = _filter_related_skills(sections)
+        if extra_sections:
+            sections = sections + extra_sections
+        content = "\n\n".join(sections)
+    elif section_overrides:
+        sections = []
+        for section_name, content in skill["sections"].items():
+            if section_name in section_overrides:
+                sections.append(section_overrides[section_name])
+            else:
+                sections.append(content)
+        if not include_related:
+            sections = _filter_related_skills(sections)
+        if extra_sections:
+            sections = sections + extra_sections
+        content = "\n\n".join(sections)
+    else:
+        content = resolved_md_path.read_text(encoding="utf-8")
+        if not include_related:
+            content = re.sub(
+                r"<related_skills>.*?</related_skills>\s*",
+                "",
+                content,
+                flags=re.DOTALL,
+            )
+        if extra_sections:
+            content = content + "\n\n" + "\n\n".join(extra_sections)
+
+    if suffix and variant in ("py", "ts"):
+        content = _add_language_suffix(content, variant)
+
+    cfg = skill_config([content], skill["scripts_dir"], skill.get("script_filter"))
+    cfg["source_dir"] = skill_path
+    return cfg
+
+
+def build_treatment_skills(skill_configs: list[dict[str, Any]]) -> dict[str, dict]:
+    skills = {}
+    for cfg in skill_configs:
+        name = cfg.get("name")
+        skill_dir = cfg.get("skill")
+        if not name and skill_dir:
+            name = skill_dir.replace("_", "-")
+        if cfg.get("source") == "path":
+            skills[name] = _build_path_skill_config(name, cfg["path"])
+            continue
+        if "content" in cfg:
+            skills[name] = skill_config([cfg["content"]], None, None)
+            continue
+        variant = cfg.get("variant", "all")
+        suffix = cfg.get("suffix", False)
+        include_related = cfg.get("include_related", False)
+        noise = cfg.get("noise", False)
+        base = cfg.get("base", "benchmarks")
+        included_sections = cfg.get("included_sections")
+        extra_sections = cfg.get("extra_sections")
+        section_overrides = cfg.get("section_overrides")
+        skill_cfg = _build_skill_config(
+            skill_dir=skill_dir, variant=variant, suffix=suffix,
+            include_related=include_related, noise=noise, base=base,
+            included_sections=included_sections, extra_sections=extra_sections,
+            section_overrides=section_overrides,
+        )
+        if skill_cfg:
+            skills[name] = skill_cfg
+    return skills
+
+
+def load_treatments_yaml(path: Path) -> dict[str, TreatmentConfig]:
+    if not path.exists():
+        raise FileNotFoundError(f"Treatments file not found: {path}")
+    with open(path, encoding="utf-8") as f:
+        data = yaml.safe_load(f)
+    treatments = {}
+    for name, cfg in data.items():
+        if name.startswith("_"):
+            continue
+        treatments[name] = TreatmentConfig(
+            name=name,
+            description=cfg.get("description", ""),
+            claude_md=cfg.get("claude_md", ""),
+            skills=cfg.get("skills", []),
+            noise_tasks=cfg.get("noise_tasks", []),
+        )
+    return treatments
+
+
+def load_treatments() -> dict[str, TreatmentConfig]:
+    treatments_folder = get_treatments_dir()
+    if not treatments_folder.exists():
+        return {}
+    treatments = {}
+    for category in sorted(treatments_folder.iterdir()):
+        if not category.is_dir():
+            continue
+        if category.name not in TREATMENT_CATEGORIES:
+            continue
+        for yaml_file in sorted(category.glob("*.yaml")):
+            category_treatments = load_treatments_yaml(yaml_file)
+            treatments.update(category_treatments)
+    return treatments
+
+
+def load_treatment(name: str):
+    from scaffold import Treatment
+    configs = load_treatments()
+    if name not in configs:
+        raise KeyError(f"Treatment not found: {name}. Available: {list(configs.keys())}")
+    cfg = configs[name]
+    skills = build_treatment_skills(cfg.skills) if cfg.skills else {}
+    return Treatment(
+        description=cfg.description,
+        skills=skills,
+        claude_md=cfg.claude_md if cfg.claude_md else None,
+        validators=[],
+    )
+
+
+def list_treatments() -> list[str]:
+    return sorted(load_treatments().keys())