@rpamis/comet 0.4.0-beta.3 → 0.4.0-beta.5

package/eval/scaffold/python/evidence.py

@@ -0,0 +1,38 @@
+"""Evidence references for eval reports."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+TREATMENT_NAME_ALIASES = {
+    "COMET-FULL": "COMET_FULL_040_BETA",
+}
+
+
+@dataclass(frozen=True)
+class EvalArtifactReference:
+    kind: str
+    path: str
+
+
+def stable_treatment_name(treatment_name: str) -> str:
+    alias = TREATMENT_NAME_ALIASES.get(treatment_name)
+    if alias:
+        return alias
+    return treatment_name.replace("-", "_")
+
+
+def build_eval_artifact_references(
+    base_dir: Path,
+    treatment_name: str,
+    rep: int,
+) -> dict[str, str]:
+    name = stable_treatment_name(treatment_name)
+    return {
+        "events": str(base_dir / "events" / f"{name}_rep{rep}.json"),
+        "raw_stdout": str(base_dir / "raw" / f"{name}_rep{rep}_stdout.json"),
+        "raw_stderr": str(base_dir / "raw" / f"{name}_rep{rep}_stderr.txt"),
+        "report": str(base_dir / "reports" / f"{name}_rep{rep}_report.json"),
+        "artifacts": str(base_dir / "artifacts" / f"{name}_rep{rep}"),
+    }

package/eval/scaffold/python/external_data_handler.py

@@ -0,0 +1,18 @@
+"""External data handler - stub for comet skill benchmarks.
+
+The original project uses this for LangSmith trace uploads and dataset management.
+For comet-skill-eval, we keep the interface but remove LangSmith dependencies.
+"""
+
+from pathlib import Path
+from typing import Any
+
+
+def run_handler(handler_name: str, **kwargs) -> Any:
+    """Run a named handler. No-op for comet benchmarks."""
+    return None
+
+
+def run_task_handlers(handlers: list, data_dir: Path, langsmith_env: str, run_id: str) -> dict:
+    """Run task data handlers. Returns empty trace map for comet benchmarks."""
+    return {}

package/eval/scaffold/python/generic_llm_judge.py

@@ -0,0 +1,235 @@
+"""LLM-as-judge scoring for generic Skill eval targets.
+
+The rule-based generic rubric (``generic_rubric.py``) catches structural signals
+(file presence, command execution) but misses qualitative depth: did the agent
+produce meaningful output, or just a stub? This module asks a judge LLM to read
+workspace artifacts and score three generic quality dimensions on a 0.00-1.00
+scale with a cited reason.
+
+Runs on the host (not in Docker) via the same Claude CLI + proxy used for the
+subject agent, so it adds no new dependency. Scores are emitted in the same
+``[RUBRIC-JUDGE] dim: score - reason`` format the logging layer understands.
+
+Usage::
+
+    from scaffold.python.generic_llm_judge import judge_generic_messages
+    results = judge_generic_messages(test_dir, outputs)
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from pathlib import Path
+from typing import Any
+
+from scaffold.python.judge_config import build_judge_invocation, run_judge_prompt
+
+try:
+    from dotenv import load_dotenv
+
+    load_dotenv(Path(__file__).resolve().parents[3] / ".env")
+except Exception:
+    pass
+
+GENERIC_JUDGE_DIMENSIONS = ("task_completion", "output_quality", "instruction_adherence")
+
+_JUDGE_RE = re.compile(r"\[RUBRIC-JUDGE\]\s+(\S+):\s*([0-9.]+)\s*-\s*(.+)")
+
+# Directories to skip when collecting workspace artifacts.
+_SKIP_DIRS = {
+    ".git",
+    "node_modules",
+    "__pycache__",
+    ".comet",
+    ".pytest_cache",
+    "venv",
+    ".venv",
+    "env",
+}
+
+# Max chars per file, total budget across all files.
+_PER_FILE_LIMIT = 3000
+_TOTAL_BUDGET = 20_000
+
+
+def _read_file(path: Path, limit: int = _PER_FILE_LIMIT) -> str:
+    """Read up to *limit* chars of a file, gracefully handling missing files."""
+    try:
+        text = path.read_text(errors="ignore")
+        return text[:limit] if len(text) > limit else text
+    except Exception:
+        return f"(missing: {path.name})"
+
+
+def _collect_workspace_artifacts(test_dir: Path) -> str:
+    """Walk *test_dir* and return a concatenated dump of workspace files.
+
+    Skips hidden directories, common vendor dirs, and binary-looking files.
+    Total output is capped at ``_TOTAL_BUDGET`` chars.
+    """
+    parts: list[str] = []
+    total = 0
+
+    for path in sorted(test_dir.rglob("*")):
+        if not path.is_file():
+            continue
+        # Skip hidden paths.
+        if any(part.startswith(".") for part in path.relative_to(test_dir).parts):
+            continue
+        # Skip known vendor / cache dirs.
+        if any(part in _SKIP_DIRS for part in path.relative_to(test_dir).parts):
+            continue
+        # Skip large or binary-looking files.
+        try:
+            if path.stat().st_size > 50_000:
+                continue
+        except OSError:
+            continue
+
+        rel = path.relative_to(test_dir)
+        text = _read_file(path, _PER_FILE_LIMIT)
+        chunk = f"\n=== {rel} ===\n{text}\n"
+
+        if total + len(chunk) > _TOTAL_BUDGET:
+            remaining = _TOTAL_BUDGET - total
+            if remaining > 200:
+                parts.append(chunk[:remaining])
+            break
+        parts.append(chunk)
+        total += len(chunk)
+
+    return "".join(parts) if parts else "(no workspace files found)"
+
+
+def _build_generic_judge_prompt(
+    test_dir: Path,
+    outputs: dict[str, Any],
+) -> str:
+    """Assemble workspace artifacts and the judging rubric into one prompt."""
+    artifacts = _collect_workspace_artifacts(test_dir)
+
+    # Include completion results summary if available.
+    completion = outputs.get("completion") or {}
+    passed_checks = completion.get("passed", [])
+    failed_checks = completion.get("failed", [])
+    completion_summary = (
+        f"Passed: {len(passed_checks)}, Failed: {len(failed_checks)}\n"
+        if passed_checks or failed_checks
+        else "(no baseline checks)\n"
+    )
+    if failed_checks:
+        completion_summary += "Failed checks:\n" + "\n".join(
+            f"  - {item}" for item in failed_checks[:10]
+        )
+
+    # Custom rubric criteria from task.toml.
+    custom_criteria = outputs.get("rubric_criteria") or []
+    custom_section = ""
+    if custom_criteria:
+        lines = "\n".join(f"- {c}" for c in custom_criteria)
+        custom_section = f"\nAdditional task-specific criteria:\n{lines}\n"
+
+    # Build the list of dimensions the judge should output.
+    all_dims = list(GENERIC_JUDGE_DIMENSIONS)
+    dim_lines = [
+        f"[RUBRIC-JUDGE] {dim}: <score> - <reason>" for dim in all_dims
+    ]
+    extra_dim_lines = [
+        f"[RUBRIC-JUDGE] custom_{i}: <score> - <reason>"
+        for i in range(len(custom_criteria))
+    ]
+    all_output_lines = "\n".join(dim_lines + extra_dim_lines)
+
+    # Dimension count for the instruction.
+    total_dims = len(all_dims) + len(custom_criteria)
+
+    return f"""You are an impartial judge scoring an AI agent's task execution. Read the workspace artifacts below and score {total_dims} dimensions on a 0.00-1.00 scale. Output EXACTLY {total_dims} lines, each in this format (no other text):
+
+{all_output_lines}
+
+Scoring criteria:
+
+task_completion (0.00-1.00):
+- 1.0: all task requirements met, output artifacts present and correct, baseline checks pass
+- 0.5: partial completion, some requirements met but others missing or incorrect
+- 0.0: task not completed, key output missing or empty
+
+output_quality (0.00-1.00):
+- 1.0: output is well-structured, complete, non-trivial; shows evidence of thoughtful work
+- 0.5: output present but shallow, minimal, or barely adequate
+- 0.0: output missing, empty, or clearly a stub
+
+instruction_adherence (0.00-1.00):
+- 1.0: agent followed all instructions and constraints, no forbidden patterns, appropriate tool usage
+- 0.5: minor deviations from instructions, but core intent respected
+- 0.0: significant instruction violations, dangerous commands, or constraint failures
+{custom_section}
+Cite specific content from the artifacts in your reason (<=25 words each).
+
+Baseline validation results:
+{completion_summary}
+
+=== Workspace artifacts ===
+{artifacts}
+"""
+
+
+def _run_judge(prompt: str, timeout: int = 120) -> str:
+    """Call the judge LLM through the configured judge provider."""
+    return run_judge_prompt(prompt, timeout=timeout)
+
+
+def judge_generic_artifacts(
+    test_dir: Path,
+    outputs: dict[str, Any],
+    timeout: int = 120,
+) -> dict[str, tuple[float, str]]:
+    """Score qualitative dimensions via an LLM judge for generic skills.
+
+    Returns ``{dim: (score, reason)}``. On any failure (model error, no
+    parseable output) the dimension is omitted so the caller falls back to
+    rule-based scores.
+    """
+    prompt = _build_generic_judge_prompt(test_dir, outputs)
+    raw = _run_judge(prompt, timeout=timeout)
+
+    # Collect all valid dimension names (standard + custom).
+    custom_criteria = outputs.get("rubric_criteria") or []
+    valid_dims = set(GENERIC_JUDGE_DIMENSIONS)
+    for i in range(len(custom_criteria)):
+        valid_dims.add(f"custom_{i}")
+
+    scores: dict[str, tuple[float, str]] = {}
+    for line in raw.splitlines():
+        m = _JUDGE_RE.search(line)
+        if not m:
+            continue
+        dim = m.group(1)
+        try:
+            score = float(m.group(2))
+        except ValueError:
+            continue
+        reason = m.group(3).strip()
+        if dim in valid_dims:
+            scores[dim] = (max(0.0, min(1.0, score)), reason)
+    return scores
+
+
+def judge_generic_messages(
+    test_dir: Path,
+    outputs: dict[str, Any],
+    timeout: int = 120,
+) -> list[str]:
+    """Convenience wrapper returning ``[RUBRIC-JUDGE]`` check messages."""
+    out: list[str] = []
+    try:
+        build_judge_invocation()
+    except ValueError as e:
+        return [f"[RUBRIC-JUDGE] status: skipped - {e}"]
+
+    for dim, (score, reason) in judge_generic_artifacts(
+        test_dir, outputs, timeout=timeout
+    ).items():
+        out.append(f"[RUBRIC-JUDGE] {dim}: {score:.2f} - {reason}")
+    return out

package/eval/scaffold/python/judge_config.py

@@ -0,0 +1,168 @@
+"""Dedicated LLM judge provider configuration."""
+
+from __future__ import annotations
+
+import os
+import json
+import subprocess
+import urllib.error
+import urllib.request
+from dataclasses import dataclass
+from typing import Mapping
+
+
+_ANTHROPIC_PROVIDER_KEYS = (
+    "ANTHROPIC_API_KEY",
+    "ANTHROPIC_AUTH_TOKEN",
+    "ANTHROPIC_BASE_URL",
+    "ANTHROPIC_MODEL",
+    "ANTHROPIC_DEFAULT_HAIKU_MODEL",
+    "ANTHROPIC_DEFAULT_SONNET_MODEL",
+    "ANTHROPIC_DEFAULT_OPUS_MODEL",
+    "ANTHROPIC_DEFAULT_OPUS_MODEL_NAME",
+    "ANTHROPIC_DEFAULT_SONNET_MODEL_NAME",
+)
+
+
+@dataclass(frozen=True)
+class JudgeInvocation:
+    env: dict[str, str]
+    model_flag: list[str]
+    model: str
+    api_key: str
+    auth_token: str
+    base_url: str
+
+
+def build_judge_invocation(
+    source_env: Mapping[str, str] | None = None,
+) -> JudgeInvocation:
+    """Build an isolated Claude CLI invocation environment for LLM-as-judge.
+
+    Judge configuration is intentionally separate from the subject agent's
+    ANTHROPIC_* provider settings. This prevents accidentally judging a run
+    with the same model or endpoint under test.
+    """
+    source = source_env if source_env is not None else os.environ
+    model = (source.get("BENCH_JUDGE_MODEL") or "").strip()
+    if not model:
+        raise ValueError("BENCH_JUDGE_MODEL is required when BENCH_LLM_JUDGE=1")
+
+    env = dict(source)
+    for key in _ANTHROPIC_PROVIDER_KEYS:
+        env.pop(key, None)
+
+    env["ANTHROPIC_MODEL"] = model
+
+    api_key = (source.get("BENCH_JUDGE_API_KEY") or "").strip()
+    auth_token = (source.get("BENCH_JUDGE_AUTH_TOKEN") or "").strip()
+    base_url = (source.get("BENCH_JUDGE_BASE_URL") or "").strip()
+
+    if api_key:
+        env["ANTHROPIC_API_KEY"] = api_key
+    if auth_token:
+        env["ANTHROPIC_AUTH_TOKEN"] = auth_token
+        env.pop("ANTHROPIC_API_KEY", None)
+    if base_url:
+        env["ANTHROPIC_BASE_URL"] = base_url
+
+    return JudgeInvocation(
+        env=env,
+        model_flag=["--model", model],
+        model=model,
+        api_key=api_key,
+        auth_token=auth_token,
+        base_url=base_url,
+    )
+
+
+def run_judge_prompt(prompt: str, timeout: int = 120) -> str:
+    """Run the judge prompt through a dedicated provider configuration.
+
+    Anthropic-compatible HTTP is preferred when a judge base URL and credential
+    are configured. This avoids Claude CLI request-shape incompatibilities with
+    stricter proxy providers. Without a dedicated judge endpoint, fall back to
+    the local Claude CLI for existing host-authenticated setups.
+    """
+    try:
+        invocation = build_judge_invocation()
+    except ValueError as e:
+        return f"[RUBRIC-JUDGE] status: skipped - {e}"
+
+    if invocation.base_url and (invocation.auth_token or invocation.api_key):
+        return _run_judge_http(prompt, invocation, timeout=timeout)
+
+    return _run_judge_claude_cli(prompt, invocation, timeout=timeout)
+
+
+def _messages_url(base_url: str) -> str:
+    base = base_url.rstrip("/")
+    if base.endswith("/messages"):
+        return base
+    if base.endswith("/v1"):
+        return f"{base}/messages"
+    return f"{base}/v1/messages"
+
+
+def _run_judge_http(prompt: str, invocation: JudgeInvocation, timeout: int = 120) -> str:
+    body = json.dumps(
+        {
+            "model": invocation.model,
+            "max_tokens": 2048,
+            "messages": [{"role": "user", "content": prompt}],
+        }
+    ).encode("utf-8")
+    headers = {
+        "anthropic-version": "2023-06-01",
+        "content-type": "application/json",
+    }
+    if invocation.auth_token:
+        headers["authorization"] = f"Bearer {invocation.auth_token}"
+    else:
+        headers["x-api-key"] = invocation.api_key
+
+    request = urllib.request.Request(
+        _messages_url(invocation.base_url),
+        data=body,
+        headers=headers,
+        method="POST",
+    )
+    try:
+        with urllib.request.urlopen(request, timeout=timeout) as response:
+            payload = json.loads(response.read().decode("utf-8", errors="replace"))
+    except urllib.error.HTTPError as e:
+        detail = e.read().decode("utf-8", errors="replace").strip()
+        return f"(judge error: HTTP {e.code} {detail})"
+    except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as e:
+        return f"(judge error: {e})"
+
+    content = payload.get("content") or []
+    text_parts = [
+        item.get("text", "")
+        for item in content
+        if isinstance(item, dict) and item.get("type") == "text"
+    ]
+    return "\n".join(part for part in text_parts if part).strip()
+
+
+def _run_judge_claude_cli(
+    prompt: str,
+    invocation: JudgeInvocation,
+    timeout: int = 120,
+) -> str:
+    import shutil
+
+    claude_bin = shutil.which("claude") or "claude"
+    try:
+        result = subprocess.run(
+            [claude_bin, "-p", "", "--dangerously-skip-permissions", *invocation.model_flag],
+            input=prompt,
+            capture_output=True,
+            timeout=timeout,
+            env=invocation.env,
+            encoding="utf-8",
+            errors="replace",
+        )
+        return result.stdout or ""
+    except (subprocess.TimeoutExpired, FileNotFoundError) as e:
+        return f"(judge error: {e})"

package/eval/scaffold/python/llm_judge.py

@@ -0,0 +1,191 @@
+"""LLM-as-judge scoring for qualitative rubric dimensions.
+
+The rule-based rubric (``rubric.py``) catches structural signals (file
+presence, keyword counts) but misses qualitative depth: did the design doc
+genuinely explore alternatives, or did it just happen to contain the word
+"risk"? This module asks a judge LLM to read the actual artifacts and score
+three dimensions on a 0.00–1.00 scale with a cited reason.
+
+Runs on the host (not in Docker) via the same Claude CLI + proxy used for the
+subject agent, so it adds no new dependency. Scores are emitted in the same
+``[RUBRIC-JUDGE] dim: score - reason`` format the logging layer understands.
+
+Usage from the rubric validator or rescore tool::
+
+    from scaffold.python.llm_judge import judge_artifacts
+    scores = judge_artifacts(test_dir)  # {dim: (score, reason)}
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from pathlib import Path
+from typing import Any
+
+from scaffold.python.judge_config import build_judge_invocation, run_judge_prompt
+
+# Ensure proxy credentials (ANTHROPIC_AUTH_TOKEN etc.) are available when this
+# module is imported outside pytest (e.g. by the rescore tool).
+try:
+    from dotenv import load_dotenv
+    load_dotenv(Path(__file__).resolve().parents[3] / ".env")
+except Exception:
+    pass
+
+# Dimensions the judge evaluates (subset of the rubric where rules are weak).
+JUDGE_DIMENSIONS = ("artifact_quality", "spec_drift", "main_flow")
+
+_JUDGE_RE = re.compile(r"\[RUBRIC-JUDGE\]\s+(\S+):\s*([0-9.]+)\s*-\s*(.+)")
+
+
+def _read_file(path: Path, limit: int = 4000) -> str:
+    """Read up to `limit` chars of a file, gracefully handling missing files."""
+    try:
+        text = path.read_text(errors="ignore")
+        return text[:limit] if len(text) > limit else text
+    except Exception:
+        return f"(missing: {path.name})"
+
+
+def _find_change_dir(test_dir: Path) -> Path | None:
+    """Locate the comet change dir, handling both active and archived layouts."""
+    changes_root = test_dir / "openspec" / "changes"
+    if not changes_root.exists():
+        return None
+    for d in changes_root.iterdir():
+        if not d.is_dir():
+            continue
+        if d.name == "archive":
+            for sub in d.iterdir():
+                if sub.is_dir() and (sub / "proposal.md").exists():
+                    return sub
+        elif (d / "proposal.md").exists():
+            return d
+    # Fallback: any subdir of archive/
+    archive = changes_root / "archive"
+    if archive.exists():
+        for sub in archive.iterdir():
+            if sub.is_dir():
+                return sub
+    return None
+
+
+def _build_judge_prompt(test_dir: Path) -> str:
+    """Assemble the artifacts and the judging rubric into one prompt."""
+    change = _find_change_dir(test_dir)
+    proposal = _read_file(change / "proposal.md") if change else "(no change dir)"
+    design = _read_file(change / "design.md") if change else "(no design.md)"
+    tasks = _read_file(change / "tasks.md") if change else "(no tasks.md)"
+
+    # Design doc may live under docs/superpowers/specs/ instead.
+    if design.startswith("(missing"):
+        specs_dir = test_dir / "docs" / "superpowers" / "specs"
+        if specs_dir.exists():
+            spec_files = sorted(specs_dir.glob("*.md"))
+            if spec_files:
+                design = _read_file(spec_files[0])
+
+    # Verify report (phase 4)
+    verify_report = "(none)"
+    reports_dir = test_dir / "docs" / "superpowers" / "reports"
+    if reports_dir.exists():
+        rep_files = sorted(reports_dir.glob("*.md"))
+        if rep_files:
+            verify_report = _read_file(rep_files[0], 2000)
+
+    # Delta spec (phase drift signal)
+    delta_spec = "(none)"
+    if change:
+        specs = list(change.rglob("spec.md"))
+        if specs:
+            delta_spec = _read_file(specs[0], 2000)
+
+    return f"""You are an impartial judge scoring a software-development workflow run. Read the artifacts below and score THREE dimensions on a 0.00–1.00 scale. Output EXACTLY three lines, each in this format (no other text):
+
+[RUBRIC-JUDGE] artifact_quality: <score> - <reason>
+[RUBRIC-JUDGE] spec_drift: <score> - <reason>
+[RUBRIC-JUDGE] main_flow: <score> - <reason>
+
+Scoring criteria:
+
+artifact_quality (0.00–1.00):
+- 1.0: proposal states real problem/goals/scope/non-goals; design explores >=2 alternatives with explicit tradeoffs and risks; tasks are actionable checkboxes; tests have real assertions
+- 0.5: artifacts present but shallow (stub proposal, design lists one approach without alternatives, tasks vague)
+- 0.0: key artifacts missing or empty
+
+spec_drift (0.00–1.00):
+- 1.0: if a delta spec was written, it was reconciled (synced to main spec or archived cleanly); if none needed, 0.9
+- 0.5: delta spec written but not reconciled, or minor inconsistency
+- 0.0: delta spec contradicts implementation, or implementation drifted from spec with no delta
+
+main_flow (0.00–1.00):
+- 1.0: evidence of all 5 phases (open proposal → design doc → build plan/code → verify report → archive)
+- 0.5: 2-3 phases evidenced
+- 0.0: <2 phases
+
+Cite specific content from the artifacts in your reason (<=25 words each).
+
+=== proposal.md ===
+{proposal}
+
+=== design.md ===
+{design}
+
+=== tasks.md ===
+{tasks}
+
+=== verify report ===
+{verify_report}
+
+=== delta spec ===
+{delta_spec}
+"""
+
+
+def _run_judge(prompt: str, timeout: int = 120) -> str:
+    """Call the judge LLM through the configured judge provider."""
+    return run_judge_prompt(prompt, timeout=timeout)
+
+
+def judge_artifacts(test_dir: Path, timeout: int = 120) -> dict[str, tuple[float, str]]:
+    """Score the three qualitative dimensions via an LLM judge.
+
+    Returns ``{dim: (score, reason)}``. On any failure (model error, no
+    parseable output) the dimension is omitted so the caller falls back to the
+    rule-based score.
+    """
+    if not _find_change_dir(test_dir):
+        return {}
+
+    prompt = _build_judge_prompt(test_dir)
+    raw = _run_judge(prompt, timeout=timeout)
+
+    scores: dict[str, tuple[float, str]] = {}
+    for line in raw.splitlines():
+        m = _JUDGE_RE.search(line)
+        if not m:
+            continue
+        dim = m.group(1)
+        try:
+            score = float(m.group(2))
+        except ValueError:
+            continue
+        reason = m.group(3).strip()
+        if dim in JUDGE_DIMENSIONS:
+            scores[dim] = (max(0.0, min(1.0, score)), reason)
+    return scores
+
+
+def judge_messages(test_dir: Path, timeout: int = 120) -> list[str]:
+    """Convenience wrapper returning ``[RUBRIC-JUDGE]`` check messages."""
+    out: list[str] = []
+    try:
+        build_judge_invocation()
+    except ValueError as e:
+        return [f"[RUBRIC-JUDGE] status: skipped - {e}"]
+
+    for dim, (score, reason) in judge_artifacts(test_dir, timeout=timeout).items():
+        out.append(f"[RUBRIC-JUDGE] {dim}: {score:.2f} - {reason}")
+    return out