@rpamis/comet 0.4.0-beta.6 → 0.4.0-beta.7

package/eval/scaffold/python/utils.py

@@ -1,18 +1,36 @@
 """Python utilities - thin wrappers around shell scripts."""
-import json, os, random, shutil, subprocess, time
+
+import json
+import ntpath
+import os
+import random
+import shutil
+import subprocess
+import time
 from pathlib import Path
+
 from dotenv import load_dotenv
-from pydantic import BaseModel, Field
+
 from scaffold.python.paths import EVAL_ROOT, get_suite_root
 
 TEST_CONTEXT_FILE = os.environ.get("BENCH_TEST_CONTEXT", "_test_context.json")
 TEST_RESULTS_FILE = os.environ.get("BENCH_TEST_RESULTS", "_test_results.json")
-load_dotenv(EVAL_ROOT / ".env")
-load_dotenv(get_suite_root() / ".env", override=True)
 SHELL_DIR = Path(__file__).parent.parent / "shell"
 SCAFFOLD_PYTHON_DIR = Path(__file__).parent
 
-def _resolve_bash() -> str:
+
+def load_eval_environment() -> None:
+    """Load eval credentials only at an explicit execution boundary, never on import."""
+    load_dotenv(EVAL_ROOT / ".env")
+    load_dotenv(get_suite_root() / ".env", override=True)
+
+
+def _uses_wsl_bash(bash_exec: str) -> bool:
+    normalized = bash_exec.replace("\\", "/").lower()
+    return normalized.endswith("/windowsapps/bash.exe") or normalized.endswith("/system32/bash.exe")
+
+
+def _resolve_bash(os_name: str | None = None) -> str:
     """Resolve a reliable bash executable for running MSYS shell scripts.
 
     On Windows, ``subprocess.run(['bash', ...])`` may resolve ``bash`` via
@@ -24,7 +42,8 @@ def _resolve_bash() -> str:
     """
     import shutil
 
-    if os.name != "nt":
+    platform_name = os_name or os.name
+    if platform_name != "nt":
         return "bash"
 
     env_bash = os.environ.get("GIT_BASH")
@@ -32,9 +51,19 @@ def _resolve_bash() -> str:
         return env_bash
 
     resolved = shutil.which("bash")
-    if resolved and os.path.isfile(resolved):
+    if resolved and os.path.isfile(resolved) and not _uses_wsl_bash(resolved):
         return resolved
 
+    git_exec = shutil.which("git")
+    if git_exec:
+        git_root = ntpath.dirname(ntpath.dirname(git_exec))
+        for candidate in (
+            ntpath.join(git_root, "bin", "bash.exe"),
+            ntpath.join(git_root, "usr", "bin", "bash.exe"),
+        ):
+            if os.path.isfile(candidate):
+                return candidate
+
     return "bash"
 
 
@@ -90,11 +119,6 @@ def _to_bash_path(value) -> str:
     return s
 
 
-def _uses_wsl_bash(bash_exec: str) -> bool:
-    normalized = bash_exec.replace("\\", "/").lower()
-    return normalized.endswith("/windowsapps/bash.exe") or normalized.endswith("/system32/bash.exe")
-
-
 def _bash_env() -> dict[str, str]:
     env = os.environ.copy()
     if os.name != "nt" or not _uses_wsl_bash(BASH_EXEC):
@@ -121,12 +145,14 @@ def run_shell(script, *args, timeout=None, check=True):
         env=_bash_env(),
     )
 
+
 def check_docker_available():
     try:
         return run_shell("docker.sh", "check", check=False, timeout=10).returncode == 0
     except (subprocess.TimeoutExpired, FileNotFoundError):
         return False
 
+
 def build_docker_image(test_dir, force=False, verbose=False):
     try:
         args = ["build", str(test_dir)] + (["--force"] if force else [])
@@ -135,43 +161,90 @@ def build_docker_image(test_dir, force=False, verbose=False):
     except subprocess.TimeoutExpired:
         return None
 
+
 def _docker_run_script(mode, test_dir, script_name, timeout=120, args=None):
     if not check_docker_available():
         return False, "Docker not available"
     try:
         cmd = [mode, str(test_dir), script_name] + (args or [])
         result = run_shell("docker.sh", *cmd, timeout=timeout, check=False)
-        return result.returncode == 0, result.stdout
+        output = result.stdout
+        if result.returncode != 0 and result.stderr:
+            output = "\n".join(part for part in (result.stdout, result.stderr) if part).strip()
+        return result.returncode == 0, output
     except subprocess.TimeoutExpired:
         return False, f"Timeout ({timeout}s)"
     except Exception as e:
         return False, str(e)
 
+
 def run_python_in_docker(test_dir, script_name, timeout=120, args=None):
     return _docker_run_script("run-python", test_dir, script_name, timeout, args)
 
+
 def run_node_in_docker(test_dir, script_name, timeout=120, args=None):
     return _docker_run_script("run-node", test_dir, script_name, timeout, args)
 
-def run_claude_in_docker(test_dir, prompt, timeout=300, model=None):
+
+def run_claude_in_docker(test_dir, prompt, timeout=300, model=None, image_id=None):
     if not check_docker_available():
         raise RuntimeError("Docker not available")
     cmd = ["run-claude", str(test_dir), prompt, "--timeout", str(timeout)]
     if model:
         cmd.extend(["--model", model])
+    if image_id:
+        cmd.extend(["--image-id", image_id])
     try:
         return run_shell("docker.sh", *cmd, timeout=timeout + 30, check=False)
     except subprocess.TimeoutExpired:
         return subprocess.CompletedProcess(cmd, 124, "", f"Timeout after {timeout}s")
 
+
+def run_claude_loop_in_docker(test_dir, loop_args, timeout=600):
+    """Run the interactive driver and remove its container after host-side timeout."""
+    cmd = ["run-claude-loop", str(test_dir), *loop_args]
+    try:
+        return run_shell("docker.sh", *cmd, timeout=timeout, check=False)
+    except subprocess.TimeoutExpired as error:
+        cleanup_error = ""
+        try:
+            cleanup = run_shell(
+                "docker.sh",
+                "cleanup-claude-loop",
+                test_dir,
+                timeout=30,
+                check=False,
+            )
+            if cleanup.returncode != 0:
+                cleanup_error = f"; cleanup failed: {cleanup.stderr or cleanup.stdout}"
+        except Exception as cleanup_exception:  # pragma: no cover - defensive cleanup
+            cleanup_error = f"; cleanup failed: {cleanup_exception}"
+        stdout = (
+            error.stdout.decode("utf-8", errors="replace")
+            if isinstance(error.stdout, bytes)
+            else (error.stdout or "")
+        )
+        stderr = (
+            error.stderr.decode("utf-8", errors="replace")
+            if isinstance(error.stderr, bytes)
+            else (error.stderr or "")
+        )
+        message = f"Timeout after {timeout}s{cleanup_error}"
+        return subprocess.CompletedProcess(
+            cmd, 124, stdout, "\n".join(filter(None, (stderr, message)))
+        )
+
+
 def _copy_scaffold_to_docker(test_dir):
-    scaffold_root = SCAFFOLD_PYTHON_DIR.parent
     scaffold_dir = test_dir / "scaffold"
     scaffold_dir.mkdir(parents=True, exist_ok=True)
-    (scaffold_dir / "__init__.py").touch()
+    (scaffold_dir / "__init__.py").write_text("", encoding="utf-8")
     py_dest = scaffold_dir / "python"
     py_dest.mkdir(exist_ok=True)
-    (py_dest / "__init__.py").touch()
+    # The repository package initializer imports host-only orchestration
+    # dependencies.  Validator containers need only the copied helper modules,
+    # so always replace a stale/full initializer with a minimal package stub.
+    (py_dest / "__init__.py").write_text("", encoding="utf-8")
     shutil.copy(SCAFFOLD_PYTHON_DIR / "utils.py", py_dest / "utils.py")
     py_validation = SCAFFOLD_PYTHON_DIR / "validation"
     if py_validation.is_dir():
@@ -185,6 +258,7 @@ def _copy_scaffold_to_docker(test_dir):
     if comet_checks_src.exists():
         shutil.copy(comet_checks_src, test_dir / "comet_checks.py")
 
+
 def _parse_json_output(output):
     stripped = output.strip()
     try:
@@ -202,6 +276,7 @@ def _parse_json_output(output):
             continue
     return None
 
+
 def run_eval_in_docker(test_dir, validation_dir, test_script, timeout=120, data_dir=None):
     val_dir = test_dir / "validation"
     val_dir.mkdir(exist_ok=True)
@@ -255,7 +330,10 @@ def _normalise_validation_results(results):
 
     return passed, failed
 
-def make_execution_validator(validation_dir, test_scripts, target_artifacts, timeout=120, data_dir=None):
+
+def make_execution_validator(
+    validation_dir, test_scripts, target_artifacts, timeout=120, data_dir=None
+):
     test_scripts = [test_scripts] if isinstance(test_scripts, str) else test_scripts
     artifacts = [target_artifacts] if isinstance(target_artifacts, str) else target_artifacts
 
@@ -273,7 +351,9 @@ def make_execution_validator(validation_dir, test_scripts, target_artifacts, tim
         context["target_artifacts"] = artifacts
         (test_dir / TEST_CONTEXT_FILE).write_text(json.dumps(context, default=str))
         for script in test_scripts:
-            results = run_eval_in_docker(test_dir, validation_dir, script, timeout=timeout, data_dir=data_dir)
+            results = run_eval_in_docker(
+                test_dir, validation_dir, script, timeout=timeout, data_dir=data_dir
+            )
             script_passed, script_failed = _normalise_validation_results(results)
             passed.extend(script_passed)
             failed.extend(script_failed)
@@ -283,12 +363,16 @@ def make_execution_validator(validation_dir, test_scripts, target_artifacts, tim
 
     return validate_execution
 
+
 def check_claude_available():
     try:
-        return subprocess.run(["claude", "--version"], capture_output=True, timeout=10).returncode == 0
+        return (
+            subprocess.run(["claude", "--version"], capture_output=True, timeout=10).returncode == 0
+        )
     except (subprocess.TimeoutExpired, FileNotFoundError):
         return False
 
+
 def retry_with_backoff(func, max_retries=3, base_delay=1.0, max_delay=10.0, retry_on=None):
     retry_on = retry_on or (lambda e: "429" in str(e) or "rate limit" in str(e).lower())
     for attempt in range(max_retries + 1):
@@ -299,6 +383,7 @@ def retry_with_backoff(func, max_retries=3, base_delay=1.0, max_delay=10.0, retr
                 raise
             time.sleep(min(base_delay * (2**attempt) + random.uniform(0, 1), max_delay))
 
+
 def read_json_file(path):
     if not path.exists():
         return None, f"file not found: {path.name}"
@@ -346,6 +431,7 @@ def safe_api_call(func, *args, **kwargs):
     except Exception as e:
         return None, f"API error: {e}"
 
+
 def get_field(obj, *keys, default=None):
     if not isinstance(obj, dict):
         return default
@@ -354,10 +440,12 @@ def get_field(obj, *keys, default=None):
             return obj[key]
     return default
 
+
 def get_nested_field(obj, outer_keys, inner_keys, default=None):
     outer = get_field(obj, *outer_keys) or {}
     return get_field(outer, *inner_keys, default=default) if isinstance(outer, dict) else default
 
+
 def normalize_score(score):
     if isinstance(score, bool):
         return 1.0 if score else 0.0

package/eval/scaffold/python/validation/__init__.py

@@ -8,11 +8,21 @@ from scaffold.python.validation.core import (
     compose_validators,
     run_validators,
 )
-from scaffold.python.validation.docker import (
-    check_code_execution,
-    check_python_execution,
-    check_typescript_execution,
-)
+
+_DOCKER_EXPORTS = {
+    "check_code_execution",
+    "check_python_execution",
+    "check_typescript_execution",
+}
+
+
+def __getattr__(name: str):
+    """Load host-only Docker helpers only when callers explicitly request them."""
+    if name not in _DOCKER_EXPORTS:
+        raise AttributeError(name)
+    from scaffold.python.validation import docker
+
+    return getattr(docker, name)
 
 __all__ = [
     "ValidatorFn", "compose_validators", "run_validators",

package/eval/scaffold/python/validation/comet_workflow.py

@@ -97,23 +97,28 @@ def check_workflow_phases() -> dict:
 
     # Phase 1 (open): proposal/tasks
     if _glob_exists("openspec/changes/**/proposal.md") or _glob_exists("openspec/changes/**/tasks.md"):
-        evidence += 1; found.append("open")
+        evidence += 1
+        found.append("open")
     # Phase 2 (design): design.md or docs/superpowers/specs/
     if _glob_exists("openspec/changes/**/design.md") or _glob_exists("docs/superpowers/specs/*.md"):
-        evidence += 1; found.append("design")
+        evidence += 1
+        found.append("design")
     # Phase 3 (build): plan.md, docs/superpowers/plans/, or .comet/ handoff
     if _glob_exists("openspec/changes/**/plan.md") or _glob_exists("docs/superpowers/plans/*.md") or _glob_exists("openspec/changes/**/.comet/"):
-        evidence += 1; found.append("build")
+        evidence += 1
+        found.append("build")
     # Phase 4 (verify): verification report or docs/superpowers/reports/
     if (
         _glob_exists("openspec/changes/**/verification.md")
         or _glob_exists("openspec/changes/**/verification-report.md")
         or _glob_exists("docs/superpowers/reports/*.md")
     ):
-        evidence += 1; found.append("verify")
+        evidence += 1
+        found.append("verify")
     # Phase 5 (archive): openspec/changes/archive/
     if (WORKSPACE / "openspec" / "changes" / "archive").exists():
-        evidence += 1; found.append("archive")
+        evidence += 1
+        found.append("archive")
 
     if evidence >= 4:
         return _passed("workflow_phases", f"{evidence}/5 phases ({','.join(found)})")

package/eval/scaffold/python/validation/generic_rubric.py

@@ -75,11 +75,10 @@ def _score_instruction_following(outputs: dict[str, Any]) -> tuple[float, str]:
 
 def _score_interaction_compliance(outputs: dict[str, Any]) -> tuple[float, str]:
     interaction = outputs.get("interaction") or {}
-    events = outputs.get("events") or {}
     if interaction.get("mode") != "auto_user":
         return 1.0, "single-turn or no simulator"
     max_turns = int(interaction.get("max_turns") or 12)
-    turns = int(events.get("num_turns") or 0)
+    turns = int(interaction.get("actual_turns") or 0)
     if turns <= max_turns:
         return 1.0, f"turns={turns}, max={max_turns}"
     return 0.0, f"turns={turns}, max={max_turns}"