@rpamis/comet 0.4.0-beta.1 → 0.4.0-beta.11

package/eval/scaffold/python/validation/runner.py

@@ -0,0 +1,237 @@
+"""Test runner for validation scripts.
+
+Handles all boilerplate so contributors just write check functions.
+Each check function receives a TestRunner and must call runner.passed()
+or runner.failed() — returning without calling either is an error.
+
+Example:
+
+    from scaffold.python.validation.runner import TestRunner
+
+    def check_has_function(runner):
+        source = runner.read("agent.py")
+        if "def my_function" in source:
+            runner.passed("has my_function")
+        else:
+            runner.failed("missing my_function")
+
+    def check_runs(runner):
+        output = runner.execute("agent.py")
+        if output is not None:
+            runner.passed("produced output")
+        else:
+            runner.failed("execution failed")
+
+    if __name__ == "__main__":
+        TestRunner.run([check_has_function, check_runs])
+"""
+
+import importlib.util
+import json
+import os
+import subprocess
+import sys
+from contextlib import contextmanager
+from pathlib import Path
+from types import ModuleType
+
+from scaffold.python.validation.core import (
+    TEST_CONTEXT_FILE,
+    load_test_context,
+    write_test_results,
+)
+
+
+class TestRunner:
+    """Runner that handles boilerplate for validation test scripts.
+
+    Properties:
+        runner.artifacts  — target artifact paths (from task.toml via _test_context.json)
+        runner.context    — full run context dict (run_id, events, langsmith_env, etc.)
+
+    Methods:
+        runner.passed(msg)           — record a passing check
+        runner.failed(msg)           — record a failing check
+        runner.read(path)            — read a file's contents (returns "" if not found)
+        runner.execute(path, ...)    — run a file in a subprocess, return stdout
+    """
+
+    def __init__(self):
+        self.context = load_test_context()
+        self.artifacts: list[str] = self.context.get("target_artifacts", [])
+        self._passed: list[str] = []
+        self._failed: list[str] = []
+        self._error: str | None = None
+        self._check_called = False
+        self._module_cache: dict[str, ModuleType | None] = {}  # instance-level cache
+
+    def passed(self, message: str) -> None:
+        """Record a passing check."""
+        self._check_called = True
+        self._passed.append(message)
+
+    def failed(self, message: str) -> None:
+        """Record a failing check."""
+        self._check_called = True
+        self._failed.append(message)
+
+    def read(self, path: str) -> str:
+        """Read a file's contents. Returns empty string if not found."""
+        p = Path(path)
+        if p.is_file():
+            try:
+                return p.read_text()
+            except Exception:
+                return ""
+        return ""
+
+    def load_module(self, path: str) -> ModuleType | None:
+        """Import a Python file as a module. Returns the module or None.
+
+        Cached per path — calling with the same path returns the same module
+        (avoids re-executing side effects). On first failure, records a failed
+        check and caches None. Subsequent calls return None and record a
+        cached failure (so the calling check satisfies the passed/failed requirement).
+        """
+        target = Path(path).resolve()
+        cache_key = str(target)
+        if cache_key in self._module_cache:
+            cached = self._module_cache[cache_key]
+            if cached is None:
+                # Already failed — record silently so each check shows the failure
+                self.failed(f"import failed: {path} (cached)")
+            return cached
+        if not target.is_file():
+            self.failed(f"cannot load {path}: file not found")
+            self._module_cache[cache_key] = None
+            return None
+        try:
+            module_name = f"_bench_{target.stem}_{id(target)}"
+            spec = importlib.util.spec_from_file_location(module_name, str(target))
+            if spec is None or spec.loader is None:
+                self.failed(f"cannot load {path}: invalid module spec")
+                self._module_cache[cache_key] = None
+                return None
+            module = importlib.util.module_from_spec(spec)
+            spec.loader.exec_module(module)
+            self._module_cache[cache_key] = module
+            return module
+        except Exception as e:
+            self.failed(f"import error ({path}): {e}")
+            return None
+
+    def execute(self, path: str, args: list[str] | None = None, timeout: int = 30) -> str | None:
+        """Run a file in a subprocess (inside the Docker container).
+
+        Returns stdout on success, stdout+stderr on non-zero exit,
+        or None on failure (also records a failed check).
+        """
+        target = Path(path)
+        if not target.exists():
+            self.failed(f"cannot execute {path}: file not found")
+            return None
+        cmd = [sys.executable, str(target)] + (args or [])
+        try:
+            result = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
+            if result.returncode != 0:
+                return result.stdout + result.stderr
+            return result.stdout
+        except subprocess.TimeoutExpired:
+            self.failed(f"execution timed out ({timeout}s)")
+            return None
+        except Exception as e:
+            self.failed(f"execution error: {e}")
+            return None
+
+    def _results(self) -> dict:
+        return {
+            "passed": self._passed,
+            "failed": self._failed,
+            "error": self._error,
+        }
+
+    def _run_check_traced(self, check_fn, check_name: str):
+        """Run a check function, wrapping in a trace if langsmith is available.
+
+        Traces go to 'evaluators' project to avoid polluting the data project
+        (bench-project-{uuid}) where test traces live.
+        """
+        passed_before = len(self._passed)
+        failed_before = len(self._failed)
+
+        def _run():
+            check_fn(self)
+            if not self._check_called:
+                self._failed.append(f"{check_name}: check did not call passed() or failed()")
+
+        try:
+            from langsmith.run_helpers import trace as ls_trace
+
+            with ls_trace(name=check_name, run_type="chain", project_name="evaluators") as run:
+                _run()
+                if run:
+                    run.outputs = {
+                        "passed": self._passed[passed_before:],
+                        "failed": self._failed[failed_before:],
+                    }
+        except ImportError:
+            _run()
+
+    @staticmethod
+    @contextmanager
+    def _eval_trace_context():
+        """Create a LangSmith trace context if eval trace env vars are set.
+
+        Uses distributed tracing headers to nest LLM calls
+        (e.g. evaluate_with_schema) under the host-side eval span.
+        """
+        trace_header = os.environ.get("BENCH_EVAL_LANGSMITH_TRACE")
+        baggage = os.environ.get("BENCH_EVAL_BAGGAGE")
+
+        if not trace_header:
+            yield
+            return
+
+        try:
+            import langsmith
+
+            headers: dict[str, str] = {"langsmith-trace": trace_header}
+            if baggage:
+                headers["baggage"] = baggage
+            with langsmith.tracing_context(parent=headers):
+                yield
+        except Exception:
+            # Tracing failed — run checks without tracing
+            yield
+
+    @staticmethod
+    def run(checks: list) -> None:
+        """Run check functions and handle all output.
+
+        Each check function receives a TestRunner instance and MUST call
+        runner.passed() or runner.failed() at least once. Not calling
+        either is treated as an error.
+
+        Args:
+            checks: List of check functions. Function name is used as
+                    the check name (underscores replaced with spaces).
+        """
+        runner = TestRunner()
+
+        if not runner.context:
+            print(f"Error: {TEST_CONTEXT_FILE} not found or empty", file=sys.stderr)
+            sys.exit(1)
+
+        with TestRunner._eval_trace_context():
+            for check_fn in checks:
+                check_name = check_fn.__name__.replace("check_", "").replace("_", " ")
+                runner._check_called = False
+                try:
+                    runner._run_check_traced(check_fn, check_name)
+                except Exception as e:
+                    runner._failed.append(f"{check_name}: {e}")
+
+        results = runner._results()
+        print(json.dumps(results, indent=2))
+        write_test_results(results)
+        sys.exit(1 if results["failed"] else 0)

package/eval/scaffold/python/validation/scripts.py

@@ -0,0 +1,58 @@
+"""Skill tool usage validation.
+
+Tracks which skill tools (langsmith CLI commands) Claude used during a task.
+This is informational - doesn't fail, just records patterns.
+"""
+
+# Known langsmith CLI subcommands
+CLI_COMMANDS = [
+    "langsmith trace",
+    "langsmith run",
+    "langsmith dataset",
+    "langsmith example",
+    "langsmith evaluator",
+    "langsmith experiment",
+    "langsmith thread",
+    "langsmith project",
+]
+
+
+def check_skill_scripts(
+    outputs: dict,
+    events: dict | None = None,
+    cli_commands: list[str] | None = None,
+) -> tuple[list[str], list[str]]:
+    """Track which langsmith CLI commands Claude used.
+
+    This validator doesn't fail - it just records usage patterns for analysis.
+
+    Args:
+        outputs: Outputs dict (stores CLI usage for later analysis)
+        events: Events dict containing commands_run and files_read
+        cli_commands: CLI command patterns to look for
+
+    Returns:
+        (passed, failed) lists - never fails, only passes
+    """
+    passed, failed = [], []
+    events = events or {}
+    cli_commands = cli_commands or CLI_COMMANDS
+
+    commands = " ".join(events.get("commands_run", [])).lower()
+    files_read = " ".join(events.get("files_read", [])).lower()
+    all_activity = commands + " " + files_read
+
+    # Count CLI command usage
+    cli_used = [c for c in cli_commands if c.lower() in all_activity]
+
+    # Report findings
+    if cli_used:
+        passed.append(f"CLI: {len(cli_used)} langsmith commands used ({', '.join(cli_used)})")
+    else:
+        passed.append("CLI: no langsmith CLI commands used (Claude wrote from scratch)")
+
+    # Store in outputs for later analysis
+    if outputs is not None:
+        outputs["cli_commands_used"] = cli_used
+
+    return passed, failed

package/eval/scaffold/python/validation/tracing.py

@@ -0,0 +1,313 @@
+"""LangSmith tracing pattern validation for Python and TypeScript.
+
+Validates that code has correct LangSmith tracing patterns:
+- Imports (traceable, wrap_openai/wrapOpenAI)
+- Client wrapping
+- Function decorators/wrappers
+- LangSmith API trace verification
+"""
+
+import re
+from pathlib import Path
+
+from scaffold.python.utils import get_langsmith_client
+
+# UUID pattern for trace IDs
+UUID_PATTERN = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}", re.I)
+
+
+def _to_camel_case(snake_str: str) -> str:
+    """Convert snake_case to camelCase."""
+    components = snake_str.split("_")
+    return components[0] + "".join(x.title() for x in components[1:])
+
+
+def check_python_tracing(
+    test_dir: Path,
+    filepath: str = "backend/sql_agent.py",
+    required_functions: list[str] | None = None,
+) -> tuple[list[str], list[str]]:
+    """Validate Python LangSmith tracing patterns.
+
+    Checks:
+    - Imports traceable from langsmith
+    - Imports wrap_openai from langsmith.wrappers
+    - Wraps OpenAI client with wrap_openai()
+    - Required functions have @traceable decorator
+
+    Args:
+        test_dir: Test working directory
+        filepath: Relative path to Python file
+        required_functions: Function names that must have @traceable
+
+    Returns:
+        (passed, failed) lists
+    """
+    passed, failed = [], []
+    path = test_dir / filepath
+
+    if not path.exists():
+        return [], [f"Python: {filepath} not found"]
+
+    content = path.read_text()
+
+    # Check imports
+    if re.search(r"from\s+langsmith\s+import\s+.*traceable", content, re.IGNORECASE):
+        passed.append("Python: imports traceable")
+    else:
+        failed.append("Python: missing 'from langsmith import traceable'")
+
+    if re.search(r"from\s+langsmith\.wrappers\s+import\s+wrap_openai", content, re.IGNORECASE):
+        passed.append("Python: imports wrap_openai")
+    else:
+        failed.append("Python: missing 'from langsmith.wrappers import wrap_openai'")
+
+    # Check client wrapping
+    if re.search(r"wrap_openai\s*\(\s*OpenAI\s*\(\s*\)\s*\)", content):
+        passed.append("Python: wraps OpenAI client")
+    else:
+        failed.append("Python: missing 'wrap_openai(OpenAI())'")
+
+    # Check functions are decorated
+    if required_functions:
+        traced, untraced = [], []
+        for func in required_functions:
+            pattern = rf"@traceable[^@]*def\s+{func}\s*\("
+            if re.search(pattern, content, re.DOTALL):
+                traced.append(func)
+            elif re.search(rf"def\s+{func}\s*\(", content):
+                untraced.append(func)
+
+        if traced:
+            passed.append(f"Python: traced {len(traced)} functions ({', '.join(traced)})")
+        if untraced:
+            failed.append(f"Python: missing @traceable on: {', '.join(untraced)}")
+
+    return passed, failed
+
+
+def check_typescript_tracing(
+    test_dir: Path,
+    filepath: str = "frontend/support_bot.ts",
+    required_functions: list[str] | None = None,
+) -> tuple[list[str], list[str]]:
+    """Validate TypeScript LangSmith tracing patterns.
+
+    Checks:
+    - Imports traceable from langsmith/traceable
+    - Imports wrapOpenAI from langsmith/wrappers
+    - Wraps OpenAI client with wrapOpenAI()
+    - Required functions are wrapped with traceable()
+
+    Args:
+        test_dir: Test working directory
+        filepath: Relative path to TypeScript file
+        required_functions: Function names that must be wrapped with traceable()
+
+    Returns:
+        (passed, failed) lists
+    """
+    passed, failed = [], []
+    path = test_dir / filepath
+
+    if not path.exists():
+        return [], [f"TypeScript: {filepath} not found"]
+
+    content = path.read_text()
+
+    # Check imports
+    if re.search(
+        r'import\s+\{[^}]*traceable[^}]*\}\s+from\s+["\']langsmith/traceable["\']', content
+    ):
+        passed.append("TypeScript: imports traceable")
+    else:
+        failed.append("TypeScript: missing 'import { traceable } from \"langsmith/traceable\"'")
+
+    if re.search(
+        r'import\s+\{[^}]*wrapOpenAI[^}]*\}\s+from\s+["\']langsmith/wrappers["\']', content
+    ):
+        passed.append("TypeScript: imports wrapOpenAI")
+    else:
+        failed.append("TypeScript: missing 'import { wrapOpenAI } from \"langsmith/wrappers\"'")
+
+    # Check client wrapping
+    if re.search(r"wrapOpenAI\s*\(\s*new\s+OpenAI\s*\(\s*\)\s*\)", content):
+        passed.append("TypeScript: wraps OpenAI client")
+    else:
+        failed.append("TypeScript: missing 'wrapOpenAI(new OpenAI())'")
+
+    # Check functions are wrapped
+    if required_functions:
+        traced, untraced = [], []
+        for func in required_functions:
+            camel = _to_camel_case(func)
+            patterns = [
+                rf"const\s+{camel}\s*=\s*traceable\s*\(",
+                rf"const\s+{func}\s*=\s*traceable\s*\(",
+                rf'name\s*:\s*["\']{func}["\']',
+            ]
+            if any(re.search(p, content) for p in patterns):
+                traced.append(func)
+            else:
+                func_patterns = [
+                    rf"(const|let|function)\s+{camel}\s*[=\(]",
+                    rf"async\s+function\s+{camel}\s*\(",
+                ]
+                if any(re.search(p, content) for p in func_patterns):
+                    untraced.append(func)
+
+        if traced:
+            passed.append(f"TypeScript: traced {len(traced)} functions ({', '.join(traced)})")
+        if untraced:
+            failed.append(f"TypeScript: missing traceable() on: {', '.join(untraced)}")
+
+    return passed, failed
+
+
+def check_language_syntax(
+    test_dir: Path,
+    python_file: str = "backend/sql_agent.py",
+    typescript_file: str = "frontend/support_bot.ts",
+) -> tuple[list[str], list[str]]:
+    """Validate that each file uses correct language syntax (no mixing).
+
+    Checks that Python files don't have TypeScript patterns and vice versa.
+
+    Args:
+        test_dir: Test working directory
+        python_file: Relative path to Python file
+        typescript_file: Relative path to TypeScript file
+
+    Returns:
+        (passed, failed) lists
+    """
+    passed, failed = [], []
+
+    # Python-only patterns (shouldn't appear in TypeScript)
+    py_only_patterns = [
+        (re.compile(r"^def\s+\w+\s*\(", re.MULTILINE), "Python def"),
+        (re.compile(r"^@\w+", re.MULTILINE), "Python decorator"),
+    ]
+
+    # TypeScript-only patterns (shouldn't appear in Python)
+    ts_only_patterns = [
+        (re.compile(r":\s*(string|number|boolean|Promise<)"), "TypeScript type annotation"),
+        (re.compile(r"^(const|let)\s+\w+\s*=", re.MULTILINE), "TypeScript const/let"),
+        (re.compile(r"async\s+\([^)]*\)\s*=>"), "TypeScript async arrow"),
+    ]
+
+    # Check Python file doesn't have TypeScript patterns
+    py_path = test_dir / python_file
+    if py_path.exists():
+        content = py_path.read_text()
+        ts_found = [desc for pattern, desc in ts_only_patterns if pattern.search(content)]
+        if ts_found:
+            failed.append(f"Python: contains TypeScript syntax ({len(ts_found)} patterns)")
+        else:
+            passed.append("Python: correct syntax")
+
+    # Check TypeScript file doesn't have Python patterns
+    ts_path = test_dir / typescript_file
+    if ts_path.exists():
+        content = ts_path.read_text()
+        py_found = [desc for pattern, desc in py_only_patterns if pattern.search(content)]
+        if py_found:
+            failed.append(f"TypeScript: contains Python syntax ({len(py_found)} patterns)")
+        else:
+            passed.append("TypeScript: correct syntax")
+
+    return passed, failed
+
+
+def check_langsmith_trace(
+    test_dir: Path,
+    outputs: dict,
+    trace_id_file: str = "trace_id.txt",
+    expected_functions: list[str] | None = None,
+) -> tuple[list[str], list[str]]:
+    """Validate that LangSmith traces were created correctly.
+
+    Reads trace IDs from file and verifies:
+    1. Traces exist in LangSmith
+    2. Traces have child runs for traced functions
+    3. Child run names match expected function names
+
+    Args:
+        test_dir: Test working directory
+        outputs: Outputs dict (can store trace_ids and child_run_names)
+        trace_id_file: Name of file containing trace ID(s)
+        expected_functions: Optional list of expected traced function names
+
+    Returns:
+        (passed, failed) lists
+    """
+    passed, failed = [], []
+
+    # Read trace ID from file
+    trace_file = test_dir / trace_id_file
+    if not trace_file.exists():
+        failed.append(f"LangSmith: {trace_id_file} not found")
+        return passed, failed
+
+    content = trace_file.read_text().strip()
+    if not content:
+        failed.append(f"LangSmith: {trace_id_file} is empty")
+        return passed, failed
+
+    # Extract all UUIDs from the content
+    trace_ids = UUID_PATTERN.findall(content)
+    if not trace_ids:
+        failed.append(f"LangSmith: no valid trace IDs found in {trace_id_file}")
+        return passed, failed
+
+    passed.append(f"LangSmith: found {len(trace_ids)} trace ID(s)")
+
+    # Get LangSmith client
+    client, error = get_langsmith_client()
+    if not client:
+        failed.append(f"LangSmith: client error: {error}")
+        return passed, failed
+
+    # Validate each trace
+    all_child_names = []
+    for trace_id in trace_ids:
+        try:
+            run = client.read_run(trace_id)
+            passed.append(f"LangSmith: trace {trace_id[:8]}... exists (name: {run.name})")
+
+            # Check run type
+            if run.run_type == "chain":
+                passed.append(f"LangSmith: {trace_id[:8]}... has run_type='chain'")
+
+            # Check for child runs
+            child_runs = list(client.list_runs(parent_run_id=trace_id, limit=20))
+            if child_runs:
+                child_names = [r.name for r in child_runs]
+                all_child_names.extend(child_names)
+                passed.append(f"LangSmith: {trace_id[:8]}... has {len(child_runs)} child runs")
+            else:
+                failed.append(f"LangSmith: {trace_id[:8]}... has no child runs")
+
+        except Exception as e:
+            failed.append(f"LangSmith: trace {trace_id[:8]}... error: {str(e)[:50]}")
+
+    # Check for expected function names
+    if expected_functions:
+        # Include camelCase variants
+        expected_set = set(expected_functions)
+        for func in expected_functions:
+            camel = "".join(
+                word.title() if i > 0 else word for i, word in enumerate(func.split("_"))
+            )
+            expected_set.add(camel)
+
+        found_funcs = [n for n in all_child_names if n in expected_set]
+        if found_funcs:
+            passed.append(f"LangSmith: found traced functions: {', '.join(set(found_funcs))}")
+
+    # Store trace info in outputs
+    if outputs is not None:
+        outputs["trace_ids"] = trace_ids
+        outputs["child_run_names"] = all_child_names
+
+    return passed, failed