@rpamis/comet 0.4.0-beta.3 → 0.4.0-beta.5

package/eval/scaffold/python/validation/core.py

@@ -0,0 +1,264 @@
+"""Core validation utilities.
+
+Basic utilities for file and pattern validation that can be composed together.
+"""
+
+import json
+import os
+import re
+from collections.abc import Callable
+from pathlib import Path
+
+# Type alias for validator functions
+ValidatorFn = Callable[[Path, dict], tuple[list[str], list[str]]]
+
+# Reserved filenames for host ↔ Docker data transport.
+# Configurable via environment variables; defaults match the convention.
+# BENCH_TEST_CONTEXT: host writes run metadata (run_id, events, etc.) for test scripts to read
+# BENCH_TEST_RESULTS: test scripts write validation results (passed/failed) for host to read
+TEST_CONTEXT_FILE = os.environ.get("BENCH_TEST_CONTEXT", "_test_context.json")
+TEST_RESULTS_FILE = os.environ.get("BENCH_TEST_RESULTS", "_test_results.json")
+
+
+def load_test_context(path: str = TEST_CONTEXT_FILE) -> dict:
+    """Load test context (run_id, events, langsmith_env, etc.) written by the host.
+
+    Returns empty dict if file not found (e.g. running outside factory).
+    """
+    try:
+        return json.loads(Path(path).read_text())
+    except (FileNotFoundError, json.JSONDecodeError):
+        return {}
+
+
+def write_test_results(results: dict, path: str = TEST_RESULTS_FILE) -> None:
+    """Write validation results (passed/failed lists) for the host to read."""
+    with open(path, "w") as f:
+        json.dump(results, f)
+
+
+def check_file_exists(test_dir: Path, filepath: str) -> tuple[list[str], list[str]]:
+    """Check that a file exists.
+
+    Args:
+        test_dir: Test working directory
+        filepath: Relative path to file
+
+    Returns:
+        (passed, failed) lists
+    """
+    path = test_dir / filepath
+    if path.exists():
+        return [f"File exists: {filepath}"], []
+    return [], [f"File missing: {filepath}"]
+
+
+def check_pattern(
+    filepath: Path,
+    pattern: str,
+    description: str,
+    flags: int = 0,
+) -> tuple[list[str], list[str]]:
+    """Check that a file contains a regex pattern.
+
+    Args:
+        filepath: Path to file
+        pattern: Regex pattern to search for
+        description: Human-readable description of what we're checking
+        flags: Regex flags (e.g., re.MULTILINE)
+
+    Returns:
+        (passed, failed) lists
+    """
+    if not filepath.exists():
+        return [], [f"{description}: file not found ({filepath.name})"]
+
+    content = filepath.read_text()
+    if re.search(pattern, content, flags):
+        return [description], []
+    return [], [f"Missing: {description}"]
+
+
+def check_no_pattern(
+    filepath: Path,
+    pattern: str,
+    description: str,
+    flags: int = 0,
+) -> tuple[list[str], list[str]]:
+    """Check that a file does NOT contain a regex pattern.
+
+    Args:
+        filepath: Path to file
+        pattern: Regex pattern that should NOT be present
+        description: Human-readable description of what we're checking
+        flags: Regex flags
+
+    Returns:
+        (passed, failed) lists
+    """
+    if not filepath.exists():
+        return [], [f"{description}: file not found ({filepath.name})"]
+
+    content = filepath.read_text()
+    if re.search(pattern, content, flags):
+        return [], [f"Unexpected: {description}"]
+    return [f"No {description}"], []
+
+
+def compose_validators(*validators: ValidatorFn) -> ValidatorFn:
+    """Compose multiple validator functions into one.
+
+    Args:
+        *validators: Validator functions to compose
+
+    Returns:
+        A single validator function that runs all validators
+    """
+
+    def composed(test_dir: Path, outputs: dict) -> tuple[list[str], list[str]]:
+        all_passed, all_failed = [], []
+        for validator in validators:
+            passed, failed = validator(test_dir, outputs)
+            all_passed.extend(passed)
+            all_failed.extend(failed)
+        return all_passed, all_failed
+
+    return composed
+
+
+def run_validators(
+    validators: list[ValidatorFn],
+    test_dir: Path,
+    outputs: dict,
+) -> tuple[list[str], list[str]]:
+    """Run a list of validator functions.
+
+    Args:
+        validators: List of validator functions
+        test_dir: Test working directory
+        outputs: Additional outputs dict
+
+    Returns:
+        Combined (passed, failed) lists
+    """
+    all_passed, all_failed = [], []
+    for validator in validators:
+        passed, failed = validator(test_dir, outputs)
+        all_passed.extend(passed)
+        all_failed.extend(failed)
+    return all_passed, all_failed
+
+
+def check_starter_skill_first(
+    outputs: dict,
+) -> tuple[list[str], list[str]]:
+    """Check that a starter skill (langchain-oss-primer, framework-selection, or ecosystem-primer) was invoked first.
+
+    Skipped (informational) when:
+    - No skills were invoked (e.g. CONTROL treatment)
+    - Treatment is ALL_MAIN_SKILLS (starter skills not included in that treatment)
+    """
+    # Skip for ALL_MAIN_SKILLS — starter skills are not part of that treatment
+    treatment_name = (outputs or {}).get("treatment_name", "")
+    if treatment_name == "ALL_MAIN_SKILLS":
+        return ["Note: starter skill check skipped (ALL_MAIN_SKILLS)"], []
+
+    events = outputs.get("events", {}) if outputs else {}
+    skills_invoked = events.get("skills_invoked", [])
+
+    # No skills invoked at all (e.g. CONTROL treatment) — nothing to check
+    if not skills_invoked:
+        return ["Note: no skills invoked"], []
+
+    starter_skills = {"langchain-oss-primer", "framework-selection", "ecosystem-primer"}
+
+    if skills_invoked[0] in starter_skills:
+        return [f"Starter skill invoked first: {skills_invoked[0]}"], []
+
+    first = skills_invoked[0]
+    invoked_starters = [s for s in skills_invoked if s in starter_skills]
+    if invoked_starters:
+        starters = ", ".join(invoked_starters)
+        return [], [
+            f"Starter skill not invoked first: first was '{first}', starters invoked later: {starters}"
+        ]
+    return [], [
+        f"Starter skill not invoked: first skill was '{first}' (expected langchain-oss-primer, framework-selection, or ecosystem-primer)"
+    ]
+
+
+def check_skill_invoked(
+    outputs: dict,
+    skill_name: str,
+    required: bool = False,
+) -> tuple[list[str], list[str]]:
+    """Check if a skill was invoked during the task.
+
+    This is typically informational (required=False) to track skill usage.
+
+    Args:
+        outputs: Outputs dict containing events
+        skill_name: Name of the skill to check (e.g., "langchain-agents")
+        required: If True, failing to invoke the skill is an error
+
+    Returns:
+        (passed, failed) lists
+    """
+    events = outputs.get("events", {}) if outputs else {}
+    skills_invoked = events.get("skills_invoked", [])
+
+    if skill_name in skills_invoked:
+        return [f"Invoked {skill_name} skill"], []
+    elif required:
+        return [], [f"Did NOT invoke {skill_name} skill"]
+    else:
+        return [f"Note: did not invoke {skill_name}"], []
+
+
+# Noise task configurations
+NOISE_TASK_PROMPTS = {
+    "docker_patterns": "Create a Dockerfile for a Node.js application with multi-stage build, non-root user, and health check. Save to Dockerfile.nodejs.",
+    "react_components": "Create a React component that fetches and displays user data using hooks (useState, useEffect), with loading/error states in TypeScript. Save to UserProfile.tsx.",
+    "api_docs": "Create an OpenAPI spec for a simple user API with GET /users, POST /users, proper schemas, and error responses. Save to openapi.yaml.",
+}
+
+# Noise task deliverable files
+NOISE_TASK_DELIVERABLES = {
+    "docker_patterns": "Dockerfile.nodejs",
+    "react_components": "UserProfile.tsx",
+    "api_docs": "openapi.yaml",
+}
+
+
+def get_noise_task_prompts(noise_tasks: list[str]) -> list[str]:
+    """Get prompts for noise tasks by name.
+
+    Args:
+        noise_tasks: List of noise task names (e.g., ["docker_patterns", "react_components"])
+
+    Returns:
+        List of prompt strings for the specified tasks
+    """
+    return [NOISE_TASK_PROMPTS[name] for name in noise_tasks if name in NOISE_TASK_PROMPTS]
+
+
+def check_noise_outputs(
+    noise_tasks: list[str],
+    test_dir: Path = None,
+) -> tuple[list[str], list[str]]:
+    """Validate that noise task deliverables were created."""
+    passed, failed = [], []
+    test_dir = test_dir or Path(".")
+
+    for task_name in noise_tasks:
+        deliverable = NOISE_TASK_DELIVERABLES.get(task_name)
+        if not deliverable:
+            continue
+
+        path = test_dir / deliverable
+        if path.exists():
+            passed.append(f"Noise: {deliverable} created")
+        else:
+            failed.append(f"Noise: {deliverable} NOT created")
+
+    return passed, failed

package/eval/scaffold/python/validation/dataset.py

@@ -0,0 +1,337 @@
+"""Dataset validation utilities.
+
+Validates dataset structure, trajectory accuracy, and LangSmith uploads.
+"""
+
+from pathlib import Path
+
+from scaffold.python.utils import (
+    get_field,
+    get_langsmith_client,
+    get_nested_field,
+    read_json_file,
+    safe_api_call,
+)
+
+
+def extract_examples(data) -> list:
+    """Extract examples list from various dataset formats."""
+    if isinstance(data, list):
+        return data
+    if isinstance(data, dict):
+        return data.get("examples") or data.get("data") or [data]
+    return []
+
+
+def check_dataset_structure(
+    test_dir: Path = None,
+    outputs: dict = None,
+    filename: str = "trajectory_dataset.json",
+    min_examples: int = 1,
+    dataset_type: str = "trajectory",
+    required_fields: list[str] | None = None,
+) -> tuple[list[str], list[str]]:
+    """Validate dataset file structure.
+
+    Checks file exists, is valid JSON, has enough examples with
+    required fields, and (for trajectory type) has trajectory data.
+    """
+    test_dir = test_dir or Path(".")
+    passed, failed = [], []
+    required_fields = required_fields or ["inputs", "outputs"]
+
+    data, error = read_json_file(test_dir / filename)
+    if error:
+        return [], [f"Dataset: {error}"]
+
+    examples = extract_examples(data)
+    if len(examples) < min_examples:
+        return [f"Dataset: {filename} created"], [
+            f"Dataset: {len(examples)} examples (need {min_examples})"
+        ]
+
+    passed.append(f"Dataset: {len(examples)} examples")
+
+    # Check structure
+    sample = examples[:10]
+    valid_io = sum(1 for ex in sample if _has_io(ex, required_fields))
+    if valid_io:
+        passed.append(f"Dataset: {valid_io}/{len(sample)} have input/output")
+    else:
+        failed.append("Dataset: no input/output structure")
+
+    if dataset_type == "trajectory":
+        has_traj = sum(1 for ex in sample if _get_trajectory(ex))
+        if has_traj:
+            passed.append(f"Dataset: {has_traj}/{len(sample)} have trajectory")
+        else:
+            failed.append("Dataset: no trajectory data")
+
+    return passed, failed
+
+
+def _has_io(ex: dict, required_fields: list[str]) -> bool:
+    """Check if example has required fields."""
+    if not isinstance(ex, dict):
+        return False
+    # Support both 'inputs'/'input' and 'outputs'/'output'
+    has_input = any(f in ex or f.rstrip("s") in ex for f in required_fields if "input" in f.lower())
+    has_output = any(
+        f in ex or f.rstrip("s") in ex for f in required_fields if "output" in f.lower()
+    )
+    return has_input and has_output
+
+
+def _get_trajectory(ex: dict) -> list:
+    """Extract trajectory data, accepting various field names."""
+    if not isinstance(ex, dict):
+        return []
+
+    # Known trajectory field names
+    fields = ["expected_trajectory", "trajectory", "expected_tools", "tool_calls", "tools"]
+
+    # Check top-level
+    traj = get_field(ex, *fields)
+    if isinstance(traj, list):
+        return traj
+
+    # Check nested in outputs
+    traj = get_nested_field(ex, ["outputs", "output"], fields)
+    if isinstance(traj, list):
+        return traj
+
+    # Fallback: any list of strings in outputs
+    outputs_val = get_field(ex, "outputs", "output")
+    if isinstance(outputs_val, dict):
+        for val in outputs_val.values():
+            if isinstance(val, list) and val and isinstance(val[0], str):
+                return val
+
+    return []
+
+
+def check_dataset_upload(
+    test_dir: Path = None,
+    outputs: dict = None,
+    filename: str = "trajectory_dataset.json",
+    upload_prefix: str = "test-",
+) -> tuple[list[str], list[str]]:
+    """Verify dataset was uploaded to LangSmith and matches local file."""
+    test_dir = test_dir or Path(".")
+    outputs = outputs or {}
+    passed, failed = [], []
+
+    # Read local file to get example count
+    local_data, error = read_json_file(test_dir / filename)
+    if error:
+        return [], [f"Upload: local file error - {error}"]
+    local_examples = extract_examples(local_data)
+    local_count = len(local_examples)
+
+    client, error = get_langsmith_client()
+    if error:
+        return [f"Upload: skipped ({error})"], []
+
+    datasets, error = safe_api_call(lambda: list(client.list_datasets()))
+    if error:
+        return [f"Upload: {error}"], []
+
+    # Use run_id for precise matching if available
+    run_id = outputs.get("run_id") if outputs else None
+    if run_id:
+        search_pattern = f"{upload_prefix}{run_id}"
+    else:
+        search_pattern = upload_prefix
+
+    matching = [d for d in datasets if d.name.startswith(search_pattern)]
+    if not matching:
+        return [], [f"Upload: no dataset with prefix '{search_pattern}'"]
+
+    recent = max(matching, key=lambda d: getattr(d, "created_at", d.name))
+    remote_count = getattr(recent, "example_count", None)
+
+    passed.append(f"Upload: '{recent.name}' ({remote_count} examples)")
+
+    # Verify counts match
+    if remote_count is not None and remote_count != local_count:
+        failed.append(f"Upload: count mismatch (local={local_count}, remote={remote_count})")
+        return passed, failed
+
+    # Fetch remote examples and compare content
+    remote_examples, error = safe_api_call(
+        lambda: list(client.list_examples(dataset_name=recent.name))
+    )
+    if error:
+        passed.append(f"Upload: count matches ({local_count}), content check skipped")
+        return passed, failed
+
+    # Compare trajectories
+    matches = 0
+    for local_ex in local_examples:
+        local_traj = _get_trajectory(local_ex)
+        if not local_traj:
+            continue
+
+        # Find matching remote example by comparing trajectories
+        for remote_ex in remote_examples:
+            remote_outputs = getattr(remote_ex, "outputs", {}) or {}
+            remote_traj = _get_trajectory({"outputs": remote_outputs})
+            if _to_tool_names(local_traj) == _to_tool_names(remote_traj):
+                matches += 1
+                break
+
+    if matches == local_count:
+        passed.append(f"Upload: content verified ({matches}/{local_count} match)")
+    elif matches > 0:
+        failed.append(f"Upload: partial match ({matches}/{local_count} examples)")
+    else:
+        failed.append(f"Upload: content mismatch (0/{local_count} trajectories match)")
+
+    return passed, failed
+
+
+def check_trajectory_accuracy(
+    test_dir: Path = None,
+    outputs: dict = None,
+    filename: str = "trajectory_dataset.json",
+    expected_filename: str = "expected_dataset.json",
+    data_dir: Path | None = None,
+) -> tuple[list[str], list[str]]:
+    """Validate dataset content matches ground truth."""
+    test_dir = test_dir or Path(".")
+    outputs = outputs or {}
+    passed, failed = [], []
+    data_dir = data_dir or (test_dir / "data")
+
+    # Load actual dataset
+    actual_data, error = read_json_file(test_dir / filename)
+    if error:
+        return [], [f"Accuracy: {error}"]
+    actual_examples = extract_examples(actual_data)
+
+    if not actual_examples:
+        return [], ["Accuracy: no examples in dataset"]
+
+    # Load ground truth
+    expected_data, error = read_json_file(data_dir / expected_filename)
+    if error:
+        return ["Accuracy: skipped (no ground truth)"], []
+    expected_examples = (
+        expected_data.get("examples", []) if isinstance(expected_data, dict) else expected_data
+    )
+
+    if not expected_examples:
+        return ["Accuracy: skipped (empty ground truth)"], []
+
+    # Get trace_id_map if available (maps old expected IDs to new actual IDs)
+    trace_id_map = outputs.get("trace_id_map", {}) if outputs else {}
+
+    # Compare trajectories
+    matches, mismatches, missing = _compare_datasets(
+        actual_examples, expected_examples, trace_id_map
+    )
+    total_expected = len(expected_examples)
+
+    if matches == total_expected:
+        passed.append(f"Accuracy: {matches}/{total_expected} trajectories match")
+    elif matches > 0:
+        failed.append(f"Accuracy: only {matches}/{total_expected} trajectories match")
+    else:
+        failed.append(f"Accuracy: 0/{total_expected} trajectories match")
+
+    if mismatches:
+        first_mm = mismatches[0]
+        failed.append(f"Accuracy: {len(mismatches)} wrong trajectories (e.g., {first_mm})")
+
+    if missing:
+        failed.append(f"Accuracy: {len(missing)} expected traces missing")
+
+    return passed, failed
+
+
+def _compare_datasets(
+    actual: list[dict], expected: list[dict], trace_id_map: dict[str, str] = None
+) -> tuple[int, list[str], list[str]]:
+    """Compare actual dataset against expected ground truth.
+
+    Args:
+        actual: Actual dataset examples from Claude
+        expected: Expected ground truth examples
+        trace_id_map: Optional mapping from expected trace_id -> actual trace_id
+                      (used when traces are re-uploaded with new IDs)
+
+    Returns: (match_count, mismatch_details, missing_ids)
+    """
+    trace_id_map = trace_id_map or {}
+
+    def _get_trace_id(ex):
+        return ex.get("trace_id") or ex.get("id") or None
+
+    def _get_content_key(ex):
+        inputs = ex.get("inputs") or ex.get("input") or {}
+        if isinstance(inputs, dict):
+            messages = inputs.get("messages", [])
+            if messages and isinstance(messages[0], dict):
+                return messages[0].get("content", "")
+            return inputs.get("query") or inputs.get("input") or str(inputs)
+        return str(inputs)
+
+    # Index actual by trace_id and by content (fallback)
+    actual_by_trace_id = {}
+    actual_by_content = {}
+    for ex in actual:
+        tid = _get_trace_id(ex)
+        if tid:
+            actual_by_trace_id[str(tid)] = ex
+        content_key = _get_content_key(ex)
+        if content_key:
+            actual_by_content[content_key] = ex
+
+    matches, mismatches, missing = 0, [], []
+
+    for exp in expected:
+        exp_traj = _get_trajectory(exp)
+        if not exp_traj:
+            continue
+
+        # Try trace_id match first (with remapping)
+        exp_trace_id = _get_trace_id(exp)
+        actual_ex = None
+        if exp_trace_id:
+            actual_key = trace_id_map.get(str(exp_trace_id), str(exp_trace_id))
+            actual_ex = actual_by_trace_id.get(actual_key)
+
+        # Fall back to content-based matching
+        if not actual_ex:
+            exp_content = _get_content_key(exp)
+            if exp_content:
+                actual_ex = actual_by_content.get(exp_content)
+
+        if not actual_ex:
+            label = str(exp_trace_id or _get_content_key(exp) or "unknown")[:30]
+            missing.append(label)
+            continue
+
+        actual_traj = _get_trajectory(actual_ex)
+        if _to_tool_names(actual_traj) == _to_tool_names(exp_traj):
+            matches += 1
+        else:
+            label = str(exp_trace_id or _get_content_key(exp) or "unknown")[:20]
+            mismatches.append(f"'{label}...' trajectory mismatch")
+
+    return matches, mismatches, missing
+
+
+def _to_tool_names(traj: list) -> list[str]:
+    """Convert trajectory items to tool name strings."""
+    if not traj:
+        return []
+    names = []
+    for item in traj:
+        if isinstance(item, str):
+            names.append(item)
+        elif isinstance(item, dict):
+            name = item.get("name") or item.get("tool") or item.get("function")
+            if name:
+                names.append(name)
+    return names

package/eval/scaffold/python/validation/docker.py

@@ -0,0 +1,106 @@
+"""Docker-based code execution validation.
+
+Validates that code runs successfully in Docker containers.
+"""
+
+from pathlib import Path
+
+from scaffold.python.utils import run_node_in_docker, run_python_in_docker
+
+
+def check_python_execution(
+    test_dir: Path,
+    filepath: str = "backend/sql_agent.py",
+    timeout: int = 120,
+    args: list[str] | None = None,
+) -> tuple[list[str], list[str]]:
+    """Validate that Python code runs without errors in Docker.
+
+    Args:
+        test_dir: Test working directory (contains Dockerfile)
+        filepath: Relative path to Python file
+        timeout: Execution timeout in seconds
+        args: Optional command-line arguments
+
+    Returns:
+        (passed, failed) lists
+    """
+    passed, failed = [], []
+    path = test_dir / filepath
+
+    if not path.exists():
+        return [], [f"Python: {filepath} not found"]
+
+    success, output = run_python_in_docker(test_dir, filepath, timeout=timeout, args=args)
+    if success:
+        passed.append(f"Python: {filepath} executes successfully")
+    else:
+        error = output[:100] if output else "unknown error"
+        failed.append(f"Python: execution failed ({error})")
+
+    return passed, failed
+
+
+def check_typescript_execution(
+    test_dir: Path,
+    filepath: str = "frontend/support_bot.ts",
+    timeout: int = 120,
+    args: list[str] | None = None,
+) -> tuple[list[str], list[str]]:
+    """Validate that TypeScript code runs without errors in Docker.
+
+    Args:
+        test_dir: Test working directory (contains Dockerfile)
+        filepath: Relative path to TypeScript file
+        timeout: Execution timeout in seconds
+        args: Optional command-line arguments
+
+    Returns:
+        (passed, failed) lists
+    """
+    passed, failed = [], []
+    path = test_dir / filepath
+
+    if not path.exists():
+        return [], [f"TypeScript: {filepath} not found"]
+
+    success, output = run_node_in_docker(test_dir, filepath, timeout=timeout, args=args)
+    if success:
+        passed.append(f"TypeScript: {filepath} executes successfully")
+    else:
+        error = output[:100] if output else "unknown error"
+        failed.append(f"TypeScript: execution failed ({error})")
+
+    return passed, failed
+
+
+def check_code_execution(
+    test_dir: Path,
+    python_file: str = "backend/sql_agent.py",
+    typescript_file: str = "frontend/support_bot.ts",
+    timeout: int = 120,
+) -> tuple[list[str], list[str]]:
+    """Validate that both Python and TypeScript code run without errors.
+
+    Args:
+        test_dir: Test working directory (contains Dockerfile)
+        python_file: Relative path to Python file
+        typescript_file: Relative path to TypeScript file
+        timeout: Execution timeout in seconds
+
+    Returns:
+        (passed, failed) lists
+    """
+    all_passed, all_failed = [], []
+
+    # Python execution
+    py_passed, py_failed = check_python_execution(test_dir, python_file, timeout)
+    all_passed.extend(py_passed)
+    all_failed.extend(py_failed)
+
+    # TypeScript execution
+    ts_passed, ts_failed = check_typescript_execution(test_dir, typescript_file, timeout)
+    all_passed.extend(ts_passed)
+    all_failed.extend(ts_failed)
+
+    return all_passed, all_failed