harness-evolver 2.9.0 → 3.0.0

package/tools/eval_llm_judge.py

@@ -1,233 +0,0 @@
-#!/usr/bin/env python3
-"""LLM-as-judge evaluation script for Harness Evolver.
-
-Scores harness outputs using an LLM judge across multiple quality dimensions:
-accuracy, completeness, relevance, no_hallucination.
-
-CLI interface matches existing evals: --results-dir, --tasks-dir, --scores.
-Stdlib-only. No external dependencies.
-"""
-
-import argparse
-import json
-import os
-import re
-import sys
-
-sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-from llm_api import detect_provider, call_llm
-
-DIMENSIONS = ["accuracy", "completeness", "relevance", "no_hallucination"]
-
-WEIGHTS = {
-    "accuracy": 0.4,
-    "completeness": 0.2,
-    "relevance": 0.2,
-    "no_hallucination": 0.2,
-}
-
-
-def build_judge_prompt(task, result):
-    """Build the evaluation prompt for the LLM judge."""
-    prompt_parts = [
-        "You are an expert evaluator. Assess the quality of the following output.",
-        "",
-        "QUESTION/INPUT:",
-        str(task.get("input", "")),
-        "",
-        "OUTPUT TO EVALUATE:",
-        str(result.get("output", "")),
-    ]
-
-    if "expected" in task:
-        prompt_parts.extend([
-            "",
-            "REFERENCE ANSWER:",
-            str(task["expected"]),
-        ])
-
-    prompt_parts.extend([
-        "",
-        "Score each dimension from 1 (worst) to 5 (best):",
-        "- accuracy: Is the output factually correct and properly addresses the input?",
-        "- completeness: Does it cover all relevant aspects?",
-        "- relevance: Is it focused and on-topic?",
-        "- no_hallucination: Does it avoid fabricating information not supported by context?",
-        "",
-        "Think step by step, then respond with ONLY this JSON:",
-        '{"reasoning": "your analysis", "accuracy": N, "completeness": N, "relevance": N, "no_hallucination": N}',
-    ])
-
-    return "\n".join(prompt_parts)
-
-
-def extract_json_scores(response):
-    """Extract scoring JSON from LLM response. Handles fenced and bare JSON."""
-    # Try direct parse
-    try:
-        data = json.loads(response.strip())
-        if "accuracy" in data:
-            return data
-    except (json.JSONDecodeError, ValueError):
-        pass
-
-    # Try extracting from markdown fences
-    fence_match = re.search(r'```(?:json)?\s*(\{.*?\})\s*```', response, re.DOTALL)
-    if fence_match:
-        try:
-            data = json.loads(fence_match.group(1))
-            if "accuracy" in data:
-                return data
-        except (json.JSONDecodeError, ValueError):
-            pass
-
-    # Try regex extraction for JSON with accuracy key
-    json_match = re.search(r'\{[^{}]*"accuracy"\s*:\s*\d[^{}]*\}', response)
-    if json_match:
-        try:
-            data = json.loads(json_match.group(0))
-            if "accuracy" in data:
-                return data
-        except (json.JSONDecodeError, ValueError):
-            pass
-
-    return None
-
-
-def normalize_score(raw_score):
-    """Normalize a 1-5 score to 0.0-1.0 range."""
-    clamped = max(1, min(5, int(raw_score)))
-    return (clamped - 1) / 4.0
-
-
-def compute_combined_score(scores_dict):
-    """Compute weighted combined score from normalized dimension scores."""
-    total = 0.0
-    for dim in DIMENSIONS:
-        total += scores_dict.get(dim, 0.0) * WEIGHTS[dim]
-    return total
-
-
-def evaluate_task(provider, api_key, model, task, result):
-    """Evaluate a single task with the LLM judge. Returns per-task score dict."""
-    prompt = build_judge_prompt(task, result)
-
-    try:
-        response = call_llm(provider, api_key, model, prompt, max_tokens=2048)
-    except Exception as e:
-        return {
-            "score": 0.0,
-            "accuracy": 1, "completeness": 1, "relevance": 1, "no_hallucination": 1,
-            "reasoning": f"LLM call failed: {e}",
-            "error": str(e),
-        }
-
-    parsed = extract_json_scores(response)
-    if parsed is None:
-        return {
-            "score": 0.0,
-            "accuracy": 1, "completeness": 1, "relevance": 1, "no_hallucination": 1,
-            "reasoning": f"Failed to parse judge response: {response[:200]}",
-            "error": "parse_failed",
-        }
-
-    # Extract raw scores
-    raw = {}
-    normalized = {}
-    for dim in DIMENSIONS:
-        raw[dim] = parsed.get(dim, 1)
-        normalized[dim] = normalize_score(raw[dim])
-
-    combined = compute_combined_score(normalized)
-
-    return {
-        "score": round(combined, 4),
-        "accuracy": raw["accuracy"],
-        "completeness": raw["completeness"],
-        "relevance": raw["relevance"],
-        "no_hallucination": raw["no_hallucination"],
-        "reasoning": parsed.get("reasoning", ""),
-    }
-
-
-def main():
-    parser = argparse.ArgumentParser(description="LLM-as-judge evaluation")
-    parser.add_argument("--results-dir", required=True,
-                        help="Directory with harness output JSON files")
-    parser.add_argument("--tasks-dir", required=True,
-                        help="Directory with task JSON files")
-    parser.add_argument("--scores", required=True,
-                        help="Output path for scores JSON")
-    args = parser.parse_args()
-
-    # Detect LLM provider
-    provider, api_key, model = detect_provider()
-
-    # Collect tasks
-    task_files = sorted(f for f in os.listdir(args.tasks_dir) if f.endswith(".json"))
-    if not task_files:
-        print(f"FAIL: no .json task files in {args.tasks_dir}", file=sys.stderr)
-        sys.exit(1)
-
-    per_task = {}
-    dimension_totals = {dim: 0.0 for dim in DIMENSIONS}
-    total_combined = 0.0
-    total_tasks = 0
-
-    for task_file in task_files:
-        # Load task
-        task_path = os.path.join(args.tasks_dir, task_file)
-        with open(task_path) as f:
-            task = json.load(f)
-        task_id = task["id"]
-
-        # Load result
-        result_path = os.path.join(args.results_dir, task_file)
-        if os.path.exists(result_path):
-            with open(result_path) as f:
-                result = json.load(f)
-        else:
-            result = {"id": task_id, "output": "", "error": "no output file"}
-
-        # Evaluate
-        task_scores = evaluate_task(provider, api_key, model, task, result)
-        per_task[task_id] = task_scores
-
-        # Accumulate
-        total_combined += task_scores["score"]
-        for dim in DIMENSIONS:
-            dimension_totals[dim] += normalize_score(task_scores[dim])
-        total_tasks += 1
-
-    # Compute averages
-    if total_tasks > 0:
-        combined_score = round(total_combined / total_tasks, 4)
-        avg_dimensions = {
-            dim: round(dimension_totals[dim] / total_tasks, 4) for dim in DIMENSIONS
-        }
-    else:
-        combined_score = 0.0
-        avg_dimensions = {dim: 0.0 for dim in DIMENSIONS}
-
-    scores = {
-        "combined_score": combined_score,
-        "eval_type": "llm-judge",
-        "judge_provider": provider,
-        "judge_model": model,
-        "dimensions": avg_dimensions,
-        "weights": WEIGHTS,
-        "total_tasks": total_tasks,
-        "per_task": per_task,
-    }
-
-    # Write scores
-    os.makedirs(os.path.dirname(os.path.abspath(args.scores)), exist_ok=True)
-    with open(args.scores, "w") as f:
-        json.dump(scores, f, indent=2)
-
-    print(f"LLM judge evaluation complete. combined_score: {combined_score} "
-          f"({total_tasks} tasks, provider: {provider}/{model})")
-
-
-if __name__ == "__main__":
-    main()

package/tools/eval_passthrough.py

@@ -1,55 +0,0 @@
-#!/usr/bin/env python3
-"""Passthrough eval — collects outputs for judge subagent scoring.
-
-When no custom eval.py exists, this is used as the default. It does NOT score
-outputs — it collects them and marks them for the judge subagent to evaluate.
-The evolve skill detects eval_type=pending-judge and spawns the judge agent.
-"""
-
-import argparse
-import json
-import os
-
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--results-dir", required=True)
-    parser.add_argument("--tasks-dir", required=True)
-    parser.add_argument("--scores", required=True)
-    args = parser.parse_args()
-
-    per_task = {}
-    for fname in sorted(os.listdir(args.tasks_dir)):
-        if not fname.endswith(".json"):
-            continue
-        with open(os.path.join(args.tasks_dir, fname)) as f:
-            task = json.load(f)
-        task_id = task["id"]
-
-        result_path = os.path.join(args.results_dir, fname)
-        output = ""
-        if os.path.exists(result_path):
-            with open(result_path) as f:
-                result = json.load(f)
-            output = str(result.get("output", ""))
-
-        per_task[task_id] = {
-            "score": -1,
-            "input": str(task.get("input", ""))[:500],
-            "output": output[:500],
-        }
-
-    scores = {
-        "combined_score": -1,
-        "eval_type": "pending-judge",
-        "total_tasks": len(per_task),
-        "per_task": per_task,
-    }
-    with open(args.scores, "w") as f:
-        json.dump(scores, f, indent=2)
-
-    print(f"Collected {len(per_task)} task outputs for judge scoring.")
-
-
-if __name__ == "__main__":
-    main()

package/tools/evaluate.py

@@ -1,255 +0,0 @@
-#!/usr/bin/env python3
-"""Evaluation orchestrator for Harness Evolver.
-
-Commands:
-    validate --harness PATH [--config PATH] [--timeout SECONDS]
-    run      --harness PATH --tasks-dir PATH --eval PATH --traces-dir PATH --scores PATH
-             [--config PATH] [--timeout SECONDS]
-
-Runs harness per task, captures traces (stdout/stderr/timing), then calls user's eval script.
-Stdlib-only. No external dependencies.
-"""
-
-import argparse
-import json
-import os
-import shutil
-import subprocess
-import sys
-import tempfile
-import time
-
-
-def _resolve_python():
-    """Resolve the Python interpreter to use for subprocesses.
-
-    Prefers the current interpreter (sys.executable) over a hardcoded 'python3'.
-    This is critical in monorepo setups where the harness may need a specific
-    venv Python (e.g. Python 3.12) while the system 'python3' is a different
-    version (e.g. 3.14) with incompatible site-packages.
-    """
-    exe = sys.executable
-    if exe and os.path.isfile(exe):
-        return exe
-    return "python3"
-
-
-def _run_harness_on_task(harness, config, task_input_path, output_path, task_traces_dir, timeout, env=None):
-    """Run the harness on a single task. Returns (success, elapsed_ms, stdout, stderr)."""
-    cmd = [_resolve_python(), harness, "--input", task_input_path, "--output", output_path]
-    if task_traces_dir:
-        extra_dir = os.path.join(task_traces_dir, "extra")
-        os.makedirs(extra_dir, exist_ok=True)
-        cmd.extend(["--traces-dir", extra_dir])
-    if config and os.path.exists(config):
-        cmd.extend(["--config", config])
-
-    start = time.time()
-    try:
-        result = subprocess.run(
-            cmd, capture_output=True, text=True, timeout=timeout, env=env,
-        )
-        elapsed_ms = (time.time() - start) * 1000
-        # Accept exit code 0 (success) or check if output file exists for non-zero exits.
-        # LLM agents with C extensions (numpy, httpx) often segfault (exit 139) during
-        # Python shutdown AFTER writing correct output.
-        success = result.returncode == 0
-        if not success and os.path.exists(output_path):
-            try:
-                with open(output_path) as f:
-                    json.load(f)
-                # Valid JSON output exists despite non-zero exit — treat as success
-                success = True
-            except (json.JSONDecodeError, OSError):
-                pass
-        return success, elapsed_ms, result.stdout, result.stderr
-    except subprocess.TimeoutExpired:
-        elapsed_ms = (time.time() - start) * 1000
-        return False, elapsed_ms, "", f"TIMEOUT after {timeout}s"
-    except Exception as e:
-        elapsed_ms = (time.time() - start) * 1000
-        return False, elapsed_ms, "", str(e)
-
-
-def cmd_validate(args):
-    harness = args.harness
-    config = getattr(args, "config", None)
-    timeout = getattr(args, "timeout", 30) or 30
-
-    if not os.path.exists(harness):
-        print(f"FAIL: harness not found: {harness}", file=sys.stderr)
-        sys.exit(1)
-
-    with tempfile.TemporaryDirectory() as tmpdir:
-        dummy_task = {"id": "validation", "input": "test input for validation", "metadata": {}}
-        input_path = os.path.join(tmpdir, "input.json")
-        output_path = os.path.join(tmpdir, "output.json")
-        with open(input_path, "w") as f:
-            json.dump(dummy_task, f)
-
-        success, elapsed, stdout, stderr = _run_harness_on_task(
-            harness, config, input_path, output_path, None, timeout=timeout,
-        )
-
-        if not success:
-            hint = ""
-            if "TIMEOUT" in stderr:
-                hint = (f"\nHint: validation timed out after {timeout}s. "
-                        "For LLM-powered agents that make real API calls, "
-                        "use --timeout to increase the limit: "
-                        f"evaluate.py validate --harness {harness} --timeout 120")
-            print(f"FAIL: harness exited with error.\nstderr: {stderr}{hint}", file=sys.stderr)
-            sys.exit(1)
-
-        if not os.path.exists(output_path):
-            print("FAIL: harness did not create output file.", file=sys.stderr)
-            sys.exit(1)
-
-        try:
-            with open(output_path) as f:
-                output = json.load(f)
-        except (json.JSONDecodeError, ValueError) as e:
-            print(f"FAIL: output is not valid JSON: {e}", file=sys.stderr)
-            sys.exit(1)
-
-        if "id" not in output or "output" not in output:
-            print(f"FAIL: output missing 'id' or 'output' fields. Got: {output}", file=sys.stderr)
-            sys.exit(1)
-
-        print(f"OK: harness validated in {elapsed:.0f}ms. Output: {output}")
-
-
-def cmd_run(args):
-    harness = args.harness
-    config = getattr(args, "config", None)
-    tasks_dir = args.tasks_dir
-    eval_script = getattr(args, "eval")
-    traces_dir = args.traces_dir
-    scores_path = args.scores
-    timeout = args.timeout
-
-    os.makedirs(traces_dir, exist_ok=True)
-
-    task_files = sorted(f for f in os.listdir(tasks_dir) if f.endswith(".json"))
-    if not task_files:
-        print(f"FAIL: no .json task files in {tasks_dir}", file=sys.stderr)
-        sys.exit(1)
-
-    all_stdout = []
-    all_stderr = []
-    timing = {"per_task": {}}
-    results_dir = tempfile.mkdtemp()
-
-    # LangSmith: setup auto-tracing env vars if configured
-    langsmith_env = None
-    project_config_path = os.path.join(os.path.dirname(os.path.dirname(traces_dir)), "config.json")
-    if os.path.exists(project_config_path):
-        with open(project_config_path) as f:
-            project_config = json.load(f)
-        ls = project_config.get("eval", {}).get("langsmith", {})
-        if ls.get("enabled"):
-            api_key = os.environ.get(ls.get("api_key_env", "LANGSMITH_API_KEY"), "")
-            if api_key:
-                version = os.path.basename(os.path.dirname(traces_dir))
-                ls_project = f"{ls.get('project_prefix', 'harness-evolver')}-{version}"
-                langsmith_env = {
-                    **os.environ,
-                    "LANGCHAIN_TRACING_V2": "true",
-                    "LANGCHAIN_API_KEY": api_key,
-                    "LANGCHAIN_PROJECT": ls_project,
-                }
-                # Write the project name so the evolve skill knows where to find traces
-                ls_project_file = os.path.join(os.path.dirname(os.path.dirname(traces_dir)), "langsmith_project.txt")
-                with open(ls_project_file, "w") as f:
-                    f.write(ls_project)
-
-    for task_file in task_files:
-        task_path = os.path.join(tasks_dir, task_file)
-        with open(task_path) as f:
-            task = json.load(f)
-        task_id = task["id"]
-
-        task_input = {k: v for k, v in task.items() if k != "expected"}
-
-        task_traces_dir = os.path.join(traces_dir, task_id)
-        os.makedirs(task_traces_dir, exist_ok=True)
-
-        input_path = os.path.join(task_traces_dir, "input.json")
-        with open(input_path, "w") as f:
-            json.dump(task_input, f, indent=2)
-
-        output_path = os.path.join(results_dir, task_file)
-
-        success, elapsed_ms, stdout, stderr = _run_harness_on_task(
-            harness, config, input_path, output_path, task_traces_dir, timeout,
-            env=langsmith_env,
-        )
-
-        if os.path.exists(output_path):
-            shutil.copy2(output_path, os.path.join(task_traces_dir, "output.json"))
-        else:
-            with open(os.path.join(task_traces_dir, "output.json"), "w") as f:
-                json.dump({"id": task_id, "output": "", "error": "harness failed"}, f)
-
-        timing["per_task"][task_id] = round(elapsed_ms, 1)
-        all_stdout.append(f"--- {task_id} ---\n{stdout}")
-        all_stderr.append(f"--- {task_id} ---\n{stderr}")
-
-    timing["total_ms"] = round(sum(timing["per_task"].values()), 1)
-    with open(os.path.join(traces_dir, "timing.json"), "w") as f:
-        json.dump(timing, f, indent=2)
-    with open(os.path.join(traces_dir, "stdout.log"), "w") as f:
-        f.write("\n".join(all_stdout))
-    with open(os.path.join(traces_dir, "stderr.log"), "w") as f:
-        f.write("\n".join(all_stderr))
-
-    eval_cmd = [
-        _resolve_python(), eval_script,
-        "--results-dir", results_dir,
-        "--tasks-dir", tasks_dir,
-        "--scores", scores_path,
-    ]
-    result = subprocess.run(eval_cmd, capture_output=True, text=True, timeout=120)
-    if result.returncode != 0:
-        print(f"FAIL: eval script failed.\nstderr: {result.stderr}", file=sys.stderr)
-        sys.exit(1)
-
-    if os.path.exists(scores_path):
-        scores = json.load(open(scores_path))
-        print(f"Evaluation complete. combined_score: {scores.get('combined_score', 'N/A')}")
-    else:
-        print("WARNING: eval script did not produce scores file.", file=sys.stderr)
-
-
-def main():
-    parser = argparse.ArgumentParser(description="Harness Evolver evaluation orchestrator")
-    sub = parser.add_subparsers(dest="command")
-
-    p_val = sub.add_parser("validate")
-    p_val.add_argument("--harness", required=True)
-    p_val.add_argument("--config", default=None)
-    p_val.add_argument("--timeout", type=int, default=30,
-                       help="Validation timeout in seconds (default: 30). "
-                            "Increase for LLM-powered agents that make real API calls.")
-
-    p_run = sub.add_parser("run")
-    p_run.add_argument("--harness", required=True)
-    p_run.add_argument("--config", default=None)
-    p_run.add_argument("--tasks-dir", required=True)
-    p_run.add_argument("--eval", required=True)
-    p_run.add_argument("--traces-dir", required=True)
-    p_run.add_argument("--scores", required=True)
-    p_run.add_argument("--timeout", type=int, default=60)
-
-    args = parser.parse_args()
-    if args.command == "validate":
-        cmd_validate(args)
-    elif args.command == "run":
-        cmd_run(args)
-    else:
-        parser.print_help()
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    main()