harness-evolver 2.9.0 → 3.0.0

package/tools/state.py

@@ -1,219 +0,0 @@
-#!/usr/bin/env python3
-"""State manager for Harness Evolver.
-
-Commands:
-    init   --base-dir DIR --baseline-score FLOAT
-    update --base-dir DIR --version VER --scores PATH --proposal PATH
-    show   --base-dir DIR
-
-Manages: summary.json (source of truth), STATE.md (human view), PROPOSER_HISTORY.md (log).
-Stdlib-only. No external dependencies.
-"""
-
-import argparse
-import json
-import os
-import re
-import sys
-
-
-def _read_json(path):
-    with open(path) as f:
-        return json.load(f)
-
-
-def _write_json(path, data):
-    with open(path, "w") as f:
-        json.dump(data, f, indent=2)
-
-
-def _read_text(path):
-    with open(path) as f:
-        return f.read()
-
-
-def _write_text(path, text):
-    with open(path, "w") as f:
-        f.write(text)
-
-
-def _summary_path(base_dir):
-    return os.path.join(base_dir, "summary.json")
-
-
-def _state_md_path(base_dir):
-    return os.path.join(base_dir, "STATE.md")
-
-
-def _history_path(base_dir):
-    return os.path.join(base_dir, "PROPOSER_HISTORY.md")
-
-
-def _detect_parent(proposal_text, current_best):
-    """Parse 'Based on vXXX' or 'Based on baseline' from proposal text."""
-    match = re.search(r"[Bb]ased on (v\d+|baseline)", proposal_text)
-    if match:
-        return match.group(1)
-    return current_best
-
-
-def _render_state_md(summary):
-    """Generate STATE.md from summary.json data."""
-    lines = ["# Harness Evolver Status", ""]
-    best = summary["best"]
-    worst = summary["worst"]
-    lines.append(f"**Iterations:** {summary['iterations']}")
-    lines.append(f"**Best:** {best['version']} ({best['combined_score']:.2f})")
-    lines.append(f"**Worst:** {worst['version']} ({worst['combined_score']:.2f})")
-    if summary["history"]:
-        last = summary["history"][-1]
-        lines.append(f"**Latest:** {last['version']} ({last['combined_score']:.2f})")
-    lines.append("")
-    lines.append("## History")
-    lines.append("")
-    lines.append("| Version | Score | Parent | Delta |")
-    lines.append("|---------|-------|--------|-------|")
-    prev_score = None
-    for entry in summary["history"]:
-        v = entry["version"]
-        s = entry["combined_score"]
-        p = entry["parent"] or "-"
-        if prev_score is not None and v != "baseline":
-            delta = s - prev_score
-            if delta < -0.01:
-                delta_str = f"{delta:+.2f} REGRESSION"
-            elif delta > 0.01:
-                delta_str = f"{delta:+.2f}"
-            else:
-                delta_str = f"{delta:+.2f} (stagnant)"
-        else:
-            delta_str = "-"
-        lines.append(f"| {v} | {s:.2f} | {p} | {delta_str} |")
-        prev_score = s
-    return "\n".join(lines) + "\n"
-
-
-def cmd_init(args):
-    base_dir = args.base_dir
-    score = args.baseline_score
-    os.makedirs(base_dir, exist_ok=True)
-
-    summary = {
-        "iterations": 0,
-        "best": {"version": "baseline", "combined_score": score},
-        "worst": {"version": "baseline", "combined_score": score},
-        "history": [
-            {"version": "baseline", "combined_score": score, "parent": None}
-        ],
-    }
-    _write_json(_summary_path(base_dir), summary)
-    _write_text(_state_md_path(base_dir), _render_state_md(summary))
-    _write_text(_history_path(base_dir), "# Proposer History\n")
-
-
-def cmd_update(args):
-    base_dir = args.base_dir
-    version = args.version
-    scores = _read_json(args.scores)
-    proposal_text = _read_text(args.proposal) if args.proposal else ""
-
-    summary = _read_json(_summary_path(base_dir))
-    combined = scores.get("combined_score", 0.0)
-    parent = _detect_parent(proposal_text, summary["best"]["version"])
-
-    entry = {
-        "version": version,
-        "combined_score": combined,
-        "parent": parent,
-    }
-    summary["history"].append(entry)
-    summary["iterations"] = len(summary["history"]) - 1
-
-    non_baseline = [h for h in summary["history"] if h["version"] != "baseline"]
-    if non_baseline:
-        best_entry = max(non_baseline, key=lambda h: h["combined_score"])
-        worst_entry = min(non_baseline, key=lambda h: h["combined_score"])
-        summary["best"] = {
-            "version": best_entry["version"],
-            "combined_score": best_entry["combined_score"],
-        }
-        summary["worst"] = {
-            "version": worst_entry["version"],
-            "combined_score": worst_entry["combined_score"],
-        }
-
-    _write_json(_summary_path(base_dir), summary)
-    _write_text(_state_md_path(base_dir), _render_state_md(summary))
-
-    parent_score = None
-    for h in summary["history"]:
-        if h["version"] == parent:
-            parent_score = h["combined_score"]
-            break
-
-    is_regression = parent_score is not None and combined < parent_score - 0.01
-    regression_tag = " <- REGRESSION" if is_regression else ""
-
-    proposal_lines = proposal_text.strip().split("\n")
-    summary_line = ""
-    for line in proposal_lines:
-        stripped = line.strip()
-        if stripped and not re.match(r"^[Bb]ased on", stripped):
-            summary_line = stripped
-            break
-
-    history_entry = f"\n## {version} (score: {combined:.2f}){regression_tag}\n{summary_line}\n"
-    history_path = _history_path(base_dir)
-    with open(history_path, "a") as f:
-        f.write(history_entry)
-
-
-def cmd_show(args):
-    base_dir = args.base_dir
-    summary = _read_json(_summary_path(base_dir))
-    best = summary["best"]
-    worst = summary["worst"]
-
-    print(f"Harness Evolver — Iteration {summary['iterations']}")
-    print(f"Best:  {best['version']}  score: {best['combined_score']:.2f}")
-    print(f"Worst: {worst['version']}  score: {worst['combined_score']:.2f}")
-    print()
-    for entry in summary["history"]:
-        v = entry["version"]
-        s = entry["combined_score"]
-        bar_len = int(s * 30)
-        bar = "\u2588" * bar_len
-        print(f"  {v:>10}: {s:.2f} {bar}")
-
-
-def main():
-    parser = argparse.ArgumentParser(description="Harness Evolver state manager")
-    sub = parser.add_subparsers(dest="command")
-
-    p_init = sub.add_parser("init")
-    p_init.add_argument("--base-dir", required=True)
-    p_init.add_argument("--baseline-score", type=float, required=True)
-
-    p_update = sub.add_parser("update")
-    p_update.add_argument("--base-dir", required=True)
-    p_update.add_argument("--version", required=True)
-    p_update.add_argument("--scores", required=True)
-    p_update.add_argument("--proposal", default=None)
-
-    p_show = sub.add_parser("show")
-    p_show.add_argument("--base-dir", required=True)
-
-    args = parser.parse_args()
-    if args.command == "init":
-        cmd_init(args)
-    elif args.command == "update":
-        cmd_update(args)
-    elif args.command == "show":
-        cmd_show(args)
-    else:
-        parser.print_help()
-        sys.exit(1)
-
-
-if __name__ == "__main__":
-    main()

package/tools/test_growth.py

@@ -1,230 +0,0 @@
-#!/usr/bin/env python3
-"""Test Suite Growth for Harness Evolver.
-
-Generates regression test tasks when previously-failing tasks are now passing.
-Creates mechanical variations of fixed tasks to prevent future regressions.
-
-Usage:
-    python3 test_growth.py \
-        --current-scores .harness-evolver/harnesses/v003/scores.json \
-        --previous-scores .harness-evolver/harnesses/v002/scores.json \
-        --tasks-dir .harness-evolver/eval/tasks/ \
-        --output-dir .harness-evolver/eval/tasks/ \
-        --max-total-tasks 60
-
-Stdlib-only. No external dependencies.
-"""
-
-import argparse
-import json
-import os
-import re
-import sys
-
-
-def load_json(path):
-    """Load JSON file, return None if missing or invalid."""
-    if not path or not os.path.exists(path):
-        return None
-    try:
-        with open(path) as f:
-            return json.load(f)
-    except (json.JSONDecodeError, OSError):
-        return None
-
-
-def find_fixed_tasks(current_scores, previous_scores, fix_threshold_before=0.5, fix_threshold_after=0.8):
-    """Find tasks that improved significantly: score < before_threshold → > after_threshold."""
-    current_per_task = current_scores.get("per_task", {})
-    previous_per_task = previous_scores.get("per_task", {})
-
-    fixed = []
-    for tid, curr_data in current_per_task.items():
-        if not isinstance(curr_data, dict):
-            continue
-        curr_score = curr_data.get("score", 0)
-        prev_data = previous_per_task.get(tid, {})
-        prev_score = prev_data.get("score", 0) if isinstance(prev_data, dict) else 0
-
-        if prev_score < fix_threshold_before and curr_score > fix_threshold_after:
-            fixed.append({
-                "task_id": tid,
-                "previous_score": prev_score,
-                "current_score": curr_score,
-                "improvement": curr_score - prev_score,
-            })
-
-    # Sort by improvement (biggest fixes first)
-    fixed.sort(key=lambda x: -x["improvement"])
-    return fixed
-
-
-def count_existing_tasks(directory):
-    """Count existing task JSON files in directory."""
-    if not os.path.isdir(directory):
-        return 0
-    return sum(1 for f in os.listdir(directory) if f.endswith(".json"))
-
-
-def next_regression_id(output_dir):
-    """Find the next available regression task ID."""
-    existing = set()
-    if os.path.isdir(output_dir):
-        for fname in os.listdir(output_dir):
-            m = re.match(r"regression_(\d+)\.json", fname)
-            if m:
-                existing.add(int(m.group(1)))
-    n = 1
-    while n in existing:
-        n += 1
-    return n
-
-
-def generate_variations(original_input, task_id):
-    """Generate 2-3 mechanical variations of an input string.
-
-    Uses simple string transforms — no LLM needed:
-    - Rephrase by reordering
-    - Add qualifying clause
-    - Simplify to minimal form
-    """
-    variations = []
-    text = original_input.strip()
-
-    # Variation 1: Add a qualifying clause
-    qualifiers = [
-        "Please be specific and detailed in your response.",
-        "Consider edge cases in your answer.",
-        "Provide a concise but thorough response.",
-        "Think step by step before answering.",
-    ]
-    # Pick qualifier based on hash of task_id for determinism
-    qi = hash(task_id) % len(qualifiers)
-    v1 = f"{text}\n\n{qualifiers[qi]}"
-    variations.append(("qualified", v1))
-
-    # Variation 2: Reorder sentences if multiple exist
-    sentences = re.split(r"(?<=[.!?])\s+", text)
-    if len(sentences) >= 2:
-        # Swap first two sentences
-        reordered = sentences[1:] + sentences[:1]
-        v2 = " ".join(reordered)
-        variations.append(("reordered", v2))
-    else:
-        # If single sentence, prepend "Given the context: "
-        v2 = f"Given the following context, {text[0].lower()}{text[1:]}" if len(text) > 1 else text
-        variations.append(("rephrased", v2))
-
-    # Variation 3: Minimal version — strip to core question
-    # Remove qualifiers, keep just the main ask
-    minimal = text
-    # Strip common padding phrases
-    for prefix in ["Please ", "Can you ", "Could you ", "I would like you to ", "I need you to "]:
-        if minimal.startswith(prefix):
-            minimal = minimal[len(prefix):]
-            minimal = minimal[0].upper() + minimal[1:] if minimal else minimal
-            break
-    if minimal != text:
-        variations.append(("minimal", minimal))
-
-    return variations
-
-
-def main():
-    parser = argparse.ArgumentParser(description="Generate regression test tasks from score improvements")
-    parser.add_argument("--current-scores", required=True, help="Path to current version's scores.json")
-    parser.add_argument("--previous-scores", required=True, help="Path to previous version's scores.json")
-    parser.add_argument("--tasks-dir", required=True, help="Path to eval/tasks/ (to read originals)")
-    parser.add_argument("--output-dir", required=True, help="Directory to write regression tasks")
-    parser.add_argument("--max-total-tasks", type=int, default=60, help="Cap total tasks in output-dir (default 60)")
-    args = parser.parse_args()
-
-    current = load_json(args.current_scores)
-    previous = load_json(args.previous_scores)
-
-    if not current or not previous:
-        print("Missing scores files — skipping test growth")
-        return
-
-    # Find tasks that were fixed
-    fixed = find_fixed_tasks(current, previous)
-    if not fixed:
-        print("No tasks improved significantly — no regression tasks needed")
-        return
-
-    # Check capacity
-    existing_count = count_existing_tasks(args.output_dir)
-    available_slots = args.max_total_tasks - existing_count
-    if available_slots <= 0:
-        print(f"Task suite already at capacity ({existing_count}/{args.max_total_tasks}) — skipping growth")
-        return
-
-    os.makedirs(args.output_dir, exist_ok=True)
-    regression_id = next_regression_id(args.output_dir)
-    tasks_added = 0
-    fixed_ids = []
-
-    for fix_info in fixed:
-        if tasks_added >= available_slots:
-            break
-
-        tid = fix_info["task_id"]
-        # Load original task
-        task_path = os.path.join(args.tasks_dir, f"{tid}.json")
-        original = load_json(task_path)
-        if not original:
-            continue
-
-        original_input = original.get("input", "")
-        if not original_input:
-            continue
-
-        original_meta = original.get("metadata", {})
-        variations = generate_variations(original_input, tid)
-
-        for var_type, var_input in variations:
-            if tasks_added >= available_slots:
-                break
-
-            reg_id = f"regression_{regression_id:03d}"
-            task = {
-                "id": reg_id,
-                "input": var_input,
-                "metadata": {
-                    "difficulty": original_meta.get("difficulty", "medium"),
-                    "category": original_meta.get("category", "unknown"),
-                    "type": "regression",
-                    "source": "regression",
-                    "regression_for": tid,
-                    "variation": var_type,
-                    "previous_score": fix_info["previous_score"],
-                    "fixed_at_score": fix_info["current_score"],
-                },
-            }
-
-            # Include expected if original had it
-            if "expected" in original:
-                task["expected"] = original["expected"]
-
-            out_path = os.path.join(args.output_dir, f"{reg_id}.json")
-            with open(out_path, "w") as f:
-                json.dump(task, f, indent=2)
-
-            tasks_added += 1
-            regression_id += 1
-
-        fixed_ids.append(tid)
-
-    # Output summary
-    summary = {
-        "tasks_added": tasks_added,
-        "fixed_tasks": fixed_ids,
-        "total_tasks_now": existing_count + tasks_added,
-        "max_total_tasks": args.max_total_tasks,
-    }
-    print(json.dumps(summary))
-    print(f"Added {tasks_added} regression tasks to lock in improvements on: {', '.join(fixed_ids)}")
-
-
-if __name__ == "__main__":
-    main()

package/tools/trace_logger.py

@@ -1,42 +0,0 @@
-"""TraceLogger — optional helper for harnesses to write structured trace records.
-
-Usage in a harness:
-    from trace_logger import TraceLogger
-
-    trace = TraceLogger(traces_dir)
-    trace.step("llm_call", {"prompt": p, "response": r, "model": "gpt-4"})
-    trace.step("tool_use", {"tool": "search", "query": q, "results": results})
-    trace.save()
-
-Stdlib-only. No external dependencies.
-"""
-
-import json
-import os
-import time
-
-
-class TraceLogger:
-    def __init__(self, traces_dir):
-        self.traces_dir = traces_dir
-        self._steps = []
-        if traces_dir:
-            os.makedirs(traces_dir, exist_ok=True)
-
-    def step(self, name, data=None):
-        self._steps.append({
-            "name": name,
-            "timestamp": time.time(),
-            "data": data if data is not None else {},
-        })
-
-    def save(self):
-        if not self.traces_dir:
-            return
-        path = os.path.join(self.traces_dir, "trace.json")
-        with open(path, "w") as f:
-            json.dump(self._steps, f, indent=2)
-
-    @property
-    def steps(self):
-        return list(self._steps)