harness-evolver 4.0.3 → 4.2.0

package/tools/consolidate.py

@@ -94,24 +94,16 @@ def consolidate(orientation, signals, existing_memory=None):
     """Phase 3: Merge signals into consolidated memory."""
     insights = []
 
-    # Strategy effectiveness
+    # Winning approach tracking (from comparison data)
     winning = signals.get("winning_strategies", [])
-    strategy_map = {"a": "exploit", "b": "explore", "c": "crossover", "d": "failure-targeted-1", "e": "failure-targeted-2"}
-    win_counts = {}
-    for w in winning:
-        exp = w.get("experiment", "")
-        if exp:
-            suffix = exp[-1]
-            name = strategy_map.get(suffix, suffix)
-            win_counts[name] = win_counts.get(name, 0) + 1
-
-    if win_counts:
-        best_strategy = max(win_counts, key=win_counts.get)
+    if winning:
+        win_count = len(winning)
+        best_score = max(w.get("score", 0) for w in winning)
         insights.append({
             "type": "strategy_effectiveness",
-            "insight": f"Most winning strategy: {best_strategy} ({win_counts[best_strategy]} wins)",
-            "recurrence": win_counts[best_strategy],
-            "data": win_counts,
+            "insight": f"Best candidate score: {best_score:.3f} across {win_count} iterations",
+            "recurrence": win_count,
+            "data": {"win_count": win_count, "best_score": best_score},
         })
 
     # Recurring failures (only promote if seen 2+ times)

package/tools/dataset_health.py

@@ -0,0 +1,385 @@
+#!/usr/bin/env python3
+"""Dataset health diagnostic for Harness Evolver.
+
+Analyzes eval dataset quality: size adequacy, difficulty distribution,
+dead examples, production coverage, and split configuration.
+Outputs health_report.json with issues and recommended corrections.
+
+Usage:
+    python3 dataset_health.py --config .evolver.json --output health_report.json
+"""
+
+import argparse
+import json
+import os
+import sys
+from datetime import datetime, timezone
+
+
+def ensure_langsmith_api_key():
+    """Load API key from langsmith-cli credentials if not in env."""
+    if os.environ.get("LANGSMITH_API_KEY"):
+        return True
+    import platform
+    if platform.system() == "Darwin":
+        creds_path = os.path.expanduser("~/Library/Application Support/langsmith-cli/credentials")
+    else:
+        creds_path = os.path.expanduser("~/.config/langsmith-cli/credentials")
+    if os.path.exists(creds_path):
+        try:
+            with open(creds_path) as f:
+                for line in f:
+                    if line.strip().startswith("api_key"):
+                        key = line.split("=", 1)[1].strip().strip("'\"")
+                        if key:
+                            os.environ["LANGSMITH_API_KEY"] = key
+                            return True
+        except OSError:
+            pass
+    return False
+
+
+def load_json_safe(path):
+    """Load JSON file, return None if missing or invalid."""
+    if not path or not os.path.exists(path):
+        return None
+    try:
+        with open(path) as f:
+            return json.load(f)
+    except (json.JSONDecodeError, OSError):
+        return None
+
+
+def check_size(examples, evaluators):
+    """Check dataset size adequacy."""
+    count = len(examples)
+    min_recommended = max(20, 10 * len(evaluators))
+    return {
+        "example_count": count,
+        "min_recommended": min_recommended,
+        "adequate": count >= min_recommended,
+    }
+
+
+def check_difficulty(client, config):
+    """Check difficulty distribution from best experiment scores."""
+    best_exp = config.get("best_experiment")
+    if not best_exp:
+        return None
+
+    try:
+        runs = list(client.list_runs(project_name=best_exp, is_root=True, limit=200))
+        if not runs:
+            return None
+
+        all_run_ids = [run.id for run in runs]
+        all_feedbacks = list(client.list_feedback(run_ids=all_run_ids))
+        fb_map = {}
+        for fb in all_feedbacks:
+            fb_map.setdefault(str(fb.run_id), []).append(fb)
+
+        scores = []
+        example_difficulties = {}
+        for run in runs:
+            run_fbs = fb_map.get(str(run.id), [])
+            run_scores = [fb.score for fb in run_fbs if fb.score is not None]
+            if run_scores:
+                avg = sum(run_scores) / len(run_scores)
+                scores.append(avg)
+                eid = str(run.reference_example_id or run.id)
+                if avg > 0.9:
+                    example_difficulties[eid] = "easy"
+                elif avg >= 0.5:
+                    example_difficulties[eid] = "medium"
+                else:
+                    example_difficulties[eid] = "hard"
+
+        if not scores:
+            return None
+
+        easy = sum(1 for s in scores if s > 0.9)
+        medium = sum(1 for s in scores if 0.5 <= s <= 0.9)
+        hard = sum(1 for s in scores if s < 0.5)
+        total = len(scores)
+        skew = None
+        if total > 0 and easy / total > 0.6:
+            skew = "easy_heavy"
+        elif total > 0 and hard / total > 0.6:
+            skew = "hard_heavy"
+
+        return {
+            "easy": easy,
+            "medium": medium,
+            "hard": hard,
+            "skew": skew,
+            "example_difficulties": example_difficulties,
+        }
+    except Exception:
+        return None
+
+
+def check_dead_examples(client, config):
+    """Find examples that scored >=0.9 across all recent experiments."""
+    history = config.get("history", [])
+    if len(history) < 2:
+        return {"count": 0, "ids": []}
+
+    recent_exps = [h["experiment"] for h in history[-3:]]
+    example_scores = {}
+
+    for exp_name in recent_exps:
+        try:
+            runs = list(client.list_runs(project_name=exp_name, is_root=True, limit=200))
+            all_run_ids = [run.id for run in runs]
+            if not all_run_ids:
+                continue
+            all_feedbacks = list(client.list_feedback(run_ids=all_run_ids))
+            fb_map = {}
+            for fb in all_feedbacks:
+                fb_map.setdefault(str(fb.run_id), []).append(fb)
+
+            for run in runs:
+                eid = str(run.reference_example_id or run.id)
+                run_fbs = fb_map.get(str(run.id), [])
+                run_scores = [fb.score for fb in run_fbs if fb.score is not None]
+                if run_scores:
+                    avg = sum(run_scores) / len(run_scores)
+                    if eid not in example_scores:
+                        example_scores[eid] = []
+                    example_scores[eid].append(avg)
+        except Exception:
+            continue
+
+    dead_ids = []
+    for eid, exp_scores in example_scores.items():
+        if len(exp_scores) >= 2 and all(s >= 0.9 for s in exp_scores):
+            dead_ids.append(eid)
+
+    return {"count": len(dead_ids), "ids": dead_ids}
+
+
+def check_coverage(examples, production):
+    """Compare dataset categories vs production traffic."""
+    if not production:
+        return None
+
+    prod_categories = set()
+    for cat in production.get("categories", []):
+        if isinstance(cat, str):
+            prod_categories.add(cat)
+        elif isinstance(cat, dict) and "category" in cat:
+            prod_categories.add(cat["category"])
+
+    if not prod_categories:
+        return None
+
+    dataset_categories = set()
+    for ex in examples:
+        meta = getattr(ex, "metadata", None) or {}
+        if "category" in meta:
+            dataset_categories.add(meta["category"])
+
+    missing = prod_categories - dataset_categories
+    coverage_pct = 0
+    if prod_categories:
+        coverage_pct = int(100 * len(prod_categories - missing) / len(prod_categories))
+
+    return {
+        "production": sorted(prod_categories),
+        "dataset": sorted(dataset_categories),
+        "missing": sorted(missing),
+        "pct": coverage_pct,
+    }
+
+
+def check_splits(client, dataset_name):
+    """Check if train/held_out splits exist."""
+    has_train = False
+    has_held_out = False
+    try:
+        train = list(client.list_examples(dataset_name=dataset_name, splits=["train"], limit=1))
+        has_train = len(train) > 0
+    except Exception:
+        pass
+    try:
+        held = list(client.list_examples(dataset_name=dataset_name, splits=["held_out"], limit=1))
+        has_held_out = len(held) > 0
+    except Exception:
+        pass
+    return {"has_train": has_train, "has_held_out": has_held_out}
+
+
+def compute_health_score(size_info, difficulty, dead, coverage, splits):
+    """Compute overall health score 0-10."""
+    score = 10
+
+    if not size_info.get("adequate"):
+        score -= 3
+
+    if difficulty and difficulty.get("skew"):
+        score -= 2
+
+    if dead and dead.get("count", 0) > 0:
+        total = size_info.get("example_count", 1)
+        if dead["count"] / max(total, 1) > 0.2:
+            score -= 1
+
+    if coverage and coverage.get("pct", 100) < 75:
+        score -= 2
+
+    if splits and not splits.get("has_train"):
+        score -= 2
+
+    return max(0, score)
+
+
+def build_issues_and_corrections(size_info, difficulty, dead, coverage, splits):
+    """Build issues and corrections lists."""
+    issues = []
+    corrections = []
+
+    if not size_info.get("adequate"):
+        issues.append({
+            "type": "size_inadequate",
+            "severity": "high",
+            "message": f"Only {size_info['example_count']} examples (recommended: {size_info['min_recommended']}+)",
+        })
+        corrections.append({
+            "action": "generate_more",
+            "count": size_info["min_recommended"] - size_info["example_count"],
+        })
+
+    if difficulty and difficulty.get("skew") == "easy_heavy":
+        easy_pct = int(100 * difficulty["easy"] / max(difficulty["easy"] + difficulty["medium"] + difficulty["hard"], 1))
+        issues.append({
+            "type": "difficulty_skew",
+            "severity": "high",
+            "message": f"{easy_pct}% easy examples — low discriminative power",
+        })
+        corrections.append({
+            "action": "generate_hard",
+            "count": max(5, difficulty["easy"] // 3),
+        })
+
+    if dead and dead.get("count", 0) > 0:
+        total = size_info.get("example_count", 1)
+        dead_pct = int(100 * dead["count"] / max(total, 1))
+        if dead_pct > 10:
+            issues.append({
+                "type": "dead_examples",
+                "severity": "medium",
+                "message": f"{dead['count']} dead examples ({dead_pct}%) — scored >=0.9 in all recent experiments",
+            })
+            corrections.append({
+                "action": "retire_dead",
+                "ids": dead["ids"],
+            })
+
+    if coverage and coverage.get("missing"):
+        issues.append({
+            "type": "coverage_gap",
+            "severity": "high",
+            "message": f"Missing categories: {', '.join(coverage['missing'])} ({coverage['pct']}% coverage)",
+        })
+        corrections.append({
+            "action": "fill_coverage",
+            "categories": coverage["missing"],
+        })
+
+    if splits and not splits.get("has_train"):
+        issues.append({
+            "type": "no_splits",
+            "severity": "medium",
+            "message": "No train/held-out split — proposer overfit risk",
+        })
+        corrections.append({
+            "action": "create_splits",
+            "train_pct": 70,
+        })
+
+    return issues, corrections
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Dataset health diagnostic")
+    parser.add_argument("--config", default=".evolver.json")
+    parser.add_argument("--production-seed", default="production_seed.json")
+    parser.add_argument("--output", default="health_report.json")
+    args = parser.parse_args()
+
+    ensure_langsmith_api_key()
+
+    with open(args.config) as f:
+        config = json.load(f)
+
+    production = load_json_safe(args.production_seed)
+
+    from langsmith import Client
+    client = Client()
+
+    dataset_name = config["dataset"]
+
+    # Get all examples
+    examples = list(client.list_examples(dataset_name=dataset_name, limit=500))
+
+    # Run checks
+    evaluators = config.get("evaluators", ["correctness"])
+    size_info = check_size(examples, evaluators)
+    difficulty = check_difficulty(client, config)
+    dead = check_dead_examples(client, config)
+    coverage = check_coverage(examples, production)
+    splits = check_splits(client, dataset_name)
+
+    # Tag difficulty metadata on examples if we computed it
+    if difficulty and difficulty.get("example_difficulties"):
+        for ex in examples:
+            eid = str(ex.id)
+            diff = difficulty["example_difficulties"].get(eid)
+            if diff:
+                meta = dict(getattr(ex, "metadata", None) or {})
+                if meta.get("difficulty") != diff:
+                    meta["difficulty"] = diff
+                    try:
+                        client.update_example(ex.id, metadata=meta)
+                    except Exception:
+                        pass
+
+    # Compute health score and build report
+    health_score = compute_health_score(size_info, difficulty, dead, coverage, splits)
+    issues, corrections = build_issues_and_corrections(size_info, difficulty, dead, coverage, splits)
+
+    report = {
+        "generated_at": datetime.now(timezone.utc).isoformat(),
+        "health_score": health_score,
+        "example_count": size_info["example_count"],
+        "min_recommended": size_info["min_recommended"],
+        "difficulty": {k: v for k, v in (difficulty or {}).items() if k != "example_difficulties"} or None,
+        "dead_examples": dead,
+        "coverage": coverage,
+        "splits": splits,
+        "issues": issues,
+        "corrections": corrections,
+    }
+
+    with open(args.output, "w") as f:
+        json.dump(report, f, indent=2)
+
+    # Print human-readable summary
+    print(f"Dataset Health: {health_score}/10")
+    print(f"Examples: {size_info['example_count']} (min recommended: {size_info['min_recommended']})")
+    if difficulty:
+        print(f"Difficulty: {difficulty.get('easy', 0)} easy, {difficulty.get('medium', 0)} medium, {difficulty.get('hard', 0)} hard")
+    if dead and dead["count"] > 0:
+        print(f"Dead examples: {dead['count']}")
+    if coverage:
+        print(f"Coverage: {coverage['pct']}% ({len(coverage.get('missing', []))} categories missing)")
+    if splits:
+        print(f"Splits: train={'yes' if splits['has_train'] else 'no'}, held_out={'yes' if splits['has_held_out'] else 'no'}")
+    if issues:
+        print(f"\nIssues ({len(issues)}):")
+        for issue in issues:
+            print(f"  [{issue['severity']}] {issue['message']}")
+
+
+if __name__ == "__main__":
+    main()

package/tools/read_results.py

@@ -79,6 +79,13 @@ def read_experiment(client, experiment_name):
         total_latency_ms = 0
         errors = 0
 
+        # Batch-fetch all feedback in one API call instead of N+1
+        all_run_ids = [run.id for run in runs]
+        all_feedbacks = list(client.list_feedback(run_ids=all_run_ids))
+        fb_map = {}
+        for fb in all_feedbacks:
+            fb_map.setdefault(str(fb.run_id), []).append(fb)
+
         for run in runs:
             example_id = str(run.reference_example_id or run.id)
             tokens = run.total_tokens or 0
@@ -93,8 +100,8 @@ def read_experiment(client, experiment_name):
             if has_error:
                 errors += 1
 
-            # Read feedback/scores
-            feedbacks = list(client.list_feedback(run_ids=[run.id]))
+            # Read feedback/scores from pre-fetched batch
+            feedbacks = fb_map.get(str(run.id), [])
             scores = {}
             for fb in feedbacks:
                 if fb.score is not None:
@@ -220,6 +227,7 @@ def main():
     parser.add_argument("--config", default=".evolver.json", help="Path to .evolver.json")
     parser.add_argument("--output", default=None, help="Output JSON path")
     parser.add_argument("--format", default="json", choices=["json", "markdown"], help="Output format")
+    parser.add_argument("--split", default=None, help="Filter by dataset split (e.g., 'train')")
     args = parser.parse_args()
     ensure_langsmith_api_key()
 
@@ -233,6 +241,17 @@ def main():
             print(f"No results found for experiment: {args.experiment}", file=sys.stderr)
             sys.exit(1)
 
+        if args.split and result and "per_example" in result:
+            with open(args.config) as f:
+                cfg = json.load(f)
+            split_example_ids = set()
+            for ex in client.list_examples(dataset_name=cfg["dataset"], splits=[args.split]):
+                split_example_ids.add(str(ex.id))
+            result["per_example"] = {k: v for k, v in result["per_example"].items() if k in split_example_ids}
+            all_scores = [v["score"] for v in result["per_example"].values()]
+            result["combined_score"] = sum(all_scores) / len(all_scores) if all_scores else 0.0
+            result["num_examples"] = len(result["per_example"])
+
         if args.format == "markdown":
             output = format_markdown(result)
         else:

package/tools/regression_tracker.py

@@ -17,6 +17,7 @@ import argparse
 import json
 import os
 import platform
+import random
 import sys
 
 
@@ -60,9 +61,14 @@ def get_per_example_scores(client, experiment_name):
     scores = {}
     try:
         runs = list(client.list_runs(project_name=experiment_name, is_root=True, limit=200))
+        all_run_ids = [run.id for run in runs]
+        all_feedbacks = list(client.list_feedback(run_ids=all_run_ids))
+        fb_map = {}
+        for fb in all_feedbacks:
+            fb_map.setdefault(str(fb.run_id), []).append(fb)
         for run in runs:
             example_id = str(run.reference_example_id or run.id)
-            feedbacks = list(client.list_feedback(run_ids=[run.id]))
+            feedbacks = fb_map.get(str(run.id), [])
             fb_scores = {}
             for fb in feedbacks:
                 if fb.score is not None:
@@ -107,16 +113,23 @@ def find_transitions(prev_scores, curr_scores, fail_threshold=0.5, pass_threshol
     return transitions, regressions
 
 
-def add_regression_guards(client, dataset_id, transitions, max_guards=5):
+def add_regression_guards(client, dataset_id, transitions, max_guards=5, config=None):
     """Add regression guard examples to the dataset."""
+    config = config or {}
     added = 0
     for t in transitions[:max_guards]:
         try:
             input_data = json.loads(t["input"]) if t["input"].startswith("{") else {"input": t["input"]}
+            split = "train" if random.random() < 0.7 else "held_out"
             client.create_example(
                 inputs=input_data,
                 dataset_id=dataset_id,
-                metadata={"source": "regression_guard", "original_example_id": t["example_id"]},
+                metadata={
+                    "source": "regression_guard",
+                    "original_example_id": t["example_id"],
+                    "added_at_iteration": config.get("iterations", 0),
+                },
+                split=split,
             )
             added += 1
         except Exception as e:
@@ -148,7 +161,7 @@ def main():
 
     added = 0
     if args.add_guards and transitions:
-        added = add_regression_guards(client, config["dataset_id"], transitions, args.max_guards)
+        added = add_regression_guards(client, config["dataset_id"], transitions, args.max_guards, config=config)
 
     result = {
         "previous": args.previous_experiment,

package/tools/setup.py

@@ -87,6 +87,19 @@ def check_dependencies():
     return missing
 
 
+def assign_splits(client, dataset_id, train_pct=70):
+    """Assign train/held_out splits to all examples in a dataset."""
+    import random
+    examples = list(client.list_examples(dataset_id=dataset_id))
+    random.shuffle(examples)
+    split_point = int(len(examples) * train_pct / 100)
+    for ex in examples[:split_point]:
+        client.update_example(ex.id, split="train")
+    for ex in examples[split_point:]:
+        client.update_example(ex.id, split="held_out")
+    return len(examples[:split_point]), len(examples[split_point:])
+
+
 def resolve_dataset_name(client, base_name):
     """Find an available dataset name by auto-incrementing the version suffix.
 
@@ -148,10 +161,17 @@ def create_dataset_from_file(client, dataset_name, file_path):
             if "metadata" in item:
                 ex["metadata"] = item["metadata"]
 
+            if "metadata" not in ex:
+                ex["metadata"] = {}
+            ex["metadata"].setdefault("source", "file")
+            ex["metadata"].setdefault("added_at_iteration", 0)
+
             examples.append(ex)
 
     if examples:
         client.create_examples(dataset_id=dataset.id, examples=examples)
+        train_n, held_n = assign_splits(client, dataset.id)
+        print(f"Assigned splits: {train_n} train, {held_n} held_out", file=sys.stderr)
 
     return dataset, len(examples)
 
@@ -178,10 +198,13 @@ def create_dataset_from_langsmith(client, dataset_name, source_project, limit=10
             ex = {"inputs": run.inputs}
             if run.outputs:
                 ex["outputs"] = run.outputs
+            ex["metadata"] = {"source": "production", "added_at_iteration": 0}
             examples.append(ex)
 
     if examples:
         client.create_examples(dataset_id=dataset.id, examples=examples)
+        train_n, held_n = assign_splits(client, dataset.id)
+        print(f"Assigned splits: {train_n} train, {held_n} held_out", file=sys.stderr)
 
     return dataset, len(examples)