harness-evolver 4.5.1 → 4.5.2

package/.claude-plugin/plugin.json

@@ -1,7 +1,7 @@
 {
   "name": "harness-evolver",
   "description": "LangSmith-native autonomous agent optimization — evolves LLM agent code using multi-agent proposers, LangSmith experiments, and git worktrees",
-  "version": "4.5.1",
+  "version": "4.5.2",
   "author": {
     "name": "Raphael Valdetaro"
   },

package/README.md

@@ -48,8 +48,9 @@ export LANGSMITH_API_KEY="lsv2_pt_..."
 claude
 
 /evolver:setup      # explores project, configures LangSmith
+/evolver:health     # check dataset quality (auto-corrects issues)
 /evolver:evolve     # runs the optimization loop
-/evolver:status     # check progress
+/evolver:status     # check progress (rich ASCII chart)
 /evolver:deploy     # tag, push, finalize
 ```
 
@@ -64,19 +65,43 @@ claude
 </tr>
 <tr>
 <td><b>Real Code Evolution</b></td>
-<td>Proposers modify your actual agent code — not a wrapper. Each candidate works in an isolated git worktree. Winners are merged automatically.</td>
+<td>Proposers modify your actual agent code — not a wrapper. Each candidate works in an isolated git worktree. Winners are merged automatically. Config files (.evolver.json, .env) are auto-propagated to worktrees.</td>
 </tr>
 <tr>
 <td><b>Self-Organizing Proposers</b></td>
 <td>Each iteration generates dynamic investigation lenses from failure data, architecture analysis, production traces, and evolution memory. Proposers self-organize their approach — no fixed strategies. They can self-abstain when their contribution would be redundant. Inspired by <a href="https://arxiv.org/abs/2603.28990">Dochkina (2026)</a>.</td>
 </tr>
 <tr>
+<td><b>Rubric-Based Evaluation</b></td>
+<td>Dataset examples support <code>expected_behavior</code> rubrics — specific criteria the judge evaluates against ("should mention null safety and Android development"), not just generic correctness. Partial scoring (0.5) for partially-met rubrics. Inspired by <a href="https://github.com/NousResearch/hermes-agent-self-evolution">Hermes Agent Self-Evolution</a>.</td>
+</tr>
+<tr>
+<td><b>Constraint Gates</b></td>
+<td>Proposals must pass hard constraints before merge: code growth ≤30%, entry point syntax valid, test suite passes. Candidates that fail are rejected and the next-best is tried. Prevents code bloat and broken merges.</td>
+</tr>
+<tr>
+<td><b>Weighted Evaluators + Pareto</b></td>
+<td>Configure <code>evaluator_weights</code> to prioritize what matters (e.g., correctness 50%, latency 30%). When candidates offer genuinely different tradeoffs, the Pareto front is reported instead of forcing a single winner.</td>
+</tr>
+<tr>
 <td><b>Agent-Based Evaluation</b></td>
-<td>The evaluator agent reads experiment outputs via langsmith-cli, judges correctness using the same Claude model powering the other agents, and writes scores back. No OpenAI API key or openevals dependency needed.</td>
+<td>The evaluator agent reads experiment outputs via langsmith-cli, judges correctness using rubrics when available, and writes scores back. Judge feedback (textual comments explaining WHY scores were given) is surfaced to proposers for targeted mutations.</td>
+</tr>
+<tr>
+<td><b>Canary Preflight</b></td>
+<td>Before running the full evaluation, 1 example is tested as a canary. If the agent produces no output, evaluation stops immediately — no API quota wasted on broken agents.</td>
+</tr>
+<tr>
+<td><b>Secret Detection</b></td>
+<td>Detects 15+ secret patterns (API keys, tokens, PEM keys) in production traces and dataset examples. Secrets are filtered from <code>seed_from_traces</code> and flagged as critical issues in dataset health checks.</td>
+</tr>
+<tr>
+<td><b>Evolution Chart</b></td>
+<td>Rich ASCII visualization with ANSI colors: sparkline trend, score progression table (per-evaluator breakdown), what-changed narrative, horizontal bar chart, and code growth tracking with warnings.</td>
 </tr>
 <tr>
 <td><b>Production Traces</b></td>
-<td>Auto-discovers existing LangSmith production projects. Uses real user inputs for test generation and real error patterns for targeted optimization.</td>
+<td>Auto-discovers existing LangSmith production projects. Uses real user inputs for test generation and real error patterns for targeted optimization. Can also mine Claude Code session history for eval data.</td>
 </tr>
 <tr>
 <td><b>Active Critic</b></td>
@@ -92,11 +117,11 @@ claude
 </tr>
 <tr>
 <td><b>Dataset Health</b></td>
-<td>Pre-flight dataset quality check: size adequacy, difficulty distribution, dead example detection, production coverage analysis, train/held-out splits. Auto-corrects issues before evolution starts.</td>
+<td>Pre-flight dataset quality check: size adequacy, difficulty distribution, dead example detection, production coverage analysis, train/held-out splits, and secret scanning. Auto-corrects issues before evolution starts.</td>
 </tr>
 <tr>
 <td><b>Smart Gating</b></td>
-<td>Claude assesses gate conditions directly — score plateau, target reached, diminishing returns. No hardcoded thresholds. State validation ensures config hasn't diverged from LangSmith.</td>
+<td>Claude assesses gate conditions directly — score plateau, target reached, diminishing returns. Holdout enforcement ensures final comparison uses unseen data. Baseline is re-scored with LLM-judge before the loop to prevent inflated starting scores.</td>
 </tr>
 <tr>
 <td><b>Background Mode</b></td>
@@ -111,9 +136,9 @@ claude
 | Command | What it does |
 |---|---|
 | `/evolver:setup` | Explore project, configure LangSmith (dataset, evaluators), run baseline |
-| `/evolver:health` | Check dataset quality (size, difficulty, coverage, splits), auto-correct issues |
+| `/evolver:health` | Check dataset quality (size, difficulty, coverage, splits, secrets), auto-correct |
 | `/evolver:evolve` | Run the optimization loop (dynamic self-organizing proposers in worktrees) |
-| `/evolver:status` | Show progress, scores, history |
+| `/evolver:status` | Show progress with rich ASCII evolution chart |
 | `/evolver:deploy` | Tag, push, clean up temporary files |
 
 ---
@@ -123,11 +148,11 @@ claude
 | Agent | Role | Color |
 |---|---|---|
 | **Proposer** | Self-organizing — investigates a data-driven lens, decides own approach, may abstain | Green |
-| **Evaluator** | LLM-as-judge — reads outputs via langsmith-cli, scores correctness | Yellow |
+| **Evaluator** | LLM-as-judge — rubric-aware scoring via langsmith-cli, textual feedback | Yellow |
 | **Architect** | ULTRAPLAN mode — deep topology analysis with Opus model | Blue |
 | **Critic** | Active — detects gaming AND implements stricter evaluators | Red |
 | **Consolidator** | Cross-iteration memory consolidation (autoDream-inspired) | Cyan |
-| **TestGen** | Generates test inputs + adversarial injection mode | Cyan |
+| **TestGen** | Generates test inputs with rubrics + adversarial injection mode | Cyan |
 
 ---
 
@@ -136,20 +161,22 @@ claude
 ```
 /evolver:evolve
   |
-  +- 0.5  Validate state (skeptical memory — check .evolver.json vs LangSmith)
-  +- 0.6  /evolver:health — dataset quality check + auto-correct
+  +- 0.5  Validate state (check .evolver.json vs LangSmith)
+  +- 0.6  /evolver:health — dataset quality + secret scan + auto-correct
+  +- 0.7  Baseline LLM-judge — re-score baseline with correctness if only has_output exists
   +- 1.   Read state (.evolver.json + LangSmith experiments)
-  +- 1.5  Gather trace insights (cluster errors, tokens, latency)
-  +- 1.8  Analyze per-task failures (train split only — proposers don't see held-out)
+  +- 1.5  Gather trace insights + judge feedback (cluster errors, tokens, latency)
+  +- 1.8  Analyze per-task failures with judge comments (train split only)
   +- 1.8a Claude generates strategy.md + lenses.json from analysis data
   +- 1.9  Prepare shared proposer context (KV cache-optimized prefix)
   +- 2.   Spawn N self-organizing proposers in parallel (each in a git worktree)
-  +- 3.   Run target for each candidate (code-based evaluators)
-  +- 3.5  Spawn evaluator agent (LLM-as-judge via langsmith-cli)
-  +- 4.   Compare experiments -> select winner + per-task champion
+  +- 3.   Copy .evolver.json + .env to worktrees, run canary, evaluate candidates
+  +- 3.5  Spawn evaluator agent (rubric-aware LLM-as-judge via langsmith-cli)
+  +- 4.   Compare experiments on held-out split -> winner + Pareto front
+  +- 4.5  Constraint gate — reject candidates that break size/tests/entry-point
   +- 5.   Merge winning worktree into main branch
   +- 5.5  Regression tracking (auto-add guard examples to dataset)
-  +- 6.   Report results
+  +- 6.   Report results + evolution chart
   +- 6.2  Consolidator agent updates evolution memory (runs in background)
   +- 6.5  Auto-trigger Active Critic (detect + fix evaluator gaming)
   +- 7.   Auto-trigger ULTRAPLAN Architect (opus model, deep analysis)
@@ -166,27 +193,31 @@ Plugin hook (SessionStart)
 
 Skills (markdown)
   ├── /evolver:setup    → explores project, smart defaults, runs setup.py
-  ├── /evolver:health   → dataset quality check + auto-correct
+  ├── /evolver:health   → dataset quality + secret scan + auto-correct
   ├── /evolver:evolve   → orchestrates the evolution loop
-  ├── /evolver:status   → reads .evolver.json + LangSmith
+  ├── /evolver:status   → rich ASCII evolution chart + stagnation detection
   └── /evolver:deploy   → tags and pushes
 
 Agents (markdown)
   ├── Proposer (xN)     → self-organizing, lens-driven, isolated git worktrees
-  ├── Evaluator          → LLM-as-judge via langsmith-cli
+  ├── Evaluator          → rubric-aware LLM-as-judge via langsmith-cli
   ├── Critic             → detects gaming + implements stricter evaluators
   ├── Architect          → ULTRAPLAN deep analysis (opus model)
   ├── Consolidator       → cross-iteration memory (autoDream-inspired)
-  └── TestGen            → generates test inputs + adversarial injection
+  └── TestGen            → generates test inputs with rubrics + adversarial injection
 
-Tools (Python + langsmith SDK)
-  ├── setup.py              → creates datasets, configures evaluators
-  ├── run_eval.py           → runs target against dataset
-  ├── read_results.py       → compares experiments
+Tools (Python)
+  ├── setup.py              → creates datasets, configures evaluators + weights
+  ├── run_eval.py           → runs target against dataset (canary preflight, {input_text})
+  ├── read_results.py       → weighted scoring, Pareto front, judge feedback
   ├── trace_insights.py     → clusters errors from traces
-  ├── seed_from_traces.py   → imports production traces
+  ├── seed_from_traces.py   → imports production traces (secret-filtered)
+  ├── evolution_chart.py    → rich ASCII chart (stdlib-only)
+  ├── constraint_check.py   → validates proposals (growth, syntax, tests) (stdlib-only)
+  ├── secret_filter.py      → detects 15+ secret patterns (stdlib-only)
+  ├── mine_sessions.py      → extracts eval data from Claude Code history (stdlib-only)
+  ├── dataset_health.py     → dataset quality diagnostic + secret scanning
   ├── validate_state.py     → validates config vs LangSmith state
-  ├── dataset_health.py     → dataset quality diagnostic (size, difficulty, coverage, splits)
   ├── regression_tracker.py → tracks regressions, adds guard examples
   ├── add_evaluator.py      → programmatically adds evaluators
   └── adversarial_inject.py → detects memorization, injects adversarial tests
@@ -194,6 +225,27 @@ Tools (Python + langsmith SDK)
 
 ---
 
+## Entry Point Placeholders
+
+When configuring your agent's entry point during setup, use the placeholder that matches how your agent takes input:
+
+| Placeholder | Behavior | Use when |
+|---|---|---|
+| `{input_text}` | Extracts plain text, shell-escapes it | Agent takes `--query "text"` or positional args |
+| `{input}` | Passes path to a JSON file | Agent reads structured JSON from file |
+| `{input_json}` | Passes raw JSON string inline | Agent parses JSON from command line |
+
+**Example:**
+```bash
+# Agent that takes a query as text:
+python agent.py --query {input_text}
+
+# Agent that reads a JSON file:
+python agent.py {input}
+```
+
+---
+
 ## Requirements
 
 - **LangSmith account** + `LANGSMITH_API_KEY`
@@ -223,6 +275,7 @@ LangSmith traces **any** AI framework. The evolver works with all of them:
 
 - [Meta-Harness: End-to-End Optimization of Model Harnesses](https://arxiv.org/abs/2603.28052) — Lee et al., 2026
 - [Drop the Hierarchy and Roles: How Self-Organizing LLM Agents Outperform Designed Structures](https://arxiv.org/abs/2603.28990) — Dochkina, 2026
+- [Hermes Agent Self-Evolution](https://github.com/NousResearch/hermes-agent-self-evolution) — NousResearch (rubric-based eval, constraint gates)
 - [Darwin Godel Machine](https://sakana.ai/dgm/) — Sakana AI
 - [AlphaEvolve](https://deepmind.google/blog/alphaevolve/) — DeepMind
 - [LangSmith Evaluation](https://docs.smith.langchain.com/evaluation) — LangChain

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "harness-evolver",
-  "version": "4.5.1",
+  "version": "4.5.2",
   "description": "LangSmith-native autonomous agent optimization for Claude Code",
   "author": "Raphael Valdetaro",
   "license": "MIT",

package/tools/add_evaluator.py

@@ -39,26 +39,39 @@ CODE_EVALUATOR_TEMPLATES = {
 
 
 def add_evaluator(config_path, evaluator_name, eval_type, pattern=None):
-    """Add evaluator to config."""
+    """Add evaluator to config using partial update to avoid race conditions.
+
+    Re-reads the config immediately before writing to minimize the window
+    where concurrent updates (e.g., main loop updating best_score) could
+    be lost. Only modifies 'evaluators' and 'code_evaluators' fields.
+    """
+    # First read to check if evaluator already exists
     with open(config_path) as f:
         config = json.load(f)
 
-    evaluators = config.get("evaluators", [])
-
-    if evaluator_name in evaluators:
+    if evaluator_name in config.get("evaluators", []):
         print(f"Evaluator '{evaluator_name}' already exists", file=sys.stderr)
         return False
 
-    evaluators.append(evaluator_name)
-    config["evaluators"] = evaluators
-
+    # Prepare what we need to add
+    new_code_eval = None
     if eval_type == "code" and pattern:
-        code_evals = config.get("code_evaluators", {})
-        code_evals[evaluator_name] = {"pattern": pattern, "type": "regex"}
-        config["code_evaluators"] = code_evals
+        new_code_eval = {"pattern": pattern, "type": "regex"}
     elif eval_type == "code" and evaluator_name in CODE_EVALUATOR_TEMPLATES:
+        new_code_eval = CODE_EVALUATOR_TEMPLATES[evaluator_name]
+
+    # Re-read config right before write to pick up concurrent changes
+    with open(config_path) as f:
+        config = json.load(f)
+
+    evaluators = config.get("evaluators", [])
+    if evaluator_name not in evaluators:
+        evaluators.append(evaluator_name)
+    config["evaluators"] = evaluators
+
+    if new_code_eval:
         code_evals = config.get("code_evaluators", {})
-        code_evals[evaluator_name] = CODE_EVALUATOR_TEMPLATES[evaluator_name]
+        code_evals[evaluator_name] = new_code_eval
         config["code_evaluators"] = code_evals
 
     with open(config_path, "w") as f:
@@ -77,12 +90,16 @@ def main():
     args = parser.parse_args()
 
     if args.remove:
+        # Re-read right before write to avoid race conditions
         with open(args.config) as f:
             config = json.load(f)
         evaluators = config.get("evaluators", [])
         if args.evaluator in evaluators:
             evaluators.remove(args.evaluator)
             config["evaluators"] = evaluators
+            code_evals = config.get("code_evaluators", {})
+            code_evals.pop(args.evaluator, None)
+            config["code_evaluators"] = code_evals
             with open(args.config, "w") as f:
                 json.dump(config, f, indent=2)
             print(f"Removed evaluator: {args.evaluator}")

package/tools/constraint_check.py

@@ -80,7 +80,30 @@ def check_entry_point(worktree_path, entry_point):
     return {"pass": True, "reason": "entry point exists and has valid syntax"}
 
 
-def check_tests(worktree_path):
+def find_project_python(worktree_path, config=None):
+    """Find the project's Python interpreter (venv > entry_point > system).
+
+    Checks for venv in the worktree, then extracts from entry_point config,
+    then falls back to system python3.
+    """
+    # Check for venv in worktree
+    for venv_dir in [".venv", "venv"]:
+        venv_python = os.path.join(worktree_path, venv_dir, "bin", "python")
+        if os.path.isfile(venv_python):
+            return venv_python
+
+    # Extract from entry_point in config
+    if config:
+        entry = config.get("entry_point", "")
+        for part in entry.split():
+            if part.endswith("/python") or part.endswith("/python3"):
+                if os.path.isfile(part):
+                    return part
+
+    return "python3"
+
+
+def check_tests(worktree_path, config=None):
     """Run test suite if it exists. Returns pass if no tests found."""
     test_dirs = ["tests", "test"]
     has_tests = False
@@ -95,9 +118,11 @@ def check_tests(worktree_path):
     if not has_tests:
         return {"pass": True, "reason": "no test suite found (skipped)", "skipped": True}
 
+    python = find_project_python(worktree_path, config)
+
     try:
         result = subprocess.run(
-            ["python3", "-m", "pytest", "-q", "--tb=no"],
+            [python, "-m", "pytest", "-q", "--tb=no"],
             capture_output=True, text=True,
             cwd=worktree_path, timeout=120,
         )
@@ -135,7 +160,7 @@ def main():
             args.max_growth,
         ),
         "entry_point": check_entry_point(args.worktree_path, ep_for_check),
-        "tests": check_tests(args.worktree_path),
+        "tests": check_tests(args.worktree_path, config),
     }
 
     all_pass = all(r["pass"] for r in results.values())

package/tools/evolution_chart.py

@@ -73,7 +73,16 @@ def render_header(config, history, scores, c):
     project = config.get('project', 'unknown')
     dataset = config.get('dataset', 'unknown')
     evals = config.get('evaluators', [])
-    total = history[0].get('total', config.get('num_examples', '?'))
+    # Find example count from multiple sources
+    total = history[0].get('total')
+    if not total:
+        # Check any history entry that has it
+        for h in history:
+            if h.get('total'):
+                total = h['total']
+                break
+    if not total:
+        total = config.get('num_examples', '?')
     base_score = scores[0]
     best_score = max(scores)
     iters = len(history) - 1