harness-evolver 3.1.0 → 3.2.0

package/.claude-plugin/plugin.json

@@ -0,0 +1,20 @@
+{
+  "name": "harness-evolver",
+  "description": "LangSmith-native autonomous agent optimization — evolves LLM agent code using multi-agent proposers, LangSmith experiments, and git worktrees",
+  "version": "3.2.0",
+  "author": {
+    "name": "Raphael Valdetaro"
+  },
+  "homepage": "https://github.com/raphaelchristi/harness-evolver",
+  "repository": "https://github.com/raphaelchristi/harness-evolver",
+  "license": "MIT",
+  "keywords": [
+    "langsmith",
+    "optimization",
+    "evolution",
+    "llm",
+    "agent",
+    "evaluator",
+    "langsmith-cli"
+  ]
+}

package/hooks/hooks.json

@@ -0,0 +1,15 @@
+{
+  "description": "Harness Evolver — ensures Python deps and env vars are available each session",
+  "hooks": {
+    "SessionStart": [
+      {
+        "hooks": [
+          {
+            "type": "command",
+            "command": "bash \"${CLAUDE_PLUGIN_ROOT}/hooks/session-start.sh\""
+          }
+        ]
+      }
+    ]
+  }
+}

package/hooks/session-start.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+# Harness Evolver — SessionStart hook
+# Ensures Python venv, langsmith, langsmith-cli, and env vars are ready.
+# Runs silently on every session start. Installs deps only if missing.
+
+set -euo pipefail
+
+# Resolve paths — plugin root is set by Claude Code
+PLUGIN_ROOT="${CLAUDE_PLUGIN_ROOT:-}"
+PLUGIN_DATA="${CLAUDE_PLUGIN_DATA:-}"
+
+# Fallback: if running outside plugin system (npx install), use legacy paths
+if [ -z "$PLUGIN_ROOT" ]; then
+    PLUGIN_ROOT="$HOME/.evolver"
+    PLUGIN_DATA="$HOME/.evolver"
+fi
+
+TOOLS_DIR="$PLUGIN_ROOT/tools"
+VENV_DIR="$PLUGIN_DATA/venv"
+VENV_PY="$VENV_DIR/bin/python"
+
+# --- 1. Create venv if missing ---
+if [ ! -f "$VENV_PY" ]; then
+    if command -v uv >/dev/null 2>&1; then
+        uv venv "$VENV_DIR" >/dev/null 2>&1
+    else
+        python3 -m venv "$VENV_DIR" >/dev/null 2>&1
+    fi
+fi
+
+# --- 2. Install langsmith if missing ---
+if [ -f "$VENV_PY" ]; then
+    "$VENV_PY" -c "import langsmith" 2>/dev/null || {
+        if command -v uv >/dev/null 2>&1; then
+            uv pip install --python "$VENV_PY" langsmith >/dev/null 2>&1
+        else
+            "$VENV_DIR/bin/pip" install --upgrade langsmith >/dev/null 2>&1 || \
+            "$VENV_PY" -m pip install --upgrade langsmith >/dev/null 2>&1
+        fi
+    }
+fi
+
+# --- 3. Install langsmith-cli if missing ---
+command -v langsmith-cli >/dev/null 2>&1 || {
+    if command -v uv >/dev/null 2>&1; then
+        uv tool install langsmith-cli >/dev/null 2>&1
+    else
+        pip install langsmith-cli >/dev/null 2>&1 || pip3 install langsmith-cli >/dev/null 2>&1
+    fi
+} || true
+
+# --- 4. Load API key from credentials file if not in env ---
+if [ -z "${LANGSMITH_API_KEY:-}" ]; then
+    if [ "$(uname)" = "Darwin" ]; then
+        CREDS="$HOME/Library/Application Support/langsmith-cli/credentials"
+    else
+        CREDS="$HOME/.config/langsmith-cli/credentials"
+    fi
+    if [ -f "$CREDS" ]; then
+        KEY=$(grep '^LANGSMITH_API_KEY=' "$CREDS" 2>/dev/null | head -1 | cut -d= -f2-)
+        if [ -n "$KEY" ]; then
+            echo "export LANGSMITH_API_KEY=\"$KEY\"" >> "$CLAUDE_ENV_FILE"
+        fi
+    fi
+fi
+
+# --- 5. Export env vars for skills ---
+if [ -n "${CLAUDE_ENV_FILE:-}" ]; then
+    echo "export EVOLVER_TOOLS=\"$TOOLS_DIR\"" >> "$CLAUDE_ENV_FILE"
+    echo "export EVOLVER_PY=\"$VENV_PY\"" >> "$CLAUDE_ENV_FILE"
+fi

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "harness-evolver",
-  "version": "3.1.0",
+  "version": "3.2.0",
   "description": "LangSmith-native autonomous agent optimization for Claude Code",
   "author": "Raphael Valdetaro",
   "license": "MIT",
@@ -24,6 +24,8 @@
     "bin/",
     "skills/",
     "agents/",
-    "tools/"
+    "tools/",
+    "hooks/",
+    ".claude-plugin/"
   ]
 }

package/skills/evolve/SKILL.md

@@ -16,8 +16,9 @@ Run the autonomous propose-evaluate-iterate loop using LangSmith as the evaluati
 ## Resolve Tool Path and Python
 
 ```bash
-TOOLS=$([ -d ".evolver/tools" ] && echo ".evolver/tools" || echo "$HOME/.evolver/tools")
-EVOLVER_PY=$([ -f "$HOME/.evolver/venv/bin/python" ] && echo "$HOME/.evolver/venv/bin/python" || echo "python3")
+# Prefer env vars set by plugin hook; fallback to legacy npx paths
+TOOLS="${EVOLVER_TOOLS:-$([ -d ".evolver/tools" ] && echo ".evolver/tools" || echo "$HOME/.evolver/tools")}"
+EVOLVER_PY="${EVOLVER_PY:-$([ -f "$HOME/.evolver/venv/bin/python" ] && echo "$HOME/.evolver/venv/bin/python" || echo "python3")}"
 ```
 
 Use `$EVOLVER_PY` instead of `python3` for ALL tool invocations.
@@ -75,13 +76,15 @@ python3 -c "import json; c=json.load(open('.evolver.json')); print(f'v{c[\"itera
 
 ### 1.5. Gather Trace Insights
 
-Run trace insights from the best experiment:
+Read the best experiment from config. If null (no baseline was run), skip trace insights for this iteration — proposers will work blind on the first pass:
 
 ```bash
-BEST=$(python3 -c "import json; print(json.load(open('.evolver.json'))['best_experiment'])")
-$EVOLVER_PY $TOOLS/trace_insights.py \
-    --from-experiment "$BEST" \
-    --output trace_insights.json 2>/dev/null
+BEST=$(python3 -c "import json; b=json.load(open('.evolver.json')).get('best_experiment'); print(b if b else '')")
+if [ -n "$BEST" ]; then
+    $EVOLVER_PY $TOOLS/trace_insights.py \
+        --from-experiment "$BEST" \
+        --output trace_insights.json 2>/dev/null
+fi
 ```
 
 If a production project is configured, also gather production insights:
@@ -99,17 +102,20 @@ fi
 
 ### 1.8. Analyze Per-Task Failures
 
-Read the best experiment results and cluster failures:
+If `$BEST` is set (not the first iteration without baseline), read results and cluster failures:
 
 ```bash
-$EVOLVER_PY $TOOLS/read_results.py \
-    --experiment "$BEST" \
-    --config .evolver.json \
-    --output best_results.json 2>/dev/null
+if [ -n "$BEST" ]; then
+    $EVOLVER_PY $TOOLS/read_results.py \
+        --experiment "$BEST" \
+        --config .evolver.json \
+        --output best_results.json 2>/dev/null
+fi
 ```
 
-Parse `best_results.json` to find failing examples (score < 0.7). Group by metadata or error pattern.
+If `best_results.json` exists, parse it to find failing examples (score < 0.7). Group by metadata or error pattern.
 Generate adaptive briefings for Candidates D and E (same logic as v2).
+If no best_results.json (first iteration without baseline), all proposers work from code analysis only — no failure data available.
 
 ### 2. Spawn 5 Proposers in Parallel
 

package/skills/setup/SKILL.md

@@ -38,8 +38,9 @@ The tools auto-load the key from the credentials file, but the env var takes pre
 ## Resolve Tool Path and Python
 
 ```bash
-TOOLS=$([ -d ".evolver/tools" ] && echo ".evolver/tools" || echo "$HOME/.evolver/tools")
-EVOLVER_PY=$([ -f "$HOME/.evolver/venv/bin/python" ] && echo "$HOME/.evolver/venv/bin/python" || echo "python3")
+# Prefer env vars set by plugin hook; fallback to legacy npx paths
+TOOLS="${EVOLVER_TOOLS:-$([ -d ".evolver/tools" ] && echo ".evolver/tools" || echo "$HOME/.evolver/tools")}"
+EVOLVER_PY="${EVOLVER_PY:-$([ -f "$HOME/.evolver/venv/bin/python" ] && echo "$HOME/.evolver/venv/bin/python" || echo "python3")}"
 ```
 
 Use `$EVOLVER_PY` instead of `python3` for ALL tool invocations. This ensures the venv with langsmith is used.

package/tools/setup.py

@@ -87,6 +87,29 @@ def check_dependencies():
     return missing
 
 
+def resolve_dataset_name(client, base_name):
+    """Find an available dataset name by auto-incrementing the version suffix.
+
+    Tries base_name-eval-v1, v2, v3... until an unused name is found.
+    Returns (resolved_name, version_number).
+    """
+    existing = set()
+    try:
+        for ds in client.list_datasets():
+            existing.add(ds.name)
+    except Exception:
+        pass
+
+    for v in range(1, 100):
+        candidate = f"{base_name}-eval-v{v}"
+        if candidate not in existing:
+            return candidate, v
+
+    # Fallback: timestamp-based
+    ts = datetime.now(timezone.utc).strftime("%Y%m%d%H%M%S")
+    return f"{base_name}-eval-{ts}", 0
+
+
 def create_dataset_from_file(client, dataset_name, file_path):
     """Create a LangSmith dataset from a JSON file of inputs."""
     with open(file_path) as f:
@@ -320,6 +343,7 @@ def main():
     parser.add_argument("--dataset-from-file", default=None, help="Create dataset from JSON file")
     parser.add_argument("--dataset-from-langsmith", default=None, help="Create dataset from LangSmith project")
     parser.add_argument("--production-project", default=None, help="Production LangSmith project")
+    parser.add_argument("--dataset-name", default=None, help="Explicit dataset name (skip auto-versioning)")
     parser.add_argument("--evaluators", default=None, help="Comma-separated evaluator names")
     parser.add_argument("--skip-baseline", action="store_true", help="Skip baseline evaluation")
     parser.add_argument("--output", default=".evolver.json", help="Output config path")
@@ -351,9 +375,19 @@ def main():
         sys.exit(1)
 
     project_name = f"evolver-{args.project_name}"
-    dataset_name = f"{args.project_name}-eval-v1"
     goals = [g.strip() for g in args.goals.split(",")]
 
+    # Resolve dataset name (explicit or auto-versioned)
+    if args.dataset_name:
+        dataset_name = args.dataset_name
+        print(f"Using explicit dataset name: '{dataset_name}'")
+    else:
+        dataset_name, version = resolve_dataset_name(client, args.project_name)
+        if version > 1:
+            print(f"Dataset name auto-versioned to '{dataset_name}' (v1-v{version-1} already exist)")
+        else:
+            print(f"Dataset: '{dataset_name}'")
+
     # Create dataset
     print(f"Creating dataset '{dataset_name}'...")
     if args.dataset_from_file: