harness-evolver 0.2.1 → 0.5.0

package/bin/install.js

@@ -1,26 +1,53 @@
 #!/usr/bin/env node
 /**
  * Harness Evolver installer.
- * Detects Claude Code, copies skills/agents/tools to the right locations.
+ * Interactive setup with runtime selection, global/local choice.
  *
  * Usage: npx harness-evolver@latest
  */
 
 const fs = require("fs");
 const path = require("path");
+const readline = require("readline");
 const { execSync } = require("child_process");
 
+const VERSION = require("../package.json").version;
 const PLUGIN_ROOT = path.resolve(__dirname, "..");
 const HOME = process.env.HOME || process.env.USERPROFILE;
 
-const CLAUDE_DIR = path.join(HOME, ".claude");
-const COMMANDS_DIR = path.join(CLAUDE_DIR, "commands", "harness-evolver");
-const AGENTS_DIR = path.join(CLAUDE_DIR, "agents");
-const TOOLS_DIR = path.join(HOME, ".harness-evolver", "tools");
-const EXAMPLES_DIR = path.join(HOME, ".harness-evolver", "examples");
-
-function log(msg) {
-  console.log(`  ${msg}`);
+// ANSI colors
+const CYAN = "\x1b[36m";
+const GREEN = "\x1b[32m";
+const YELLOW = "\x1b[33m";
+const RED = "\x1b[31m";
+const DIM = "\x1b[2m";
+const BOLD = "\x1b[1m";
+const RESET = "\x1b[0m";
+
+const LOGO = `
+${CYAN}  ██╗  ██╗ █████╗ ██████╗ ███╗   ██╗███████╗███████╗███████╗
+  ██║  ██║██╔══██╗██╔══██╗████╗  ██║██╔════╝██╔════╝██╔════╝
+  ███████║███████║██████╔╝██╔██╗ ██║█████╗  ███████╗███████╗
+  ██╔══██║██╔══██║██╔══██╗██║╚██╗██║██╔══╝  ╚════██║╚════██║
+  ██║  ██║██║  ██║██║  ██║██║ ╚████║███████╗███████║███████║
+  ╚═╝  ╚═╝╚═╝  ╚═╝╚═╝  ╚═╝╚═╝  ╚═══╝╚══════╝╚══════╝╚══════╝
+  ${BOLD}███████╗██╗   ██╗ ██████╗ ██╗    ██╗   ██╗███████╗██████╗
+  ██╔════╝██║   ██║██╔═══██╗██║    ██║   ██║██╔════╝██╔══██╗
+  █████╗  ██║   ██║██║   ██║██║    ██║   ██║█████╗  ██████╔╝
+  ██╔══╝  ╚██╗ ██╔╝██║   ██║██║    ╚██╗ ██╔╝██╔══╝  ██╔══██╗
+  ███████╗ ╚████╔╝ ╚██████╔╝███████╗╚████╔╝ ███████╗██║  ██║
+  ╚══════╝  ╚═══╝   ╚═════╝ ╚══════╝ ╚═══╝  ╚══════╝╚═╝  ╚═╝${RESET}
+`;
+
+const RUNTIMES = [
+  { name: "Claude Code", dir: ".claude", detected: () => fs.existsSync(path.join(HOME, ".claude")) },
+  { name: "Cursor", dir: ".cursor", detected: () => fs.existsSync(path.join(HOME, ".cursor")) },
+  { name: "Codex", dir: ".codex", detected: () => fs.existsSync(path.join(HOME, ".codex")) },
+  { name: "Windsurf", dir: ".windsurf", detected: () => fs.existsSync(path.join(HOME, ".windsurf")) },
+];
+
+function ask(rl, question) {
+  return new Promise((resolve) => rl.question(question, resolve));
 }
 
 function copyDir(src, dest) {
@@ -50,76 +77,152 @@ function checkPython() {
   }
 }
 
-function main() {
-  console.log("\n  Harness Evolver v0.1.0\n");
+function installForRuntime(runtimeDir, scope) {
+  const baseDir = scope === "local"
+    ? path.join(process.cwd(), runtimeDir)
+    : path.join(HOME, runtimeDir);
 
-  if (!checkPython()) {
-    console.error("  ERROR: python3 not found in PATH. Install Python 3.8+ first.");
-    process.exit(1);
-  }
-  log("\u2713 python3 found");
+  const commandsDir = path.join(baseDir, "commands", "harness-evolver");
+  const agentsDir = path.join(baseDir, "agents");
 
-  if (!fs.existsSync(CLAUDE_DIR)) {
-    console.error(`  ERROR: Claude Code directory not found at ${CLAUDE_DIR}`);
-    console.error("  Install Claude Code first: https://claude.ai/code");
-    process.exit(1);
-  }
-  log("\u2713 Claude Code detected");
-
-  // Copy skills
+  // Skills
   const skillsSource = path.join(PLUGIN_ROOT, "skills");
   if (fs.existsSync(skillsSource)) {
     for (const skill of fs.readdirSync(skillsSource, { withFileTypes: true })) {
       if (skill.isDirectory()) {
-        const src = path.join(skillsSource, skill.name);
-        const dest = path.join(COMMANDS_DIR, skill.name);
-        copyDir(src, dest);
-        log(`  skill: ${skill.name}`);
+        copyDir(path.join(skillsSource, skill.name), path.join(commandsDir, skill.name));
+        console.log(`  ${GREEN}✓${RESET} Installed skill: ${skill.name}`);
       }
     }
   }
 
-  // Copy agents
+  // Agents
   const agentsSource = path.join(PLUGIN_ROOT, "agents");
   if (fs.existsSync(agentsSource)) {
-    fs.mkdirSync(AGENTS_DIR, { recursive: true });
+    fs.mkdirSync(agentsDir, { recursive: true });
     for (const agent of fs.readdirSync(agentsSource)) {
-      copyFile(
-        path.join(agentsSource, agent),
-        path.join(AGENTS_DIR, agent)
-      );
-      log(`  agent: ${agent}`);
+      copyFile(path.join(agentsSource, agent), path.join(agentsDir, agent));
+      console.log(`  ${GREEN}✓${RESET} Installed agent: ${agent}`);
     }
   }
+}
 
-  // Copy tools
+function installTools() {
+  const toolsDir = path.join(HOME, ".harness-evolver", "tools");
   const toolsSource = path.join(PLUGIN_ROOT, "tools");
   if (fs.existsSync(toolsSource)) {
-    fs.mkdirSync(TOOLS_DIR, { recursive: true });
+    fs.mkdirSync(toolsDir, { recursive: true });
     for (const tool of fs.readdirSync(toolsSource)) {
       if (tool.endsWith(".py")) {
-        copyFile(
-          path.join(toolsSource, tool),
-          path.join(TOOLS_DIR, tool)
-        );
-        log(`  tool: ${tool}`);
+        copyFile(path.join(toolsSource, tool), path.join(toolsDir, tool));
+        console.log(`  ${GREEN}✓${RESET} Installed tool: ${tool}`);
       }
     }
   }
+}
 
-  // Copy examples
+function installExamples() {
+  const examplesDir = path.join(HOME, ".harness-evolver", "examples");
   const examplesSource = path.join(PLUGIN_ROOT, "examples");
   if (fs.existsSync(examplesSource)) {
-    copyDir(examplesSource, EXAMPLES_DIR);
-    log("  examples: classifier");
+    copyDir(examplesSource, examplesDir);
+    console.log(`  ${GREEN}✓${RESET} Installed examples: classifier`);
+  }
+}
+
+async function main() {
+  console.log(LOGO);
+  console.log(`  ${DIM}Harness Evolver v${VERSION}${RESET}`);
+  console.log(`  ${DIM}Meta-Harness-style autonomous harness optimization${RESET}`);
+  console.log();
+
+  // Check python
+  if (!checkPython()) {
+    console.error(`  ${RED}ERROR:${RESET} python3 not found in PATH. Install Python 3.8+ first.`);
+    process.exit(1);
+  }
+  console.log(`  ${GREEN}✓${RESET} python3 found`);
+
+  // Detect runtimes
+  const available = RUNTIMES.filter((r) => r.detected());
+  if (available.length === 0) {
+    console.error(`\n  ${RED}ERROR:${RESET} No supported runtime detected.`);
+    console.error(`  Install Claude Code, Cursor, Codex, or Windsurf first.`);
+    process.exit(1);
   }
 
-  console.log("\n  \u2713 Installed successfully!\n");
-  console.log("  Next steps:");
-  console.log("    1. Copy an example:  cp -r ~/.harness-evolver/examples/classifier ./my-project");
-  console.log("    2. cd my-project");
-  console.log("    3. /harness-evolve-init --harness harness.py --eval eval.py --tasks tasks/");
-  console.log("    4. /harness-evolve --iterations 5\n");
+  const rl = readline.createInterface({ input: process.stdin, output: process.stdout });
+
+  // Runtime selection
+  console.log(`\n  ${YELLOW}Which runtime(s) would you like to install for?${RESET}\n`);
+  available.forEach((r, i) => {
+    console.log(`  ${i + 1}) ${r.name.padEnd(14)} (~/${r.dir})`);
+  });
+  if (available.length > 1) {
+    console.log(`  ${available.length + 1}) All`);
+    console.log(`\n  ${DIM}Select multiple: 1,2 or 1 2${RESET}`);
+  }
+
+  const defaultChoice = "1";
+  const runtimeAnswer = await ask(rl, `\n  ${YELLOW}Choice [${defaultChoice}]:${RESET} `);
+  const runtimeInput = (runtimeAnswer.trim() || defaultChoice);
+
+  let selectedRuntimes;
+  if (runtimeInput === String(available.length + 1)) {
+    selectedRuntimes = available;
+  } else {
+    const indices = runtimeInput.split(/[,\s]+/).map((s) => parseInt(s, 10) - 1);
+    selectedRuntimes = indices
+      .filter((i) => i >= 0 && i < available.length)
+      .map((i) => available[i]);
+  }
+
+  if (selectedRuntimes.length === 0) {
+    selectedRuntimes = [available[0]];
+  }
+
+  // Scope selection
+  console.log(`\n  ${YELLOW}Where would you like to install?${RESET}\n`);
+  console.log(`  1) Global  (~/${selectedRuntimes[0].dir}) - available in all projects`);
+  console.log(`  2) Local   (./${selectedRuntimes[0].dir}) - this project only`);
+
+  const scopeAnswer = await ask(rl, `\n  ${YELLOW}Choice [1]:${RESET} `);
+  const scope = (scopeAnswer.trim() === "2") ? "local" : "global";
+
+  console.log();
+
+  // Install for each selected runtime
+  for (const runtime of selectedRuntimes) {
+    const target = scope === "local" ? `./${runtime.dir}` : `~/${runtime.dir}`;
+    console.log(`  Installing for ${CYAN}${runtime.name}${RESET} to ${target}`);
+    console.log();
+    installForRuntime(runtime.dir, scope);
+  }
+
+  // Tools and examples are always global
+  installTools();
+  installExamples();
+
+  // Write version file
+  const versionPath = path.join(HOME, ".harness-evolver", "VERSION");
+  fs.mkdirSync(path.dirname(versionPath), { recursive: true });
+  fs.writeFileSync(versionPath, VERSION);
+  console.log(`  ${GREEN}✓${RESET} Wrote VERSION (${VERSION})`);
+
+  console.log(`\n  ${GREEN}Done!${RESET} Open a project in Claude Code and run ${CYAN}/harness-evolver:init${RESET}`);
+  console.log(`\n  ${DIM}Quick start with example:${RESET}`);
+  console.log(`    cp -r ~/.harness-evolver/examples/classifier ./my-project`);
+  console.log(`    cd my-project && claude`);
+  console.log(`    /harness-evolver:init`);
+  console.log(`    /harness-evolver:evolve`);
+
+  console.log(`\n  ${DIM}GitHub: https://github.com/raphaelchristi/harness-evolver${RESET}`);
+  console.log();
+
+  rl.close();
 }
 
-main();
+main().catch((err) => {
+  console.error(`  ${RED}ERROR:${RESET} ${err.message}`);
+  process.exit(1);
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "harness-evolver",
-  "version": "0.2.1",
+  "version": "0.5.0",
   "description": "Meta-Harness-style autonomous harness optimization for Claude Code",
   "author": "Raphael Valdetaro Christi Cordeiro",
   "license": "MIT",

package/skills/compare/SKILL.md

@@ -0,0 +1,73 @@
+---
+name: compare
+description: "Use when the user wants to compare two harness versions, understand what changed between iterations, see why one version scored better than another, or debug a regression."
+argument-hint: "<vA> <vB>"
+allowed-tools: [Read, Bash, Glob, Grep]
+---
+
+# /harness-evolver:compare
+
+Compare two harness versions side by side.
+
+## Arguments
+
+- `vA` — first version (e.g., `v001`, `baseline`)
+- `vB` — second version (e.g., `v003`)
+
+If only one version given, compare it against the current best.
+If no versions given, compare the two most recent.
+
+## What To Do
+
+### 1. Code Diff
+
+```bash
+diff .harness-evolver/harnesses/{vA}/harness.py .harness-evolver/harnesses/{vB}/harness.py
+```
+
+If config changed:
+```bash
+diff .harness-evolver/harnesses/{vA}/config.json .harness-evolver/harnesses/{vB}/config.json
+```
+
+### 2. Score Comparison
+
+```bash
+cat .harness-evolver/harnesses/{vA}/scores.json
+cat .harness-evolver/harnesses/{vB}/scores.json
+```
+
+Report: combined_score delta, per-task wins/losses.
+
+### 3. Per-Task Analysis
+
+For tasks where scores diverge, show what each version produced:
+
+```bash
+cat .harness-evolver/harnesses/{vA}/traces/task_{ID}/output.json
+cat .harness-evolver/harnesses/{vB}/traces/task_{ID}/output.json
+```
+
+### 4. Proposal Context
+
+```bash
+cat .harness-evolver/harnesses/{vB}/proposal.md
+```
+
+Show what the proposer intended and whether the result matched expectations.
+
+## Report Format
+
+```
+v001 (0.62) vs v003 (0.71) — +0.09 improvement
+
+Code changes:
+  + Added few-shot examples (3 examples)
+  ~ Changed prompt template
+  - Removed retry logic
+
+Per-task:
+  task_001: 1.0 → 1.0 (unchanged)
+  task_007: 0.0 → 1.0 (FIXED — was cardiac, now correctly classified)
+  task_008: 1.0 → 0.0 (REGRESSION — was neurological, now wrong)
+```

package/skills/deploy/SKILL.md

@@ -0,0 +1,53 @@
+---
+name: deploy
+description: "Use when the user wants to use the best evolved harness in their project, promote a version to production, copy the winning harness back, or is done evolving and wants to apply the result."
+argument-hint: "[version]"
+allowed-tools: [Read, Write, Bash, Glob]
+---
+
+# /harness-evolver:deploy
+
+Promote the best (or specified) harness version back to the user's project.
+
+## Arguments
+
+- `version` — optional. If not given, deploys the best version from `summary.json`.
+
+## What To Do
+
+### 1. Identify Best Version
+
+```bash
+python3 -c "import json; s=json.load(open('.harness-evolver/summary.json')); print(s['best']['version'], s['best']['combined_score'])"
+```
+
+Or use the user-specified version.
+
+### 2. Show What Will Be Deployed
+
+```bash
+cat .harness-evolver/harnesses/{version}/proposal.md
+cat .harness-evolver/harnesses/{version}/scores.json
+```
+
+Report: version, score, improvement over baseline, what changed.
+
+### 3. Ask for Confirmation
+
+> Deploy `{version}` (score: {score}, +{delta} over baseline) to your project?
+> This will copy `harness.py` and `config.json` to the project root.
+
+### 4. Copy Files
+
+```bash
+cp .harness-evolver/harnesses/{version}/harness.py ./harness.py
+cp .harness-evolver/harnesses/{version}/config.json ./config.json  # if exists
+```
+
+If the original entry point had a different name (e.g., `graph.py`), ask the user where to put it.
+
+### 5. Report
+
+- What was copied and where
+- Score improvement: baseline → deployed version
+- Suggest: review the diff before committing

package/skills/diagnose/SKILL.md

@@ -0,0 +1,96 @@
+---
+name: diagnose
+description: "Use when the user wants to understand why a specific harness version failed, investigate a regression, analyze trace data, or debug a low score. Also use when the user says 'why did v003 fail' or 'what went wrong'."
+argument-hint: "[version]"
+allowed-tools: [Read, Bash, Glob, Grep]
+---
+
+# /harness-evolver:diagnose
+
+Deep analysis of a harness version's execution traces and scores.
+
+## Arguments
+
+- `version` — version to diagnose (e.g., `v003`). If not given, diagnose the worst or most recent regression.
+
+## Resolve Tool Path
+
+```bash
+TOOLS=$([ -d ".harness-evolver/tools" ] && echo ".harness-evolver/tools" || echo "$HOME/.harness-evolver/tools")
+```
+
+## What To Do
+
+### 1. Identify the Version
+
+If not specified, find the worst or most recent regression:
+
+```bash
+python3 $TOOLS/state.py show --base-dir .harness-evolver
+cat .harness-evolver/summary.json
+```
+
+### 2. Score Breakdown
+
+```bash
+cat .harness-evolver/harnesses/{version}/scores.json
+```
+
+Identify which tasks failed (`score: 0.0`) and which passed.
+
+### 3. Trace Analysis (failed tasks)
+
+For each failed task:
+
+```bash
+cat .harness-evolver/harnesses/{version}/traces/{task_id}/input.json
+cat .harness-evolver/harnesses/{version}/traces/{task_id}/output.json
+```
+
+Look for patterns: wrong format? wrong category? empty output? crash?
+
+### 4. Error Search
+
+```bash
+grep -r "error\|Error\|FAIL\|exception\|Traceback" .harness-evolver/harnesses/{version}/traces/
+cat .harness-evolver/harnesses/{version}/traces/stderr.log
+```
+
+### 5. Compare with Parent
+
+Read the proposal to find the parent version:
+
+```bash
+cat .harness-evolver/harnesses/{version}/proposal.md
+```
+
+Then diff:
+
+```bash
+diff .harness-evolver/harnesses/{parent}/harness.py .harness-evolver/harnesses/{version}/harness.py
+```
+
+### 6. LangSmith (if available)
+
+If `langsmith-cli` is installed and LangSmith is configured:
+
+```bash
+langsmith-cli --json runs list --project harness-evolver-{version} --failed --fields id,name,error,inputs
+langsmith-cli --json runs stats --project harness-evolver-{version}
+```
+
+### 7. Report
+
+```
+Diagnosis: v003 (score: 0.31) — REGRESSION from v001 (0.62)
+
+Root cause: Prompt template change broke JSON parsing
+  - 4/10 tasks returned malformed output
+  - stderr shows: json.JSONDecodeError on 4 tasks
+  - The change on line 42 removed the "Reply with ONLY..." instruction
+
+Affected tasks: task_002, task_005, task_007, task_010
+Unaffected tasks: task_001, task_003, task_004, task_006, task_008, task_009
+
+Recommendation: Revert the prompt change, keep the retry logic from v002
+```

package/skills/evolve/SKILL.md

@@ -0,0 +1,94 @@
+---
+name: evolve
+description: "Use when the user wants to run the optimization loop, improve harness performance, evolve the harness, or iterate on harness quality. Requires .harness-evolver/ to exist (run harness-evolver:init first)."
+argument-hint: "[--iterations N]"
+allowed-tools: [Read, Write, Edit, Bash, Glob, Grep, Agent]
+---
+
+# /harness-evolve
+
+Run the autonomous propose-evaluate-iterate loop.
+
+## Prerequisites
+
+`.harness-evolver/summary.json` must exist. If not, tell user to run `harness-evolver:init`.
+
+## Resolve Tool Path
+
+```bash
+TOOLS=$([ -d ".harness-evolver/tools" ] && echo ".harness-evolver/tools" || echo "$HOME/.harness-evolver/tools")
+```
+
+## Parse Arguments
+
+- `--iterations N` (default: 10)
+- Read `config.json` for `evolution.stagnation_limit` (default: 3) and `evolution.target_score`
+
+## The Loop
+
+For each iteration:
+
+### 1. Get Next Version
+
+```bash
+python3 -c "import json; s=json.load(open('.harness-evolver/summary.json')); print(f'v{s[\"iterations\"]+1:03d}')"
+```
+
+### 2. Propose
+
+Spawn the `harness-evolver-proposer` agent:
+
+> You are proposing iteration {i}. Create version {version} in `.harness-evolver/harnesses/{version}/`.
+> Working directory contains `.harness-evolver/` with all prior candidates and traces.
+
+The proposer creates: `harness.py`, `config.json`, `proposal.md`.
+
+### 3. Validate
+
+```bash
+python3 $TOOLS/evaluate.py validate \
+    --harness .harness-evolver/harnesses/{version}/harness.py \
+    --config .harness-evolver/harnesses/{version}/config.json
+```
+
+If fails: one retry via proposer. If still fails: score 0.0, continue.
+
+### 4. Evaluate
+
+```bash
+python3 $TOOLS/evaluate.py run \
+    --harness .harness-evolver/harnesses/{version}/harness.py \
+    --config .harness-evolver/harnesses/{version}/config.json \
+    --tasks-dir .harness-evolver/eval/tasks/ \
+    --eval .harness-evolver/eval/eval.py \
+    --traces-dir .harness-evolver/harnesses/{version}/traces/ \
+    --scores .harness-evolver/harnesses/{version}/scores.json \
+    --timeout 60
+```
+
+### 5. Update State
+
+```bash
+python3 $TOOLS/state.py update \
+    --base-dir .harness-evolver \
+    --version {version} \
+    --scores .harness-evolver/harnesses/{version}/scores.json \
+    --proposal .harness-evolver/harnesses/{version}/proposal.md
+```
+
+### 6. Report
+
+Read `summary.json`. Print: `Iteration {i}/{N}: {version} scored {score} (best: {best} at {best_score})`
+
+### 7. Check Stop Conditions
+
+- **Stagnation**: last 3 scores within 1% of each other → stop
+- **Target**: `combined_score >= target_score` → stop
+- **N reached**: done
+
+## When Loop Ends — Final Report
+
+- Best version and score
+- Improvement over baseline (absolute and %)
+- Total iterations run
+- Suggest: "The best harness is at `.harness-evolver/harnesses/{best}/harness.py`. Copy it to your project."

package/skills/init/SKILL.md

@@ -0,0 +1,66 @@
+---
+name: init
+description: "Use when the user wants to set up harness optimization in their project, optimize an LLM agent, improve a harness, or mentions harness-evolver for the first time in a project without .harness-evolver/ directory."
+argument-hint: "[directory]"
+allowed-tools: [Read, Write, Edit, Bash, Glob, Grep, Agent]
+---
+
+# /harness-evolve-init
+
+Set up the Harness Evolver in a project. Scans the codebase, identifies the entry point, creates missing artifacts, runs baseline evaluation.
+
+## Resolve Tool Path
+
+```bash
+TOOLS=$([ -d ".harness-evolver/tools" ] && echo ".harness-evolver/tools" || echo "$HOME/.harness-evolver/tools")
+```
+
+Use `$TOOLS` prefix for all tool calls below.
+
+## Phase 1: Scan
+
+```bash
+find . -maxdepth 3 -type f -name "*.py" | head -30
+python3 $TOOLS/detect_stack.py .
+```
+
+Look for:
+- Entry points: files with `if __name__`, or named `main.py`, `app.py`, `agent.py`, `graph.py`, `pipeline.py`, `bot.py`
+- Existing eval: `eval.py`, `score.py`, `judge.py`
+- Existing tasks: directories with JSON files containing `id` + `input` fields
+- Config: `config.json`, `config.yaml`, `.env`
+
+## Phase 2: Create What's Missing
+
+Three artifacts needed. For each — use existing if found, create if not.
+
+**Harness** (`harness.py`): If user's entry point doesn't match our CLI interface (`--input`, `--output`, `--traces-dir`, `--config`), create a thin wrapper that imports their code. Read their entry point first to understand the I/O format. Ask if unsure.
+
+**Eval** (`eval.py`): Ask the user what "correct" means for their domain. Generate the simplest eval that gives signal. Even rough scoring works — the evolver iterates.
+
+**Tasks** (`tasks/`): If no test data exists, ask the user for 5-10 example input/output pairs. Each task is `{"id": "task_001", "input": "...", "expected": "...", "metadata": {}}`.
+
+## Phase 3: Run Init
+
+```bash
+python3 $TOOLS/init.py [directory] \
+    --harness harness.py --eval eval.py --tasks tasks/ \
+    --tools-dir $TOOLS
+```
+
+Add `--harness-config config.json` if a config exists.
+
+## After Init — Report
+
+- What was detected vs created
+- Stack + integrations (LangSmith, Context7)
+- Baseline score
+- Next: `harness-evolver:evolve` to start
+
+## Gotchas
+
+- The harness must write valid JSON to `--output`. If the user's code returns non-JSON, the wrapper must serialize it.
+- Tasks must have unique `id` fields. Duplicate IDs cause silent eval errors.
+- The `expected` field is never shown to the harness — only the eval script sees it.
+- If `.harness-evolver/` already exists, warn before overwriting.
+- If no Python files exist in CWD, the user is probably in the wrong directory.

package/skills/status/SKILL.md

@@ -0,0 +1,34 @@
+---
+name: status
+description: "Use when the user asks about evolution progress, current scores, best harness version, how many iterations ran, or whether the loop is stagnating. Also use when the user says 'status', 'progress', or 'how is it going'."
+allowed-tools: [Read, Bash]
+---
+
+# /harness-evolve-status
+
+Show evolution progress.
+
+## Resolve Tool Path
+
+```bash
+TOOLS=$([ -d ".harness-evolver/tools" ] && echo ".harness-evolver/tools" || echo "$HOME/.harness-evolver/tools")
+```
+
+## What To Do
+
+If `.harness-evolver/` does not exist, tell user to run `harness-evolver:init` first.
+
+Otherwise:
+
+```bash
+python3 $TOOLS/state.py show --base-dir .harness-evolver
+```
+
+Then read and display `.harness-evolver/STATE.md` for the full history table.
+
+## If User Wants More Detail
+
+- Scores per task: `cat .harness-evolver/harnesses/{version}/scores.json`
+- What changed: `cat .harness-evolver/harnesses/{version}/proposal.md`
+- Compare two versions: `diff .harness-evolver/harnesses/{vA}/harness.py .harness-evolver/harnesses/{vB}/harness.py`
+- Full history: `cat .harness-evolver/PROPOSER_HISTORY.md`

package/skills/harness-evolve/SKILL.md

@@ -1,93 +0,0 @@
----
-name: harness-evolve
-description: "Run the harness evolution loop. Autonomously proposes, evaluates, and iterates on harness designs using full execution traces as feedback."
-argument-hint: "[--iterations N] [--candidates-per-iter N]"
-allowed-tools: [Read, Write, Edit, Bash, Glob, Grep, Agent]
----
-
-# /harness-evolve
-
-Run the Meta-Harness optimization loop.
-
-## Arguments
-
-- `--iterations N` (default: 10) — number of evolution iterations
-- `--candidates-per-iter N` (default: 1) — harnesses per iteration
-
-## Prerequisites
-
-Run `/harness-evolve-init` first. The `.harness-evolver/` directory must exist with a valid `summary.json`.
-
-## The Loop
-
-For each iteration i from 1 to N:
-
-### 1. PROPOSE
-
-Determine the next version number by reading `summary.json`:
-
-```bash
-python3 -c "import json; s=json.load(open('.harness-evolver/summary.json')); print(f'v{s[\"iterations\"]+1:03d}')"
-```
-
-Spawn the `harness-evolver-proposer` agent with this prompt:
-
-> You are proposing iteration {i}. Create version {version_number} in `.harness-evolver/harnesses/{version_number}/`.
-> Working directory contains `.harness-evolver/` with all prior candidates and traces.
-
-The proposer agent will create:
-- `.harness-evolver/harnesses/v{NNN}/harness.py`
-- `.harness-evolver/harnesses/v{NNN}/config.json`
-- `.harness-evolver/harnesses/v{NNN}/proposal.md`
-
-### 2. VALIDATE
-
-```bash
-python3 ~/.harness-evolver/tools/evaluate.py validate \
-    --harness .harness-evolver/harnesses/v{NNN}/harness.py \
-    --config .harness-evolver/harnesses/v{NNN}/config.json
-```
-
-If validation fails, ask the proposer to fix (1 retry). If it fails again, set score to 0.0 and continue.
-
-### 3. EVALUATE
-
-```bash
-python3 ~/.harness-evolver/tools/evaluate.py run \
-    --harness .harness-evolver/harnesses/v{NNN}/harness.py \
-    --config .harness-evolver/harnesses/v{NNN}/config.json \
-    --tasks-dir .harness-evolver/eval/tasks/ \
-    --eval .harness-evolver/eval/eval.py \
-    --traces-dir .harness-evolver/harnesses/v{NNN}/traces/ \
-    --scores .harness-evolver/harnesses/v{NNN}/scores.json \
-    --timeout 60
-```
-
-### 4. UPDATE STATE
-
-```bash
-python3 ~/.harness-evolver/tools/state.py update \
-    --base-dir .harness-evolver \
-    --version v{NNN} \
-    --scores .harness-evolver/harnesses/v{NNN}/scores.json \
-    --proposal .harness-evolver/harnesses/v{NNN}/proposal.md
-```
-
-### 5. REPORT
-
-Read the updated `summary.json` and report:
-- `Iteration {i}/{N}: v{NNN} scored {score} (best: v{best} at {best_score})`
-- If regression (score < parent score): warn
-- If new best: celebrate
-
-### Stop Conditions
-
-- All N iterations completed
-- **Stagnation**: 3 consecutive iterations without >1% improvement. Read `summary.json` history to check.
-- **Target reached**: if `config.json` has `target_score` set and achieved.
-
-When stopping, report final summary: best version, score, number of iterations, improvement over baseline.
-
-## Tool Path Resolution
-
-Check `.harness-evolver/tools/` first (local override), then `~/.harness-evolver/tools/` (global install).

package/skills/harness-evolve-init/SKILL.md

@@ -1,198 +0,0 @@
----
-name: harness-evolve-init
-description: "Initialize harness evolution in the current project. Scans the codebase, identifies the entry point, and helps create harness wrapper, eval script, and test cases if they don't exist."
-argument-hint: "[directory]"
-allowed-tools: [Read, Write, Edit, Bash, Glob, Grep, Agent]
----
-
-# /harness-evolve-init
-
-Initialize the Harness Evolver for this project.
-
-## Usage
-
-```
-/harness-evolve-init                    # setup in current directory
-/harness-evolve-init ./my-project       # setup in a specific directory
-```
-
-## Your Job: A 3-Phase Setup Wizard
-
-You are the intelligent layer. The init.py tool is dumb — it takes paths. Your job is to figure out what to pass it, creating files if needed.
-
-### Phase 1: SCAN the project
-
-Read the project structure to understand what exists:
-
-```bash
-find . -maxdepth 3 -type f -name "*.py" | head -30
-ls -la
-```
-
-Look for:
-- **Entry point candidates:** `main.py`, `app.py`, `agent.py`, `graph.py`, `pipeline.py`, `bot.py`, `run.py`, or any file with `if __name__` block
-- **Existing eval/test files:** `eval.py`, `test_*.py`, `score.py`, `judge.py`
-- **Existing test data:** `tasks/`, `tests/`, `data/`, `examples/`, `fixtures/`, any dir with JSON/JSONL files
-- **Config files:** `config.json`, `config.yaml`, `.env`
-- **Framework clues:** imports of langchain, langgraph, openai, anthropic, crewai, etc.
-
-Also run stack detection:
-```bash
-python3 ~/.harness-evolver/tools/detect_stack.py .
-```
-
-### Phase 2: CREATE what's missing
-
-There are 3 artifacts needed. For each, check if it exists or needs to be created.
-
-#### A. The Harness (`harness.py`)
-
-**If `harness.py` already exists with the right interface** (`--input`, `--output`): use it directly.
-
-**If the project has an entry point but NOT in our format:** Create a `harness.py` wrapper.
-
-Read the entry point to understand its input/output format, then generate a wrapper:
-
-```python
-#!/usr/bin/env python3
-"""Harness wrapper for [project name]. Generated by harness-evolve-init."""
-
-import argparse
-import json
-import sys
-
-# Import the actual project code
-from [entry_module] import [main_function_or_class]
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--input", required=True)
-    parser.add_argument("--output", required=True)
-    parser.add_argument("--traces-dir", default=None)
-    parser.add_argument("--config", default=None)
-    args = parser.parse_args()
-
-    task = json.load(open(args.input))
-    config = json.load(open(args.config)) if args.config else {}
-
-    # Call the actual project code
-    result = [call_the_project](task["input"], **config)
-
-    json.dump({"id": task["id"], "output": str(result)}, open(args.output, "w"))
-
-if __name__ == "__main__":
-    main()
-```
-
-Adapt this template based on what you learned from reading the entry point. Ask the user to confirm if you're unsure about the input/output mapping.
-
-#### B. The Eval (`eval.py`)
-
-**If `eval.py` exists:** use it.
-
-**If not:** Ask the user what "correct" means for their use case, then generate one:
-
-- **Classification/extraction:** exact match or fuzzy match
-- **Chatbot/QA:** LLM-as-judge (requires API key) or keyword matching
-- **Code generation:** execution-based (run the code, check output)
-- **RAG:** relevance scoring
-
-Start with the simplest eval that works. The evolver can iterate on the harness without a perfect eval — even a rough eval gives signal.
-
-```python
-#!/usr/bin/env python3
-"""Eval script for [project]. Generated by harness-evolve-init."""
-
-import argparse
-import json
-import os
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--results-dir", required=True)
-    parser.add_argument("--tasks-dir", required=True)
-    parser.add_argument("--scores", required=True)
-    args = parser.parse_args()
-
-    correct, total = 0, 0
-    per_task = {}
-
-    for fname in sorted(os.listdir(args.tasks_dir)):
-        if not fname.endswith(".json"):
-            continue
-        with open(os.path.join(args.tasks_dir, fname)) as f:
-            task = json.load(f)
-        task_id = task["id"]
-
-        result_path = os.path.join(args.results_dir, fname)
-        if not os.path.exists(result_path):
-            per_task[task_id] = {"score": 0.0, "error": "no output"}
-            total += 1
-            continue
-
-        with open(result_path) as f:
-            result = json.load(f)
-
-        # ADAPT THIS: define what "correct" means for this project
-        expected = task.get("expected", "")
-        actual = result.get("output", "")
-        match = actual.lower().strip() == expected.lower().strip()
-
-        per_task[task_id] = {"score": 1.0 if match else 0.0, "expected": expected, "actual": actual}
-        correct += int(match)
-        total += 1
-
-    accuracy = correct / total if total > 0 else 0.0
-    json.dump({
-        "combined_score": accuracy,
-        "accuracy": accuracy,
-        "total_tasks": total,
-        "correct": correct,
-        "per_task": per_task,
-    }, open(args.scores, "w"), indent=2)
-
-if __name__ == "__main__":
-    main()
-```
-
-#### C. Test Cases (`tasks/`)
-
-**If `tasks/` exists with JSON files:** use it.
-
-**If the project has test data in another format:** Convert it to our format.
-
-**If no test data exists:** Help create 5-10 test cases. Ask the user:
-> "What are typical inputs to your system? And what are the expected outputs? Give me 3-5 examples and I'll create the task files."
-
-Each task file is:
-```json
-{"id": "task_001", "input": "...", "expected": "...", "metadata": {}}
-```
-
-### Phase 3: RUN init.py
-
-Once all 3 artifacts exist, run:
-
-```bash
-python3 ~/.harness-evolver/tools/init.py \
-    --harness harness.py \
-    --eval eval.py \
-    --tasks tasks/ \
-    --tools-dir ~/.harness-evolver/tools
-```
-
-If a harness config exists, add `--harness-config config.json`.
-
-If `~/.harness-evolver/tools/init.py` does not exist, check `.harness-evolver/tools/init.py`.
-
-### After init completes, report:
-
-- What was detected vs created
-- Stack detected (libraries)
-- Integrations available (LangSmith, Context7)
-- Baseline score
-- Next step: `/harness-evolve` to start the optimization loop
-
-## Key Principle
-
-**Don't ask the user to restructure their project.** You adapt to them. If they have a LangGraph graph in `src/graph.py`, you create a thin wrapper — you don't ask them to rename it to `harness.py`. The wrapper IS the harness.

package/skills/harness-evolve-status/SKILL.md

@@ -1,25 +0,0 @@
----
-name: harness-evolve-status
-description: "Show the current status of harness evolution: best score, iteration count, progress history."
-allowed-tools: [Read, Bash]
----
-
-# /harness-evolve-status
-
-Show the current evolution status.
-
-## What To Do
-
-```bash
-python3 ~/.harness-evolver/tools/state.py show --base-dir .harness-evolver
-```
-
-If that doesn't exist, try:
-
-```bash
-python3 .harness-evolver/tools/state.py show --base-dir .harness-evolver
-```
-
-Also read and display the contents of `.harness-evolver/STATE.md` for the full status table.
-
-If `.harness-evolver/` doesn't exist, tell the user to run `/harness-evolve-init` first.