harness-evolver 3.0.3 → 3.0.5

package/bin/install.js

@@ -190,71 +190,140 @@ function installTools() {
 }
 
 function installPythonDeps() {
-  console.log(`\n  ${YELLOW}Installing Python dependencies...${RESET}`);
-
-  // Try multiple pip variants
-  const commands = [
-    "pip install langsmith openevals",
-    "uv pip install langsmith openevals",
-    "pip3 install langsmith openevals",
-    "python3 -m pip install langsmith openevals",
+  const venvDir = path.join(HOME, ".evolver", "venv");
+  const venvPython = path.join(venvDir, "bin", "python");
+  const venvPip = path.join(venvDir, "bin", "pip");
+
+  console.log(`\n  ${YELLOW}Setting up Python environment...${RESET}`);
+
+  // Create venv if it doesn't exist
+  if (!fs.existsSync(venvPython)) {
+    console.log(`  Creating isolated venv at ~/.evolver/venv/`);
+    const venvCommands = [
+      `uv venv "${venvDir}"`,
+      `python3 -m venv "${venvDir}"`,
+    ];
+    let created = false;
+    for (const cmd of venvCommands) {
+      try {
+        execSync(cmd, { stdio: "pipe", timeout: 30000 });
+        created = true;
+        break;
+      } catch {
+        continue;
+      }
+    }
+    if (!created) {
+      console.log(`  ${RED}Failed to create venv.${RESET}`);
+      console.log(`    Run manually: ${BOLD}python3 -m venv ~/.evolver/venv${RESET}`);
+      return false;
+    }
+    console.log(`  ${GREEN}✓${RESET} venv created`);
+  } else {
+    console.log(`  ${GREEN}✓${RESET} venv exists at ~/.evolver/venv/`);
+  }
+
+  // Install/upgrade deps in the venv
+  const installCommands = [
+    `uv pip install --python "${venvPython}" langsmith openevals`,
+    `"${venvPip}" install --upgrade langsmith openevals`,
+    `"${venvPython}" -m pip install --upgrade langsmith openevals`,
   ];
 
-  for (const cmd of commands) {
+  for (const cmd of installCommands) {
     try {
       execSync(cmd, { stdio: "pipe", timeout: 120000 });
-      console.log(`  ${GREEN}✓${RESET} langsmith + openevals installed`);
+      console.log(`  ${GREEN}✓${RESET} langsmith + openevals installed in venv`);
       return true;
     } catch {
       continue;
     }
   }
 
-  console.log(`  ${YELLOW}!${RESET} Could not auto-install Python packages.`);
-  console.log(`    Run manually: ${BOLD}pip install langsmith openevals${RESET}`);
+  console.log(`  ${YELLOW}!${RESET} Could not install packages in venv.`);
+  console.log(`    Run manually: ${BOLD}~/.evolver/venv/bin/pip install langsmith openevals${RESET}`);
   return false;
 }
 
 async function configureLangSmith(rl) {
-  console.log(`\n  ${YELLOW}LangSmith Configuration${RESET} ${DIM}(required for v3)${RESET}\n`);
+  console.log(`\n  ${BOLD}${GREEN}LangSmith Configuration${RESET} ${DIM}(required)${RESET}\n`);
 
-  // Check if already configured
   const langsmithCredsDir = process.platform === "darwin"
     ? path.join(HOME, "Library", "Application Support", "langsmith-cli")
     : path.join(HOME, ".config", "langsmith-cli");
   const langsmithCredsFile = path.join(langsmithCredsDir, "credentials");
+  const hasLangsmithCli = checkCommand("langsmith-cli --version");
+
+  // --- Step 1: API Key ---
+  let hasKey = false;
 
-  // Check env var
   if (process.env.LANGSMITH_API_KEY) {
     console.log(`  ${GREEN}✓${RESET} LANGSMITH_API_KEY found in environment`);
-    return;
+    hasKey = true;
+  } else if (fs.existsSync(langsmithCredsFile)) {
+    try {
+      const content = fs.readFileSync(langsmithCredsFile, "utf8");
+      if (content.includes("LANGSMITH_API_KEY=lsv2_")) {
+        console.log(`  ${GREEN}✓${RESET} API key found in credentials file`);
+        hasKey = true;
+      }
+    } catch {}
   }
 
-  // Check credentials file
-  if (fs.existsSync(langsmithCredsFile)) {
-    console.log(`  ${GREEN}✓${RESET} LangSmith credentials found at ${DIM}${langsmithCredsFile}${RESET}`);
-    return;
+  if (!hasKey) {
+    console.log(`  ${BOLD}LangSmith API Key${RESET} — get yours at ${DIM}https://smith.langchain.com/settings${RESET}`);
+    console.log(`  ${DIM}LangSmith is required. The evolver won't work without it.${RESET}\n`);
+
+    // Keep asking until they provide a key or explicitly skip
+    let attempts = 0;
+    while (!hasKey && attempts < 3) {
+      const apiKey = await ask(rl, `  ${YELLOW}Paste your LangSmith API key (lsv2_pt_...):${RESET} `);
+      const key = apiKey.trim();
+
+      if (key && key.startsWith("lsv2_")) {
+        try {
+          fs.mkdirSync(langsmithCredsDir, { recursive: true });
+          fs.writeFileSync(langsmithCredsFile, `LANGSMITH_API_KEY=${key}\n`);
+          console.log(`  ${GREEN}✓${RESET} API key saved`);
+          hasKey = true;
+        } catch {
+          console.log(`  ${RED}Failed to save.${RESET} Add to your shell: export LANGSMITH_API_KEY=${key}`);
+          hasKey = true; // they have the key, just couldn't save
+        }
+      } else if (key) {
+        console.log(`  ${YELLOW}Invalid — LangSmith keys start with lsv2_${RESET}`);
+        attempts++;
+      } else {
+        // Empty input — skip
+        console.log(`\n  ${RED}WARNING:${RESET} No API key configured.`);
+        console.log(`  ${BOLD}/evolver:setup will not work${RESET} until you set LANGSMITH_API_KEY.`);
+        console.log(`  Run: ${DIM}export LANGSMITH_API_KEY=lsv2_pt_your_key${RESET}\n`);
+        break;
+      }
+    }
   }
 
-  // Ask for API key
-  console.log(`  ${BOLD}LangSmith API Key${RESET} — get yours at ${DIM}https://smith.langchain.com/settings${RESET}`);
-  console.log(`  ${DIM}LangSmith is required for v3 (datasets, experiments, evaluators).${RESET}\n`);
-  const apiKey = await ask(rl, `  ${YELLOW}Paste your LangSmith API key:${RESET} `);
-  const key = apiKey.trim();
+  // --- Step 2: langsmith-cli ---
+  if (hasLangsmithCli) {
+    console.log(`  ${GREEN}✓${RESET} langsmith-cli installed`);
+  } else {
+    console.log(`\n  ${BOLD}langsmith-cli${RESET} — optional but useful for debugging traces`);
+    console.log(`  ${DIM}Quick project listing, trace inspection, run stats from terminal.${RESET}`);
+    const lsCliAnswer = await ask(rl, `\n  ${YELLOW}Install langsmith-cli? [Y/n]:${RESET} `);
+    if (lsCliAnswer.trim().toLowerCase() !== "n") {
+      console.log(`\n  Installing langsmith-cli...`);
+      try {
+        execSync("uv tool install langsmith-cli 2>/dev/null || pip install langsmith-cli 2>/dev/null || pip3 install langsmith-cli", { stdio: "pipe", timeout: 60000 });
+        console.log(`  ${GREEN}✓${RESET} langsmith-cli installed`);
 
-  if (key && key.startsWith("lsv2_")) {
-    try {
-      fs.mkdirSync(langsmithCredsDir, { recursive: true });
-      fs.writeFileSync(langsmithCredsFile, `LANGSMITH_API_KEY=${key}\n`);
-      console.log(`  ${GREEN}✓${RESET} API key saved to ${DIM}${langsmithCredsFile}${RESET}`);
-    } catch {
-      console.log(`  ${RED}Failed to save.${RESET} Add to your shell: export LANGSMITH_API_KEY=${key}`);
+        // If we have a key, auto-authenticate
+        if (hasKey && fs.existsSync(langsmithCredsFile)) {
+          console.log(`  ${GREEN}✓${RESET} langsmith-cli auto-authenticated (credentials file exists)`);
+        }
+      } catch {
+        console.log(`  ${YELLOW}!${RESET} Could not install. Try manually: ${DIM}uv tool install langsmith-cli${RESET}`);
+      }
     }
-  } else if (key) {
-    console.log(`  ${YELLOW}Doesn't look like a LangSmith key (should start with lsv2_).${RESET}`);
-    console.log(`  Add to your shell: ${BOLD}export LANGSMITH_API_KEY=your_key${RESET}`);
-  } else {
-    console.log(`  ${YELLOW}Skipped.${RESET} You must set LANGSMITH_API_KEY before using /evolver:setup`);
   }
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "harness-evolver",
-  "version": "3.0.3",
+  "version": "3.0.5",
   "description": "LangSmith-native autonomous agent optimization for Claude Code",
   "author": "Raphael Valdetaro",
   "license": "MIT",

package/skills/evolve/SKILL.md

@@ -13,12 +13,15 @@ Run the autonomous propose-evaluate-iterate loop using LangSmith as the evaluati
 
 `.evolver.json` must exist. If not, tell user to run `evolver:setup`.
 
-## Resolve Tool Path
+## Resolve Tool Path and Python
 
 ```bash
 TOOLS=$([ -d ".evolver/tools" ] && echo ".evolver/tools" || echo "$HOME/.evolver/tools")
+EVOLVER_PY=$([ -f "$HOME/.evolver/venv/bin/python" ] && echo "$HOME/.evolver/venv/bin/python" || echo "python3")
 ```
 
+Use `$EVOLVER_PY` instead of `python3` for ALL tool invocations.
+
 ## Parse Arguments
 
 - `--iterations N` (default: from interactive question or 5)
@@ -76,7 +79,7 @@ Run trace insights from the best experiment:
 
 ```bash
 BEST=$(python3 -c "import json; print(json.load(open('.evolver.json'))['best_experiment'])")
-python3 $TOOLS/trace_insights.py \
+$EVOLVER_PY $TOOLS/trace_insights.py \
     --from-experiment "$BEST" \
     --output trace_insights.json 2>/dev/null
 ```
@@ -86,7 +89,7 @@ If a production project is configured, also gather production insights:
 ```bash
 PROD=$(python3 -c "import json; c=json.load(open('.evolver.json')); print(c.get('production_project',''))")
 if [ -n "$PROD" ] && [ ! -f "production_seed.json" ]; then
-    python3 $TOOLS/seed_from_traces.py \
+    $EVOLVER_PY $TOOLS/seed_from_traces.py \
         --project "$PROD" --use-sdk \
         --output-md production_seed.md \
         --output-json production_seed.json \
@@ -99,7 +102,7 @@ fi
 Read the best experiment results and cluster failures:
 
 ```bash
-python3 $TOOLS/read_results.py \
+$EVOLVER_PY $TOOLS/read_results.py \
     --experiment "$BEST" \
     --config .evolver.json \
     --output best_results.json 2>/dev/null
@@ -174,7 +177,7 @@ Wait for all 5 to complete.
 For each worktree that has changes (proposer committed something):
 
 ```bash
-python3 $TOOLS/run_eval.py \
+$EVOLVER_PY $TOOLS/run_eval.py \
     --config .evolver.json \
     --worktree-path {worktree_path} \
     --experiment-prefix v{NNN}{suffix} \
@@ -186,7 +189,7 @@ Each candidate becomes a separate LangSmith experiment.
 ### 4. Compare All Candidates
 
 ```bash
-python3 $TOOLS/read_results.py \
+$EVOLVER_PY $TOOLS/read_results.py \
     --experiments "v{NNN}a,v{NNN}b,v{NNN}c,v{NNN}d,v{NNN}e" \
     --config .evolver.json \
     --output comparison.json

package/skills/setup/SKILL.md

@@ -35,23 +35,20 @@ If `MISSING`: "Set your LangSmith API key: `export LANGSMITH_API_KEY=lsv2_pt_...
 
 The tools auto-load the key from the credentials file, but the env var takes precedence.
 
-Python 3.10+ with `langsmith` and `openevals` packages must be installed:
-
-```bash
-pip install langsmith openevals 2>/dev/null || uv pip install langsmith openevals
-```
-
-## Resolve Tool Path
+## Resolve Tool Path and Python
 
 ```bash
 TOOLS=$([ -d ".evolver/tools" ] && echo ".evolver/tools" || echo "$HOME/.evolver/tools")
+EVOLVER_PY=$([ -f "$HOME/.evolver/venv/bin/python" ] && echo "$HOME/.evolver/venv/bin/python" || echo "python3")
 ```
 
+Use `$EVOLVER_PY` instead of `python3` for ALL tool invocations. This ensures the venv with langsmith+openevals is used.
+
 ## Phase 1: Explore Project (automatic)
 
 ```bash
 find . -maxdepth 3 -type f -name "*.py" | head -30
-python3 $TOOLS/detect_stack.py .
+$EVOLVER_PY $TOOLS/detect_stack.py .
 ```
 
 Look for:
@@ -145,7 +142,7 @@ If "I have test data": ask for file path.
 Build the setup.py command based on all gathered information:
 
 ```bash
-python3 $TOOLS/setup.py \
+$EVOLVER_PY $TOOLS/setup.py \
     --project-name "{project_name}" \
     --entry-point "{run_command}" \
     --framework "{framework}" \