harness-evolver 3.0.2 → 3.0.4

package/bin/install.js

@@ -216,45 +216,84 @@ function installPythonDeps() {
 }
 
 async function configureLangSmith(rl) {
-  console.log(`\n  ${YELLOW}LangSmith Configuration${RESET} ${DIM}(required for v3)${RESET}\n`);
+  console.log(`\n  ${BOLD}${GREEN}LangSmith Configuration${RESET} ${DIM}(required)${RESET}\n`);
 
-  // Check if already configured
   const langsmithCredsDir = process.platform === "darwin"
     ? path.join(HOME, "Library", "Application Support", "langsmith-cli")
     : path.join(HOME, ".config", "langsmith-cli");
   const langsmithCredsFile = path.join(langsmithCredsDir, "credentials");
+  const hasLangsmithCli = checkCommand("langsmith-cli --version");
+
+  // --- Step 1: API Key ---
+  let hasKey = false;
 
-  // Check env var
   if (process.env.LANGSMITH_API_KEY) {
     console.log(`  ${GREEN}✓${RESET} LANGSMITH_API_KEY found in environment`);
-    return;
+    hasKey = true;
+  } else if (fs.existsSync(langsmithCredsFile)) {
+    try {
+      const content = fs.readFileSync(langsmithCredsFile, "utf8");
+      if (content.includes("LANGSMITH_API_KEY=lsv2_")) {
+        console.log(`  ${GREEN}✓${RESET} API key found in credentials file`);
+        hasKey = true;
+      }
+    } catch {}
   }
 
-  // Check credentials file
-  if (fs.existsSync(langsmithCredsFile)) {
-    console.log(`  ${GREEN}✓${RESET} LangSmith credentials found at ${DIM}${langsmithCredsFile}${RESET}`);
-    return;
+  if (!hasKey) {
+    console.log(`  ${BOLD}LangSmith API Key${RESET} — get yours at ${DIM}https://smith.langchain.com/settings${RESET}`);
+    console.log(`  ${DIM}LangSmith is required. The evolver won't work without it.${RESET}\n`);
+
+    // Keep asking until they provide a key or explicitly skip
+    let attempts = 0;
+    while (!hasKey && attempts < 3) {
+      const apiKey = await ask(rl, `  ${YELLOW}Paste your LangSmith API key (lsv2_pt_...):${RESET} `);
+      const key = apiKey.trim();
+
+      if (key && key.startsWith("lsv2_")) {
+        try {
+          fs.mkdirSync(langsmithCredsDir, { recursive: true });
+          fs.writeFileSync(langsmithCredsFile, `LANGSMITH_API_KEY=${key}\n`);
+          console.log(`  ${GREEN}✓${RESET} API key saved`);
+          hasKey = true;
+        } catch {
+          console.log(`  ${RED}Failed to save.${RESET} Add to your shell: export LANGSMITH_API_KEY=${key}`);
+          hasKey = true; // they have the key, just couldn't save
+        }
+      } else if (key) {
+        console.log(`  ${YELLOW}Invalid — LangSmith keys start with lsv2_${RESET}`);
+        attempts++;
+      } else {
+        // Empty input — skip
+        console.log(`\n  ${RED}WARNING:${RESET} No API key configured.`);
+        console.log(`  ${BOLD}/evolver:setup will not work${RESET} until you set LANGSMITH_API_KEY.`);
+        console.log(`  Run: ${DIM}export LANGSMITH_API_KEY=lsv2_pt_your_key${RESET}\n`);
+        break;
+      }
+    }
   }
 
-  // Ask for API key
-  console.log(`  ${BOLD}LangSmith API Key${RESET} — get yours at ${DIM}https://smith.langchain.com/settings${RESET}`);
-  console.log(`  ${DIM}LangSmith is required for v3 (datasets, experiments, evaluators).${RESET}\n`);
-  const apiKey = await ask(rl, `  ${YELLOW}Paste your LangSmith API key:${RESET} `);
-  const key = apiKey.trim();
+  // --- Step 2: langsmith-cli ---
+  if (hasLangsmithCli) {
+    console.log(`  ${GREEN}✓${RESET} langsmith-cli installed`);
+  } else {
+    console.log(`\n  ${BOLD}langsmith-cli${RESET} — optional but useful for debugging traces`);
+    console.log(`  ${DIM}Quick project listing, trace inspection, run stats from terminal.${RESET}`);
+    const lsCliAnswer = await ask(rl, `\n  ${YELLOW}Install langsmith-cli? [Y/n]:${RESET} `);
+    if (lsCliAnswer.trim().toLowerCase() !== "n") {
+      console.log(`\n  Installing langsmith-cli...`);
+      try {
+        execSync("uv tool install langsmith-cli 2>/dev/null || pip install langsmith-cli 2>/dev/null || pip3 install langsmith-cli", { stdio: "pipe", timeout: 60000 });
+        console.log(`  ${GREEN}✓${RESET} langsmith-cli installed`);
 
-  if (key && key.startsWith("lsv2_")) {
-    try {
-      fs.mkdirSync(langsmithCredsDir, { recursive: true });
-      fs.writeFileSync(langsmithCredsFile, `LANGSMITH_API_KEY=${key}\n`);
-      console.log(`  ${GREEN}✓${RESET} API key saved to ${DIM}${langsmithCredsFile}${RESET}`);
-    } catch {
-      console.log(`  ${RED}Failed to save.${RESET} Add to your shell: export LANGSMITH_API_KEY=${key}`);
+        // If we have a key, auto-authenticate
+        if (hasKey && fs.existsSync(langsmithCredsFile)) {
+          console.log(`  ${GREEN}✓${RESET} langsmith-cli auto-authenticated (credentials file exists)`);
+        }
+      } catch {
+        console.log(`  ${YELLOW}!${RESET} Could not install. Try manually: ${DIM}uv tool install langsmith-cli${RESET}`);
+      }
     }
-  } else if (key) {
-    console.log(`  ${YELLOW}Doesn't look like a LangSmith key (should start with lsv2_).${RESET}`);
-    console.log(`  Add to your shell: ${BOLD}export LANGSMITH_API_KEY=your_key${RESET}`);
-  } else {
-    console.log(`  ${YELLOW}Skipped.${RESET} You must set LANGSMITH_API_KEY before using /evolver:setup`);
   }
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "harness-evolver",
-  "version": "3.0.2",
+  "version": "3.0.4",
   "description": "LangSmith-native autonomous agent optimization for Claude Code",
   "author": "Raphael Valdetaro",
   "license": "MIT",

package/skills/setup/SKILL.md

@@ -11,8 +11,31 @@ Set up the Harness Evolver v3 in a project. Explores the codebase, configures La
 
 ## Prerequisites
 
-- `LANGSMITH_API_KEY` must be set. If not: "Set your LangSmith API key: `export LANGSMITH_API_KEY=lsv2_pt_...`"
-- Python 3.10+ with `langsmith` and `openevals` packages. If missing:
+Check for LangSmith API key — it can be in the environment, the credentials file, or .env:
+
+```bash
+python3 -c "
+import os, platform
+key = os.environ.get('LANGSMITH_API_KEY', '')
+if not key:
+    creds = os.path.expanduser('~/Library/Application Support/langsmith-cli/credentials') if platform.system() == 'Darwin' else os.path.expanduser('~/.config/langsmith-cli/credentials')
+    if os.path.exists(creds):
+        for line in open(creds):
+            if line.strip().startswith('LANGSMITH_API_KEY='):
+                key = line.strip().split('=',1)[1].strip()
+    if not key and os.path.exists('.env'):
+        for line in open('.env'):
+            if line.strip().startswith('LANGSMITH_API_KEY=') and not line.strip().startswith('#'):
+                key = line.strip().split('=',1)[1].strip().strip('\"').strip(\"'\")
+print('OK' if key else 'MISSING')
+"
+```
+
+If `MISSING`: "Set your LangSmith API key: `export LANGSMITH_API_KEY=lsv2_pt_...` or run `npx harness-evolver@latest` to configure."
+
+The tools auto-load the key from the credentials file, but the env var takes precedence.
+
+Python 3.10+ with `langsmith` and `openevals` packages must be installed:
 
 ```bash
 pip install langsmith openevals 2>/dev/null || uv pip install langsmith openevals

package/tools/read_results.py

@@ -21,9 +21,33 @@ Requires: pip install langsmith
 import argparse
 import json
 import os
+import platform
 import sys
 
 
+def ensure_langsmith_api_key():
+    """Load LANGSMITH_API_KEY from credentials file if not in env."""
+    if os.environ.get("LANGSMITH_API_KEY"):
+        return True
+    if platform.system() == "Darwin":
+        creds_path = os.path.expanduser("~/Library/Application Support/langsmith-cli/credentials")
+    else:
+        creds_path = os.path.expanduser("~/.config/langsmith-cli/credentials")
+    if os.path.exists(creds_path):
+        try:
+            with open(creds_path) as f:
+                for line in f:
+                    line = line.strip()
+                    if line.startswith("LANGSMITH_API_KEY="):
+                        key = line.split("=", 1)[1].strip()
+                        if key:
+                            os.environ["LANGSMITH_API_KEY"] = key
+                            return True
+        except OSError:
+            pass
+    return False
+
+
 def read_experiment(client, experiment_name):
     """Read results from a single LangSmith experiment."""
     try:
@@ -184,6 +208,7 @@ def main():
     parser.add_argument("--output", default=None, help="Output JSON path")
     parser.add_argument("--format", default="json", choices=["json", "markdown"], help="Output format")
     args = parser.parse_args()
+    ensure_langsmith_api_key()
 
     from langsmith import Client
     client = Client()

package/tools/run_eval.py

@@ -17,11 +17,47 @@ Requires: pip install langsmith openevals
 import argparse
 import json
 import os
+import platform
 import subprocess
 import sys
 import tempfile
 
 
+def ensure_langsmith_api_key():
+    """Load LANGSMITH_API_KEY from credentials file if not in env."""
+    if os.environ.get("LANGSMITH_API_KEY"):
+        return True
+    if platform.system() == "Darwin":
+        creds_path = os.path.expanduser("~/Library/Application Support/langsmith-cli/credentials")
+    else:
+        creds_path = os.path.expanduser("~/.config/langsmith-cli/credentials")
+    if os.path.exists(creds_path):
+        try:
+            with open(creds_path) as f:
+                for line in f:
+                    line = line.strip()
+                    if line.startswith("LANGSMITH_API_KEY="):
+                        key = line.split("=", 1)[1].strip()
+                        if key:
+                            os.environ["LANGSMITH_API_KEY"] = key
+                            return True
+        except OSError:
+            pass
+    if os.path.exists(".env"):
+        try:
+            with open(".env") as f:
+                for line in f:
+                    line = line.strip()
+                    if line.startswith("LANGSMITH_API_KEY=") and not line.startswith("#"):
+                        key = line.split("=", 1)[1].strip().strip("'\"")
+                        if key:
+                            os.environ["LANGSMITH_API_KEY"] = key
+                            return True
+        except OSError:
+            pass
+    return False
+
+
 def make_target(entry_point, cwd):
     """Create a target function that runs the agent from a specific directory."""
     def target(inputs):
@@ -132,6 +168,7 @@ def main():
         config = json.load(f)
 
     os.environ["EVAL_TASK_TIMEOUT"] = str(args.timeout)
+    ensure_langsmith_api_key()
 
     from langsmith import Client
     client = Client()

package/tools/setup.py

@@ -25,12 +25,58 @@ Requires: pip install langsmith openevals
 import argparse
 import json
 import os
+import platform
 import subprocess
 import sys
 import tempfile
 from datetime import datetime, timezone
 
 
+def ensure_langsmith_api_key():
+    """Load LANGSMITH_API_KEY from credentials file if not in env.
+
+    The installer saves the key to the langsmith-cli credentials file,
+    but the SDK only reads the env var. This bridges the gap.
+    """
+    if os.environ.get("LANGSMITH_API_KEY"):
+        return True
+
+    # Platform-specific credentials path (matches langsmith-cli)
+    if platform.system() == "Darwin":
+        creds_path = os.path.expanduser("~/Library/Application Support/langsmith-cli/credentials")
+    else:
+        creds_path = os.path.expanduser("~/.config/langsmith-cli/credentials")
+
+    if os.path.exists(creds_path):
+        try:
+            with open(creds_path) as f:
+                for line in f:
+                    line = line.strip()
+                    if line.startswith("LANGSMITH_API_KEY="):
+                        key = line.split("=", 1)[1].strip()
+                        if key:
+                            os.environ["LANGSMITH_API_KEY"] = key
+                            return True
+        except OSError:
+            pass
+
+    # Also check .env in current directory
+    if os.path.exists(".env"):
+        try:
+            with open(".env") as f:
+                for line in f:
+                    line = line.strip()
+                    if line.startswith("LANGSMITH_API_KEY=") and not line.startswith("#"):
+                        key = line.split("=", 1)[1].strip().strip("'\"")
+                        if key:
+                            os.environ["LANGSMITH_API_KEY"] = key
+                            return True
+        except OSError:
+            pass
+
+    return False
+
+
 def check_dependencies():
     """Verify langsmith and openevals are installed."""
     missing = []
@@ -294,6 +340,12 @@ def main():
         print(f"Install with: pip install {' '.join(missing)}", file=sys.stderr)
         sys.exit(1)
 
+    # Load API key from credentials file if not in env
+    if not ensure_langsmith_api_key():
+        print("LANGSMITH_API_KEY not found in environment, credentials file, or .env", file=sys.stderr)
+        print("Set it with: export LANGSMITH_API_KEY=lsv2_pt_...", file=sys.stderr)
+        sys.exit(1)
+
     from langsmith import Client
     client = Client()