harness-evolver 3.0.4 → 3.0.6

package/agents/evolver-proposer.md

@@ -64,7 +64,34 @@ Based on your strategy and diagnosis, modify the code:
 - **Error handling**: retry logic, fallback strategies, timeout handling
 - **Model selection**: which model for which task
 
-Use Context7 MCP tools (`resolve-library-id`, `get-library-docs`) to check current API documentation before writing code that uses library APIs.
+### Phase 3.5: Consult Documentation (MANDATORY)
+
+**Before writing ANY code**, you MUST consult Context7 for every library you'll be modifying or using. This is NOT optional.
+
+**Step 1 — Identify libraries from the code you read:**
+Read the imports in the files you're about to modify. For each framework/library (LangGraph, OpenAI, Anthropic, CrewAI, etc.):
+
+**Step 2 — Resolve library ID:**
+```
+resolve-library-id(libraryName: "langgraph", query: "what you're trying to do")
+```
+This returns up to 10 matches. Pick the one with the highest relevance.
+
+**Step 3 — Query docs for your specific task:**
+```
+get-library-docs(libraryId: "/langchain-ai/langgraph", query: "conditional edges StateGraph", topic: "routing")
+```
+Ask about the SPECIFIC API you're going to use or change.
+
+**Examples of what to query:**
+- About to modify a StateGraph? → `query: "StateGraph add_conditional_edges"` 
+- Changing prompt template? → `query: "ChatPromptTemplate from_messages"` for langchain
+- Adding a tool? → `query: "StructuredTool create tool definition"` for langchain
+- Changing model? → `query: "ChatOpenAI model parameters temperature"` for openai
+
+**Why this matters:** Your training data may be outdated. Libraries change APIs between versions. A quick Context7 lookup takes seconds and prevents proposing code that uses deprecated or incorrect patterns. The documentation is the source of truth, not your model knowledge.
+
+**If Context7 MCP is not available:** Note in proposal.md "API patterns not verified against current docs — verify before deploying."
 
 ### Phase 4: Commit and Document
 
@@ -99,15 +126,6 @@ If `production_seed.json` exists:
 
 Prioritize changes that fix real production failures over synthetic test failures.
 
-## Context7 — Documentation Lookup
-
-Use Context7 MCP tools proactively when:
-- Writing code that uses a library API
-- Unsure about method signatures or patterns
-- Checking if a better approach exists in the latest version
-
-If Context7 is not available, proceed with model knowledge but note in proposal.md.
-
 ## Rules
 
 1. **Read before writing** — understand the code before changing it

package/bin/install.js

@@ -190,28 +190,58 @@ function installTools() {
 }
 
 function installPythonDeps() {
-  console.log(`\n  ${YELLOW}Installing Python dependencies...${RESET}`);
-
-  // Try multiple pip variants
-  const commands = [
-    "pip install langsmith openevals",
-    "uv pip install langsmith openevals",
-    "pip3 install langsmith openevals",
-    "python3 -m pip install langsmith openevals",
+  const venvDir = path.join(HOME, ".evolver", "venv");
+  const venvPython = path.join(venvDir, "bin", "python");
+  const venvPip = path.join(venvDir, "bin", "pip");
+
+  console.log(`\n  ${YELLOW}Setting up Python environment...${RESET}`);
+
+  // Create venv if it doesn't exist
+  if (!fs.existsSync(venvPython)) {
+    console.log(`  Creating isolated venv at ~/.evolver/venv/`);
+    const venvCommands = [
+      `uv venv "${venvDir}"`,
+      `python3 -m venv "${venvDir}"`,
+    ];
+    let created = false;
+    for (const cmd of venvCommands) {
+      try {
+        execSync(cmd, { stdio: "pipe", timeout: 30000 });
+        created = true;
+        break;
+      } catch {
+        continue;
+      }
+    }
+    if (!created) {
+      console.log(`  ${RED}Failed to create venv.${RESET}`);
+      console.log(`    Run manually: ${BOLD}python3 -m venv ~/.evolver/venv${RESET}`);
+      return false;
+    }
+    console.log(`  ${GREEN}✓${RESET} venv created`);
+  } else {
+    console.log(`  ${GREEN}✓${RESET} venv exists at ~/.evolver/venv/`);
+  }
+
+  // Install/upgrade deps in the venv
+  const installCommands = [
+    `uv pip install --python "${venvPython}" langsmith openevals`,
+    `"${venvPip}" install --upgrade langsmith openevals`,
+    `"${venvPython}" -m pip install --upgrade langsmith openevals`,
   ];
 
-  for (const cmd of commands) {
+  for (const cmd of installCommands) {
     try {
       execSync(cmd, { stdio: "pipe", timeout: 120000 });
-      console.log(`  ${GREEN}✓${RESET} langsmith + openevals installed`);
+      console.log(`  ${GREEN}✓${RESET} langsmith + openevals installed in venv`);
       return true;
     } catch {
       continue;
     }
   }
 
-  console.log(`  ${YELLOW}!${RESET} Could not auto-install Python packages.`);
-  console.log(`    Run manually: ${BOLD}pip install langsmith openevals${RESET}`);
+  console.log(`  ${YELLOW}!${RESET} Could not install packages in venv.`);
+  console.log(`    Run manually: ${BOLD}~/.evolver/venv/bin/pip install langsmith openevals${RESET}`);
   return false;
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "harness-evolver",
-  "version": "3.0.4",
+  "version": "3.0.6",
   "description": "LangSmith-native autonomous agent optimization for Claude Code",
   "author": "Raphael Valdetaro",
   "license": "MIT",

package/skills/evolve/SKILL.md

@@ -13,12 +13,15 @@ Run the autonomous propose-evaluate-iterate loop using LangSmith as the evaluati
 
 `.evolver.json` must exist. If not, tell user to run `evolver:setup`.
 
-## Resolve Tool Path
+## Resolve Tool Path and Python
 
 ```bash
 TOOLS=$([ -d ".evolver/tools" ] && echo ".evolver/tools" || echo "$HOME/.evolver/tools")
+EVOLVER_PY=$([ -f "$HOME/.evolver/venv/bin/python" ] && echo "$HOME/.evolver/venv/bin/python" || echo "python3")
 ```
 
+Use `$EVOLVER_PY` instead of `python3` for ALL tool invocations.
+
 ## Parse Arguments
 
 - `--iterations N` (default: from interactive question or 5)
@@ -76,7 +79,7 @@ Run trace insights from the best experiment:
 
 ```bash
 BEST=$(python3 -c "import json; print(json.load(open('.evolver.json'))['best_experiment'])")
-python3 $TOOLS/trace_insights.py \
+$EVOLVER_PY $TOOLS/trace_insights.py \
     --from-experiment "$BEST" \
     --output trace_insights.json 2>/dev/null
 ```
@@ -86,7 +89,7 @@ If a production project is configured, also gather production insights:
 ```bash
 PROD=$(python3 -c "import json; c=json.load(open('.evolver.json')); print(c.get('production_project',''))")
 if [ -n "$PROD" ] && [ ! -f "production_seed.json" ]; then
-    python3 $TOOLS/seed_from_traces.py \
+    $EVOLVER_PY $TOOLS/seed_from_traces.py \
         --project "$PROD" --use-sdk \
         --output-md production_seed.md \
         --output-json production_seed.json \
@@ -99,7 +102,7 @@ fi
 Read the best experiment results and cluster failures:
 
 ```bash
-python3 $TOOLS/read_results.py \
+$EVOLVER_PY $TOOLS/read_results.py \
     --experiment "$BEST" \
     --config .evolver.json \
     --output best_results.json 2>/dev/null
@@ -174,7 +177,7 @@ Wait for all 5 to complete.
 For each worktree that has changes (proposer committed something):
 
 ```bash
-python3 $TOOLS/run_eval.py \
+$EVOLVER_PY $TOOLS/run_eval.py \
     --config .evolver.json \
     --worktree-path {worktree_path} \
     --experiment-prefix v{NNN}{suffix} \
@@ -186,7 +189,7 @@ Each candidate becomes a separate LangSmith experiment.
 ### 4. Compare All Candidates
 
 ```bash
-python3 $TOOLS/read_results.py \
+$EVOLVER_PY $TOOLS/read_results.py \
     --experiments "v{NNN}a,v{NNN}b,v{NNN}c,v{NNN}d,v{NNN}e" \
     --config .evolver.json \
     --output comparison.json

package/skills/setup/SKILL.md

@@ -35,35 +35,34 @@ If `MISSING`: "Set your LangSmith API key: `export LANGSMITH_API_KEY=lsv2_pt_...
 
 The tools auto-load the key from the credentials file, but the env var takes precedence.
 
-Python 3.10+ with `langsmith` and `openevals` packages must be installed:
-
-```bash
-pip install langsmith openevals 2>/dev/null || uv pip install langsmith openevals
-```
-
-## Resolve Tool Path
+## Resolve Tool Path and Python
 
 ```bash
 TOOLS=$([ -d ".evolver/tools" ] && echo ".evolver/tools" || echo "$HOME/.evolver/tools")
+EVOLVER_PY=$([ -f "$HOME/.evolver/venv/bin/python" ] && echo "$HOME/.evolver/venv/bin/python" || echo "python3")
 ```
 
+Use `$EVOLVER_PY` instead of `python3` for ALL tool invocations. This ensures the venv with langsmith+openevals is used.
+
 ## Phase 1: Explore Project (automatic)
 
 ```bash
-find . -maxdepth 3 -type f -name "*.py" | head -30
-python3 $TOOLS/detect_stack.py .
+find . -maxdepth 3 -type f -name "*.py" -not -path "*/.venv/*" -not -path "*/node_modules/*" -not -path "*/__pycache__/*" | head -30
 ```
 
+**Monorepo detection**: if the project root has multiple subdirectories with their own `main.py` or `pyproject.toml`, it's a monorepo. Use AskUserQuestion to ask WHICH app to optimize before proceeding — do NOT scan everything.
+
 Look for:
 - Entry points: files with `if __name__`, or named `main.py`, `app.py`, `agent.py`, `graph.py`, `pipeline.py`
-- Framework: LangGraph, CrewAI, OpenAI SDK, Anthropic SDK, etc.
 - Existing LangSmith config: `LANGCHAIN_PROJECT` / `LANGSMITH_PROJECT` in env or `.env`
 - Existing test data: JSON files with inputs, CSV files, etc.
 - Dependencies: `requirements.txt`, `pyproject.toml`
 
-Identify the **run command** — how to execute the agent. Examples:
+To identify the **framework**, read the entry point file and its immediate imports. The proposer agents will use Context7 MCP for detailed documentation lookup — you don't need to detect every library, just identify the main framework (LangGraph, CrewAI, OpenAI Agents SDK, etc.) from the imports you see.
+
+Identify the **run command** — how to execute the agent:
 - `python main.py` (if it accepts `--input` flag)
-- `python -c "from agent import run; import json,sys; print(json.dumps(run(json.load(open(sys.argv[1])))))"`
+- The command in the project's README, Makefile, or scripts/
 
 ## Phase 2: Confirm Detection (interactive)
 
@@ -145,7 +144,7 @@ If "I have test data": ask for file path.
 Build the setup.py command based on all gathered information:
 
 ```bash
-python3 $TOOLS/setup.py \
+$EVOLVER_PY $TOOLS/setup.py \
     --project-name "{project_name}" \
     --entry-point "{run_command}" \
     --framework "{framework}" \

package/tools/detect_stack.py

@@ -1,173 +0,0 @@
-#!/usr/bin/env python3
-"""Detect the technology stack of a harness by analyzing Python imports via AST.
-
-Usage:
-    detect_stack.py <file_or_directory> [-o output.json]
-
-Maps imports to known libraries and their Context7 IDs for documentation lookup.
-Stdlib-only. No external dependencies.
-"""
-
-import ast
-import json
-import os
-import sys
-
-KNOWN_LIBRARIES = {
-    "langchain": {
-        "context7_id": "/langchain-ai/langchain",
-        "display": "LangChain",
-        "modules": ["langchain", "langchain_core", "langchain_openai",
-                     "langchain_anthropic", "langchain_community"],
-    },
-    "langgraph": {
-        "context7_id": "/langchain-ai/langgraph",
-        "display": "LangGraph",
-        "modules": ["langgraph"],
-    },
-    "llamaindex": {
-        "context7_id": "/run-llama/llama_index",
-        "display": "LlamaIndex",
-        "modules": ["llama_index"],
-    },
-    "openai": {
-        "context7_id": "/openai/openai-python",
-        "display": "OpenAI Python SDK",
-        "modules": ["openai"],
-    },
-    "anthropic": {
-        "context7_id": "/anthropics/anthropic-sdk-python",
-        "display": "Anthropic Python SDK",
-        "modules": ["anthropic"],
-    },
-    "dspy": {
-        "context7_id": "/stanfordnlp/dspy",
-        "display": "DSPy",
-        "modules": ["dspy"],
-    },
-    "crewai": {
-        "context7_id": "/crewAIInc/crewAI",
-        "display": "CrewAI",
-        "modules": ["crewai"],
-    },
-    "autogen": {
-        "context7_id": "/microsoft/autogen",
-        "display": "AutoGen",
-        "modules": ["autogen"],
-    },
-    "chromadb": {
-        "context7_id": "/chroma-core/chroma",
-        "display": "ChromaDB",
-        "modules": ["chromadb"],
-    },
-    "pinecone": {
-        "context7_id": "/pinecone-io/pinecone-python-client",
-        "display": "Pinecone",
-        "modules": ["pinecone"],
-    },
-    "qdrant": {
-        "context7_id": "/qdrant/qdrant",
-        "display": "Qdrant",
-        "modules": ["qdrant_client"],
-    },
-    "weaviate": {
-        "context7_id": "/weaviate/weaviate",
-        "display": "Weaviate",
-        "modules": ["weaviate"],
-    },
-    "fastapi": {
-        "context7_id": "/fastapi/fastapi",
-        "display": "FastAPI",
-        "modules": ["fastapi"],
-    },
-    "flask": {
-        "context7_id": "/pallets/flask",
-        "display": "Flask",
-        "modules": ["flask"],
-    },
-    "pydantic": {
-        "context7_id": "/pydantic/pydantic",
-        "display": "Pydantic",
-        "modules": ["pydantic"],
-    },
-    "pandas": {
-        "context7_id": "/pandas-dev/pandas",
-        "display": "Pandas",
-        "modules": ["pandas"],
-    },
-    "numpy": {
-        "context7_id": "/numpy/numpy",
-        "display": "NumPy",
-        "modules": ["numpy"],
-    },
-}
-
-
-def detect_from_file(filepath):
-    """Analyze imports of a Python file and return detected stack."""
-    with open(filepath) as f:
-        try:
-            tree = ast.parse(f.read())
-        except SyntaxError:
-            return {}
-
-    imports = set()
-    for node in ast.walk(tree):
-        if isinstance(node, ast.Import):
-            for alias in node.names:
-                imports.add(alias.name.split(".")[0])
-        elif isinstance(node, ast.ImportFrom):
-            if node.module:
-                imports.add(node.module.split(".")[0])
-
-    detected = {}
-    for lib_key, lib_info in KNOWN_LIBRARIES.items():
-        found = imports & set(lib_info["modules"])
-        if found:
-            detected[lib_key] = {
-                "context7_id": lib_info["context7_id"],
-                "display": lib_info["display"],
-                "modules_found": sorted(found),
-            }
-
-    return detected
-
-
-def detect_from_directory(directory):
-    """Analyze all .py files in a directory and consolidate the stack."""
-    all_detected = {}
-    for root, dirs, files in os.walk(directory):
-        for f in files:
-            if f.endswith(".py"):
-                filepath = os.path.join(root, f)
-                file_detected = detect_from_file(filepath)
-                for lib_key, lib_info in file_detected.items():
-                    if lib_key not in all_detected:
-                        all_detected[lib_key] = lib_info
-                    else:
-                        existing = set(all_detected[lib_key]["modules_found"])
-                        existing.update(lib_info["modules_found"])
-                        all_detected[lib_key]["modules_found"] = sorted(existing)
-    return all_detected
-
-
-if __name__ == "__main__":
-    import argparse
-
-    parser = argparse.ArgumentParser(description="Detect stack from Python files")
-    parser.add_argument("path", help="File or directory to analyze")
-    parser.add_argument("--output", "-o", help="Output JSON path")
-    args = parser.parse_args()
-
-    if os.path.isfile(args.path):
-        result = detect_from_file(args.path)
-    else:
-        result = detect_from_directory(args.path)
-
-    output = json.dumps(result, indent=2)
-
-    if args.output:
-        with open(args.output, "w") as f:
-            f.write(output)
-    else:
-        print(output)