@evo-hq/pi-evo 0.4.2-alpha.2

package/skills/discover/references/inline_instrumentation.js

@@ -0,0 +1,97 @@
+/**
+ * Inline instrumentation for Node benchmarks. Paste into the benchmark and
+ * call logTask() per task + writeResult() once at the end.
+ *
+ * Contract:
+ * - Reads EVO_TRACES_DIR, EVO_EXPERIMENT_ID, EVO_RESULT_PATH from process.env.
+ * - Writes traces/task_<id>.json per task.
+ * - Writes the final result JSON to EVO_RESULT_PATH, or stdout if unset.
+ */
+
+import {
+  writeFileSync,
+  mkdirSync,
+  openSync,
+  closeSync,
+  renameSync,
+} from "node:fs";
+import { dirname, join } from "node:path";
+
+const TRACES_DIR = process.env.EVO_TRACES_DIR || null;
+const EXPERIMENT_ID = process.env.EVO_EXPERIMENT_ID || "unknown";
+const RESULT_PATH = process.env.EVO_RESULT_PATH || null;
+const SCORES = {};
+const TASK_META = {};
+const STARTED_AT = new Date().toISOString().replace(/\.\d{3}Z$/, "+00:00");
+
+if (TRACES_DIR) mkdirSync(TRACES_DIR, { recursive: true });
+
+/**
+ * Record the result for one task. `direction` is "max" (higher is better,
+ * default) or "min" (lower is better, e.g. latency). Set it only when this
+ * task's direction differs from the benchmark's top-level --metric.
+ * Propagates to tasks_meta in the final result JSON.
+ */
+export function logTask(taskId, score, { summary, failureReason, log, direction, ...extra } = {}) {
+  taskId = String(taskId);
+  if (direction !== undefined && direction !== "max" && direction !== "min") {
+    throw new Error(`direction must be 'max' or 'min', got ${JSON.stringify(direction)}`);
+  }
+  SCORES[taskId] = score;
+  if (direction !== undefined) TASK_META[taskId] = { direction };
+  if (!TRACES_DIR) return;
+  const trace = {
+    experiment_id: EXPERIMENT_ID,
+    task_id: taskId,
+    status: score >= 0.5 ? "passed" : "failed",
+    score,
+    ended_at: new Date().toISOString().replace(/\.\d{3}Z$/, "+00:00"),
+  };
+  if (direction !== undefined) trace.direction = direction;
+  if (summary !== undefined) trace.summary = summary;
+  if (failureReason !== undefined) trace.failure_reason = failureReason;
+  if (log !== undefined) trace.log = log;
+  Object.assign(trace, extra);
+  writeFileSync(join(TRACES_DIR, `task_${taskId}.json`), JSON.stringify(trace, null, 2), "utf-8");
+}
+
+export function writeResult(score) {
+  const ids = Object.keys(SCORES);
+  if (score === undefined) {
+    score = ids.length === 0 ? 0.0 : ids.reduce((a, id) => a + SCORES[id], 0) / ids.length;
+  }
+  score = Math.round(score * 10000) / 10000;
+  const result = {
+    score,
+    tasks: { ...SCORES },
+    started_at: STARTED_AT,
+    ended_at: new Date().toISOString().replace(/\.\d{3}Z$/, "+00:00"),
+  };
+  if (Object.keys(TASK_META).length > 0) {
+    result.tasks_meta = Object.fromEntries(
+      Object.entries(TASK_META).map(([k, v]) => [k, { ...v }])
+    );
+  }
+  const payload = JSON.stringify(result, null, 2);
+  if (RESULT_PATH) {
+    mkdirSync(dirname(RESULT_PATH), { recursive: true });
+    // Claim + tmp+rename: duplicate writers fail-fast; crash mid-publish
+    // leaves an empty file (caught by load_result) not a partial write.
+    try {
+      closeSync(openSync(RESULT_PATH, "wx"));
+    } catch (e) {
+      if (e.code === "EEXIST") {
+        throw new Error(
+          `${RESULT_PATH} already exists; only one writeResult() per attempt`
+        );
+      }
+      throw e;
+    }
+    const tmp = RESULT_PATH + ".tmp";
+    writeFileSync(tmp, payload, "utf-8");
+    renameSync(tmp, RESULT_PATH);
+  } else {
+    process.stdout.write(payload + "\n");
+  }
+  return score;
+}

package/skills/discover/references/inline_instrumentation.py

@@ -0,0 +1,109 @@
+"""Inline instrumentation for Python benchmarks. Paste into the benchmark
+and call `log_task()` per task + `write_result()` once at the end.
+
+Contract:
+- Reads EVO_TRACES_DIR, EVO_EXPERIMENT_ID, EVO_RESULT_PATH from env.
+- Writes traces/task_<id>.json per task.
+- Writes the final result JSON to EVO_RESULT_PATH, or stdout if unset.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+_TRACES_DIR = Path(os.environ["EVO_TRACES_DIR"]) if os.environ.get("EVO_TRACES_DIR") else None
+_EXPERIMENT_ID = os.environ.get("EVO_EXPERIMENT_ID", "unknown")
+_RESULT_PATH = os.environ.get("EVO_RESULT_PATH")
+_SCORES: dict[str, float] = {}
+_TASK_META: dict[str, dict[str, Any]] = {}
+_STARTED_AT = datetime.now(timezone.utc).isoformat(timespec="seconds")
+
+if _TRACES_DIR:
+    _TRACES_DIR.mkdir(parents=True, exist_ok=True)
+
+
+def log_task(
+    task_id: str,
+    score: float,
+    *,
+    summary: str | None = None,
+    failure_reason: str | None = None,
+    log: list[Any] | None = None,
+    direction: str | None = None,
+    **extra: Any,
+) -> None:
+    """Record the result for one task. Writes task_<id>.json immediately.
+
+    *direction* is "max" (higher is better, default) or "min" (lower is
+    better, e.g. latency). Only set it when this task's direction differs
+    from the benchmark's top-level `--metric`. Propagates to `tasks_meta`
+    in the final result JSON for downstream selection strategies.
+    """
+    task_id = str(task_id)
+    if direction is not None and direction not in ("max", "min"):
+        raise ValueError(f"direction must be 'max' or 'min', got {direction!r}")
+    _SCORES[task_id] = score
+    if direction is not None:
+        _TASK_META[task_id] = {"direction": direction}
+    if _TRACES_DIR is None:
+        return
+    trace: dict[str, Any] = {
+        "experiment_id": _EXPERIMENT_ID,
+        "task_id": task_id,
+        "status": "passed" if score >= 0.5 else "failed",
+        "score": score,
+        "ended_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
+    }
+    if direction is not None:
+        trace["direction"] = direction
+    if summary is not None:
+        trace["summary"] = summary
+    if failure_reason is not None:
+        trace["failure_reason"] = failure_reason
+    if log is not None:
+        trace["log"] = log
+    trace.update(extra)
+    (_TRACES_DIR / f"task_{task_id}.json").write_text(
+        json.dumps(trace, indent=2), encoding="utf-8"
+    )
+
+
+def write_result(score: float | None = None) -> float:
+    """Write the final score JSON to $EVO_RESULT_PATH (or stdout if unset)
+    and return the score. The return lets callers gate on --min-score
+    without recomputing the aggregate.
+    """
+    if score is None:
+        score = sum(_SCORES.values()) / len(_SCORES) if _SCORES else 0.0
+    score = round(score, 4)
+    result = {
+        "score": score,
+        "tasks": dict(_SCORES),
+        "started_at": _STARTED_AT,
+        "ended_at": datetime.now(timezone.utc).isoformat(timespec="seconds"),
+    }
+    if _TASK_META:
+        result["tasks_meta"] = {k: dict(v) for k, v in _TASK_META.items()}
+    payload = json.dumps(result, indent=2)
+    if _RESULT_PATH:
+        target = Path(_RESULT_PATH)
+        target.parent.mkdir(parents=True, exist_ok=True)
+        # Claim + tmp+rename: duplicate writers fail-fast; crash mid-publish
+        # leaves an empty file (caught by load_result) not a partial write.
+        try:
+            os.close(os.open(target, os.O_CREAT | os.O_EXCL | os.O_WRONLY))
+        except FileExistsError:
+            raise RuntimeError(
+                f"{target} already exists; only one write_result() per attempt"
+            ) from None
+        tmp = target.with_name(target.name + ".tmp")
+        tmp.write_text(payload, encoding="utf-8")
+        os.replace(tmp, target)
+    else:
+        print(payload)
+    return score

package/skills/discover/references/proposing-dimensions.md

@@ -0,0 +1,52 @@
+# Proposing unexplored optimization dimensions
+
+Used only when the benchmark isn't obvious — no existing eval, ambiguous user intent, or the existing eval covers a narrow slice while the interesting optimization sits elsewhere. If the right benchmark *is* obvious, use it and skip this exercise.
+
+When this step does run, the goal is to propose a handful of dimensions for this repo that aren't already measured. Existing benchmarks cover what the authors already worried about; that's where slack is lowest.
+
+## Where to look
+
+1. **Already-instrumented code.** Grep for `time.`, `perf_counter`, `@profile`, `Counter(`, `metrics.`. What's tracked hints at what authors cared about; what isn't is where slack lives.
+2. **Stated goals.** READMEs, module docstrings, and comments often name what the project values ("fast JSON parsing", "robust against malformed input"). If a stated goal isn't measured, that's a proposal.
+3. **Author pain points.** Grep for `TODO`, `FIXME`, `XXX`, `HACK`. Check the issue tracker if accessible.
+4. **Project-type defaults.** The table below, as a starting point.
+
+## Ranking
+
+For each candidate, answer three questions honestly in prose. No scores — a 1-5 slack rating from an LLM is a vibe, not a measurement.
+
+- **Signal.** Does moving this metric actually correlate with "the project is better"? Or is it a proxy that could drift from what the user cares about?
+- **Slack.** Has anyone hill-climbed this before in this repo? Is there plausibly room to improve, or is the current value already near a floor/ceiling?
+- **Cost per run.** How long and how expensive is one benchmark run? The optimization loop runs many — expensive dimensions compound into real time and money.
+
+Rank on a combined judgment of those three. Construction effort (the one-time cost of building the harness) is not a ranking input — flag it qualitatively when presenting, let the user weigh it.
+
+## Project-type defaults
+
+Start with the obvious column, then look hard at the non-obvious column.
+
+| Project type | Obvious (often already done) | Non-obvious (usually unexplored) |
+|---|---|---|
+| LLM / agent | Task pass rate on a benchmark | Token efficiency per correct answer, calibration error, refusal rate on ambiguous tasks, behavior under prompt injection, latency per tool call, recovery from tool errors |
+| Web API / backend | Test pass rate, integration tests | p99 latency on hot endpoints, memory per request, error rate under synthetic load, cold-start time, allocation count per request |
+| ML training | Validation accuracy, loss | Sample efficiency (accuracy per 1k tokens seen), robustness to input perturbations, generalization gap, inference memory, convergence speed |
+| Library / SDK | API tests passing | Import time, allocation count per call, TypeScript strict-mode coverage, docs coverage, cold-import latency, binary size |
+| Compiler / DSL | Correctness on standard suite | Output code size, compile time, optimization quality on standard benchmarks, error message quality (LLM-as-judge), stack trace usefulness |
+| Data pipeline | End-to-end correctness | Throughput (rows/sec), memory peak per batch, late-data handling, schema-drift resilience, idempotency under replay |
+| CLI tool | Unit tests | Cold-start time, memory footprint, output stability across runs, exit-code correctness on edge inputs, help-text discoverability |
+| RAG / retrieval | Recall@K | Embedding cost per indexed doc, query latency p99, answer grounding rate (% of claims traceable to source), robustness to paraphrased queries |
+
+## Presenting to the user
+
+For each ranked dimension include:
+
+- **What it measures** (one sentence)
+- **Why it matters for this project** (tied to what the repo actually does, not generic)
+- **Construction complexity**: *None* (existing eval already produces this score) / *Minor* (wrap or instrument what exists) / *Substantial* (new test cases, scoring logic, or data)
+- **Existing coverage** if any
+
+Recommend the highest-ranked dimension whose construction is *None* or *Minor*. If every top pick is *Substantial*, say so and let the user decide whether the signal is worth the work.
+
+## Non-picked dimensions
+
+Save unused dimensions to `.evo/project.md` under a "Future experiment candidates" section — useful when the first dimension plateaus.

package/skills/discover/references/sdk_node.js

@@ -0,0 +1,28 @@
+// Node SDK usage example. Install: `npm install @evo-hq/evo-agent`.
+//
+// The SDK auto-reads $EVO_TRACES_DIR, $EVO_EXPERIMENT_ID, and
+// $EVO_RESULT_PATH. Traces flush on each report() so the dashboard can
+// stream progress live.
+
+import { Run, Gate } from '@evo-hq/evo-agent';
+
+// ---- Benchmark run ----
+
+const run = new Run();
+for (const task of tasks) {
+  const result = await evaluate(task);
+  run.log(task.id, { output: result.output });
+  run.report(task.id, { score: result.score });
+}
+await run.finish();
+// finish(): writes score JSON to $EVO_RESULT_PATH (or stdout if unset)
+// and one task_<id>.json per task under $EVO_TRACES_DIR.
+
+// ---- Gate (exits 0 all-pass / 1 any-fail) ----
+
+const gate = new Gate();
+for (const task of criticalTasks) {
+  const result = await evaluate(task);
+  gate.check(task.id, { score: result.score });
+}
+await gate.finish();

package/skills/discover/references/sdk_python.py

@@ -0,0 +1,43 @@
+"""Python SDK usage examples.
+
+Install `evo-hq-agent` with this project's package manager/runtime, for example
+`uv add --dev evo-hq-agent` or `python -m pip install evo-hq-agent`.
+
+The SDK auto-reads $EVO_TRACES_DIR, $EVO_EXPERIMENT_ID, and $EVO_RESULT_PATH.
+Traces flush on each report() so the dashboard can stream progress live.
+"""
+
+from evo_agent import Run, Gate
+
+
+# ---- Benchmark run ----
+
+run = Run()
+try:
+    for task in tasks:
+        run.log(task["id"], "starting task")
+        try:
+            result = evaluate(task, agent)
+            run.log(task["id"], {"output": result.output})
+            run.report(
+                task["id"],
+                score=result.score,
+                summary=f"reward={result.score:.2f}",
+                failure_reason=None if result.passed else "task_failed",
+            )
+        except Exception as exc:
+            run.log(task["id"], {"error": repr(exc)})
+            run.report(task["id"], score=0.0, failure_reason="exception")
+finally:
+    run.finish()
+# finish() writes score JSON to $EVO_RESULT_PATH (or stdout if unset) and one
+# task_<id>.json per task under $EVO_TRACES_DIR. Catch expected per-task errors;
+# an uncaught exception before finish() means evo correctly sees a crashed run.
+
+
+# ---- Gate (exits 0 all-pass / 1 any-fail) ----
+
+with Gate() as gate:
+    for task in critical_tasks:
+        result = evaluate(task, agent)
+        gate.check(task["id"], score=result.score)

package/skills/discover/scripts/validate_result.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+"""Validate a benchmark result file.
+
+Usage: python3 validate_result.py <path-to-result.json>
+
+Exits 0 if the file exists, is non-empty, and is a JSON object with a
+numeric 'score'. Exits 1 with a diagnostic on stderr otherwise.
+"""
+
+import json
+import sys
+from pathlib import Path
+
+
+def main() -> int:
+    if len(sys.argv) != 2:
+        print(f"usage: {sys.argv[0]} <result.json>", file=sys.stderr)
+        return 1
+
+    path = Path(sys.argv[1])
+
+    if not path.exists():
+        print(f"FAIL: {path} does not exist", file=sys.stderr)
+        return 1
+
+    if path.stat().st_size == 0:
+        print(f"FAIL: {path} is empty", file=sys.stderr)
+        return 1
+
+    try:
+        obj = json.loads(path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError as exc:
+        print(f"FAIL: {path} is not valid JSON: {exc}", file=sys.stderr)
+        return 1
+
+    if not isinstance(obj, dict):
+        print(f"FAIL: expected JSON object, got {type(obj).__name__}", file=sys.stderr)
+        return 1
+
+    if "score" not in obj:
+        print(f"FAIL: missing 'score' field. Keys: {list(obj.keys())}", file=sys.stderr)
+        return 1
+
+    try:
+        score = float(obj["score"])
+    except (TypeError, ValueError):
+        print(f"FAIL: 'score' is not numeric: {obj['score']!r}", file=sys.stderr)
+        return 1
+
+    print(f"OK: {path}, score = {score}", file=sys.stderr)
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

package/skills/infra-setup/SKILL.md

@@ -0,0 +1,87 @@
+---
+name: infra-setup
+description: Non-user-invocable provider/setup reference for evo backend switching, prerequisite checks, and auth/install guidance.
+disable-model-invocation: true
+---
+
+# Infra Setup
+
+Use this when the user wants to change where experiments run: local worktrees, pool slots, or a remote provider such as Modal, E2B, Daytona, AWS, Azure, SSH, manual, or a custom dotted-path provider.
+
+## Goals
+
+- Be explicit about the target backend/provider.
+- Check prerequisites before mutating evo config.
+- Never install provider SDKs silently.
+- Give one actionable auth command per provider.
+- Keep provider credentials separate from benchmark runtime env.
+
+## Flow
+
+1. Identify the target:
+   - `worktree` or `pool` means local backends.
+   - `modal`, `e2b`, `ssh:...`, or another remote spec means `backend=remote`.
+2. If the target is remote, parse the provider choice the same way evo CLI does:
+- `modal`
+- `e2b`
+- `daytona`
+- `aws`
+- `azure`
+- `manual`
+- `ssh:user@host[:port]`
+   - another built-in provider name
+   - dotted import path for a custom provider
+3. Check whether `evo` is on PATH and whether it is the expected `evo-hq-cli` package (`evo --version`). If the provider SDK is missing, evo's provider loader prints the provider-specific extra or SDK package to install; use that message rather than guessing.
+4. For SDK-backed providers, verify the SDK import only when you can run the check in the same environment that owns the `evo` executable. If missing, ask the user before installing it.
+   - If `evo` was installed with `uv tool` or `pip`/`venv`, prefer the matching extra on `evo-hq-cli`:
+     - `uv-tool`: `uv tool install --reinstall 'evo-hq-cli[<provider-extra>]'`
+     - `venv` / `pip`: `python -m pip install 'evo-hq-cli[<provider-extra>]'`
+   - If `evo` was installed with `pipx`, inject the provider SDK into the same `evo-hq-cli` environment:
+     - `pipx`: `pipx inject evo-hq-cli <provider-sdk>`
+5. Check auth and show exactly one provider-specific auth command or setup step. Use `references/provider-matrix.md`.
+6. Once prerequisites are satisfied, run the explicit config command:
+
+```bash
+evo config backend remote --provider <provider> --provider-config ...
+```
+
+Or for local backends:
+
+```bash
+evo config backend worktree
+evo config backend pool --workspaces /abs/slot-a,/abs/slot-b
+```
+
+7. Be explicit that incomplete provider setup usually surfaces on
+   `evo new --remote <provider> ...`, because that is where remote
+   allocation and bootstrap actually happen.
+8. If the benchmark itself needs application keys, configure runtime env
+   separately with `evo env load <path> --all` or
+   `evo env load <path> --allow KEY1,KEY2`. Provider auth provisions the
+   sandbox; runtime env is what benchmark/gate processes see.
+
+## Pre-assumptions
+
+Before trying to switch a workspace to a remote provider, confirm the basics:
+
+- the target backend is clear from the user's request; only ask if the
+  intent is genuinely ambiguous between `worktree`, `pool`, and `remote`
+- the machine running evo has the right provider SDK or transport installed
+- the user has auth for that provider available now, not "somewhere else"
+- the provider-specific minimum config exists
+  - `modal`: auth + optional config
+  - `e2b`: API key + optional config
+  - `daytona`: API key and API URL/target if needed
+  - `aws`: creds, region, image, SSH key pair/private key, and usually network config
+  - `azure`: subscription, resource group, region, SSH key/private key, and VM/image choices
+  - `ssh`: reachable host, working SSH user, and key/port if needed
+  - `manual`: reachable remote endpoint URL and bearer token
+- for SSH-backed VM providers, the guest assumptions are plausible before allocation:
+  - the image enables SSH
+  - the SSH user matches the image
+  - the image architecture matches the selected instance type
+  - the host can run evo's remote workspace runtime
+
+## Provider notes
+
+See `references/provider-matrix.md` for the compact provider summary, common config, and provider-specific setup/auth command.

package/skills/infra-setup/references/provider-matrix.md

@@ -0,0 +1,25 @@
+## Provider Matrix
+
+Use this as the compact summary. This is setup guidance, not a runtime dependency list for evo itself.
+
+| Provider | What evo uses at runtime | Setup / auth | Common config |
+|---|---|---|---|
+| `modal` | Modal Python SDK | If missing, install `evo-hq-cli[modal]` (or inject `modal` with `pipx`); then run `modal token new` | `app_name`, `gpu`, `region`, `timeout_seconds`, `health_timeout_seconds`, `apt_install`, `pip_install` |
+| `e2b` | E2B Python SDK | If missing, install `evo-hq-cli[e2b]` (or inject `e2b` with `pipx`); then `export E2B_API_KEY=...` | `template`, `api_key`, `domain`, `root`, `timeout_seconds`, `health_timeout_seconds`, `allow_internet_access`, `secure` |
+| `daytona` | Daytona Python SDK | If missing, install `evo-hq-cli[daytona]` (or inject `daytona` with `pipx`); then `export DAYTONA_API_KEY=...` | `api_key`, `api_url`, `target`, `timeout_seconds`, `health_timeout_seconds`, `ssh_host`, `ssh_port`, `ssh_token_ttl_minutes`, `sandbox_timeout_seconds` |
+| `aws` | `boto3` | If missing, install `evo-hq-cli[aws]` (or inject `boto3` with `pipx`); then export AWS creds and region | `region`, `image_id`, `key_name`, `key`, `instance_type`, `subnet_id`, `security_group_ids`, `ssh_user`, `ssh_port`, `timeout_seconds`, `health_timeout_seconds`, `keep_warm` |
+| `azure` | Azure Python SDK (`azure-identity`, `azure-mgmt-resource`, `azure-mgmt-network`, `azure-mgmt-compute`) | If missing, install `evo-hq-cli[azure]`; then use `az login` or Azure env creds, and provide subscription/resource-group config | `subscription_id`, `resource_group`, `location`, `vm_size`, `image`, `key`, `ssh_public_key`, `ssh_user`, `ssh_cidr`, `vnet_cidr`, `subnet_cidr`, `ssh_port`, `timeout_seconds`, `health_timeout_seconds`, `keep_warm` |
+| `ssh` | local `ssh` transport | `ssh user@host` must work first; then add `-i` / `-p` if needed | `host`, `key`, `port`, `tunnel_port`, `keep_warm`, `health_timeout_seconds` |
+| `manual` | existing remote workspace endpoint | no provisioning; only ask for URL/token if the user explicitly wants manual mode | `base_url`, `bearer_token`, `workspace_root`, `bundle_dir` |
+
+Notes:
+- `evo` runtime uses the provider SDK or transport listed in the second column.
+- The `evo-hq-cli[<provider>]` extras are the preferred install path when the provider SDK is missing.
+- Provider auth/setup is operator guidance. It is not the same thing as evo's runtime dependency surface.
+- Common failures are usually one of: missing SDK import, missing auth state/env var, unreachable host/port, or provider-specific bootstrap mismatch.
+- Incomplete provider setup usually surfaces on `evo new --remote <provider> ...`, because that is where remote allocation and bootstrap actually happen.
+- For SSH-backed VM providers, also validate the guest assumptions:
+  - the instance image has SSH enabled
+  - the SSH user matches the image
+  - the image architecture matches the selected instance type
+  - the remote host can run evo's remote workspace runtime