@evolvingmachines/evolve 0.0.55-project-sable.20260916.db0bc0bb

package/docs-evals/docs.json

@@ -0,0 +1,180 @@
+{
+  "$schema": "https://mintlify.com/docs.json",
+  "name": "Evolve",
+  "description": "Evolve hosted evals: run any model on any agent harness against datasets of Harbor-format tasks, in cloud sandboxes, from the evolve CLI and the TypeScript and Python SDKs. Use when the user wants to: (1) start, watch, or read a hosted eval job (evolve run, evolve job, jobs()), (2) browse the dataset catalog or publish their own dataset (evolve dataset), (3) check task quality (evolve check) or analyze trial traces (evolve analyze), (4) inspect or download trials, artifacts and traces (evolve trial, trials()), (5) register their own agent, mount skills, attach secrets, or choose models and sandbox providers, (6) upload a job run elsewhere (evolve upload), (7) use the dashboard's jobs, datasets, checks, traces and trial viewer pages. For running agents in sandboxes from code (the SDK's run(), Swarm, Pipeline) read `evolve skills get agents`.",
+  "theme": "maple",
+  "colors": {
+    "primary": "#B026FF",
+    "light": "#B026FF",
+    "dark": "#B026FF"
+  },
+  "favicon": "/favicon.png",
+  "logo": {
+    "light": "/logo/wordmark-light.png",
+    "dark": "/logo/wordmark-dark.png",
+    "href": "https://evolvingmachines.ai"
+  },
+  "appearance": {
+    "default": "light"
+  },
+  "background": {
+    "color": {
+      "light": "#f7f6f5",
+      "dark": "#0a0a0a"
+    }
+  },
+  "fonts": {
+    "family": "Google Sans Flex",
+    "weight": 400,
+    "heading": {
+      "family": "Google Sans Flex",
+      "weight": 600
+    }
+  },
+  "icons": {
+    "library": "lucide"
+  },
+  "styling": {
+    "eyebrows": "breadcrumbs",
+    "codeblocks": "dark"
+  },
+  "navbar": {
+    "links": [
+      {
+        "type": "github",
+        "href": "https://github.com/evolving-machines-lab/evolve"
+      }
+    ],
+    "primary": {
+      "type": "button",
+      "label": "Dashboard",
+      "href": "https://dashboard.evolvingmachines.ai"
+    }
+  },
+  "footer": {
+    "socials": {
+      "website": "https://evolvingmachines.ai",
+      "github": "https://github.com/evolving-machines-lab/evolve",
+      "discord": "https://discord.gg/Q36D8dGyNF"
+    }
+  },
+  "contextual": {
+    "options": [
+      "copy",
+      "view",
+      "claude",
+      "chatgpt"
+    ],
+    "display": "header"
+  },
+  "redirects": [
+    {
+      "source": "/core-concepts/tasks-and-datasets",
+      "destination": "/core-concepts/datasets"
+    }
+  ],
+  "navigation": {
+    "tabs": [
+      {
+        "tab": "Docs",
+        "groups": [
+          {
+            "group": "Getting started",
+            "pages": [
+              "index",
+              "getting-started/installation",
+              "getting-started/quick-start"
+            ]
+          },
+          {
+            "group": "Core concepts",
+            "pages": [
+              "core-concepts/index",
+              "core-concepts/tasks",
+              "core-concepts/datasets",
+              "core-concepts/jobs",
+              "core-concepts/trials",
+              "core-concepts/trial-outputs",
+              "core-concepts/agents",
+              "core-concepts/models",
+              "core-concepts/sandboxes",
+              "core-concepts/analyze",
+              "core-concepts/check",
+              "core-concepts/skills",
+              "core-concepts/secrets",
+              "core-concepts/upload"
+            ]
+          },
+          {
+            "group": "Dashboard",
+            "pages": [
+              "dashboard/datasets",
+              "dashboard/jobs",
+              "dashboard/trial-viewer",
+              "dashboard/checks",
+              "dashboard/traces"
+            ]
+          },
+          {
+            "group": "SDK",
+            "pages": [
+              "sdk/typescript",
+              "sdk/python"
+            ]
+          }
+        ]
+      },
+      {
+        "tab": "CLI reference",
+        "groups": [
+          {
+            "group": "The evolve command",
+            "pages": [
+              "cli-reference/index",
+              "cli-reference/run",
+              "cli-reference/analyze",
+              "cli-reference/check",
+              "cli-reference/upload"
+            ]
+          },
+          {
+            "group": "Command groups",
+            "pages": [
+              "cli-reference/job",
+              "cli-reference/trial",
+              "cli-reference/analysis",
+              "cli-reference/dataset",
+              "cli-reference/skill",
+              "cli-reference/skills",
+              "cli-reference/agent",
+              "cli-reference/auth",
+              "cli-reference/secrets"
+            ]
+          }
+        ]
+      },
+      {
+        "tab": "SDK reference",
+        "groups": [
+          {
+            "group": "Clients",
+            "pages": [
+              "sdk-reference/index",
+              "sdk-reference/jobs",
+              "sdk-reference/trials",
+              "sdk-reference/datasets",
+              "sdk-reference/analyses",
+              "sdk-reference/checks",
+              "sdk-reference/skills",
+              "sdk-reference/agents",
+              "sdk-reference/auth",
+              "sdk-reference/meta",
+              "sdk-reference/types",
+              "sdk-reference/errors"
+            ]
+          }
+        ]
+      }
+    ]
+  }
+}

package/docs-evals/getting-started/installation.mdx

@@ -0,0 +1,56 @@
+---
+title: "Installation"
+description: "Install the evolve command and the SDKs, and set your API key."
+---
+
+## The CLI
+
+The `evolve` command ships inside the npm package `@evolvingmachines/evolve`.
+
+```bash
+npm install -g @evolvingmachines/evolve
+evolve --version
+```
+
+Inside a project that has the package, `npx evolve` runs it. For a one-off without installing, name the package: `npx --package=@evolvingmachines/evolve evolve --help`. A bare `npx evolve` fetches an unrelated package of that name from the public registry.
+
+## The SDKs
+
+The same surface is available as a library in TypeScript and in Python.
+
+<CodeGroup>
+
+```bash TypeScript
+npm install @evolvingmachines/evolve
+```
+
+```bash Python
+pip install evolvingmachines-evolve
+```
+
+</CodeGroup>
+
+The Python package needs Python 3.10 or newer. The hosted clients on these pages talk to the platform directly. Node.js and npm are needed only to run agents on your own machine through `run()`, which these pages do not cover.
+
+## Your API key
+
+Create a key on the dashboard's [API keys page](https://dashboard.evolvingmachines.ai/api-keys), then export it. Every command and every SDK client reads `EVOLVE_API_KEY`.
+
+```bash
+export EVOLVE_API_KEY="<your key>"
+evolve auth status
+```
+
+`auth status` prints who the platform thinks you are and which key is in use. The key itself is never printed back. To use a different key for one call, pass `--api-key <key>` on the command line, or `{ apiKey }` to an SDK client.
+
+## Your coding agent
+
+```bash
+evolve skills install
+```
+
+One small skill, `evolve`, lands in the skill folder of every coding agent on your machine: Claude Code, Codex, Cursor, Copilot, Gemini CLI and OpenCode. From then on, when you mention Evolve, evals, datasets, jobs or tasks, the agent reads the manual from the command itself, `evolve skills get evals`, one page at a time. The manual ships inside the package, so it always matches the version you installed. See [evolve skills](/cli-reference/skills).
+
+<Card title="Quick start" icon="play" href="/getting-started/quick-start">
+  Start your first job.
+</Card>

package/docs-evals/getting-started/quick-start.mdx

@@ -0,0 +1,85 @@
+---
+title: "Quick start"
+description: "Start your first job, watch it run, and download a trial."
+---
+
+This page takes you from an empty terminal to a downloaded trial: one API key, any harness, any model, any sandbox provider, and every trial's trace streamed live. A trial is one agent's attempt at one task. A job is a set of trials: the tasks of one or more datasets, run by one or more agents.
+
+<Steps>
+  <Step title="Install the CLI and set your key">
+    ```bash
+    npm install -g @evolvingmachines/evolve
+    export EVOLVE_API_KEY="<your key>"
+    evolve auth status
+    ```
+
+    Create a key on the dashboard's [API keys page](https://dashboard.evolvingmachines.ai/api-keys), then export it. See [installation](/getting-started/installation) for the SDKs.
+  </Step>
+
+  <Step title="Pick a dataset">
+    The catalog lists every dataset you can run, with its versions.
+
+    ```bash
+    evolve dataset list
+    evolve dataset show terminal-bench-4@4.0
+    ```
+
+    `dataset show` prints the version's tasks and, for each task, which sandbox providers can run it.
+  </Step>
+
+  <Step title="Start a job and watch it">
+    Name a dataset, an agent, and a model. `--watch` streams the job's events until it finishes; `-l 5` caps the run at five tasks so the first job is small.
+
+    ```bash
+    evolve run \
+      -d terminal-bench-4@4.0 \
+      -a codex \
+      -m gpt-5.5 \
+      -l 5 \
+      --watch
+    ```
+
+    The final block prints the job id, how many trials landed in each status, and what the trials spent. Without `--watch` the command returns the id at once and the job keeps running.
+  </Step>
+
+  <Step title="Read the result">
+    ```bash
+    evolve job show <job-id>
+    evolve job trials <job-id>
+    evolve trial show <trial-id>
+    ```
+
+    `job show` is the whole job: how many trials landed in each status, the rewards, the spend. `job trials` lists the trials one per line with their ids. `trial show` is one trial in full, with its reward and, when it failed, why.
+  </Step>
+
+  <Step title="Download the trial">
+    A trial downloads as a trial tree: the trajectory, the agent's stdout and stderr, the verifier log, and the reward.
+
+    ```bash
+    evolve trial download <trial-id> -o trials/
+    ```
+
+    To read one artifact without saving anything, stream it instead.
+
+    ```bash
+    evolve trial download <trial-id> --stream verifier
+    ```
+  </Step>
+</Steps>
+
+## Next
+
+<Columns cols={2}>
+  <Card title="Jobs" icon="list-checks" href="/core-concepts/jobs">
+    Attempts, concurrency, retries, spend caps, and what to do with a finished job.
+  </Card>
+  <Card title="Analyze" icon="search" href="/core-concepts/analyze">
+    Judge every trial's trace against a rubric with `--analyze`.
+  </Card>
+  <Card title="Agents" icon="bot" href="/core-concepts/agents">
+    The seven built-in harnesses, and how to bring your own.
+  </Card>
+  <Card title="CLI reference" icon="terminal" href="/cli-reference/run">
+    Every flag of `evolve run`.
+  </Card>
+</Columns>

package/docs-evals/index.mdx

@@ -0,0 +1,36 @@
+---
+title: "Evolve documentation"
+description: "Environment-driven agent evaluations: any model, any harness, any sandbox, one API key. Every task versioned and checked, every trajectory analyzed."
+---
+
+Evolve is hosted evaluation for agents: bring your own tasks or pick a public dataset, and run any model on any agent harness against them, in parallel, in cloud sandboxes. Each task is pinned to one versioned environment, so every run starts from the same place today or next month. Every trial returns its reward, its live trace, its files and its logs, with a quality verdict on any task before you spend on it and a judged analysis of any trajectory after.
+
+- **One API key.** Any model with `-m`, any agent harness with `-a`, any sandbox provider with `-e`; no provider keys to manage.
+- **Task quality control.** `evolve check` runs each task and judges it against a rubric before you spend a job on it.
+- **Full observability.** Every trial streams its trace live; its files, artifacts, agent logs and verifier logs are yours to read and download when it ends.
+- **Trace analysis after every trial.** `evolve analyze` judges every trial's trace against a rubric, cheap and scalable.
+
+```bash
+export EVOLVE_API_KEY="<your key>"   # from https://dashboard.evolvingmachines.ai/api-keys
+evolve run \
+  -d terminal-bench-4@4.0 \
+  -a codex \
+  -m gpt-5.5 \
+  -l 5 \
+  --watch
+```
+
+<Columns cols={2}>
+  <Card title="Quick start" icon="play" href="/getting-started/quick-start">
+    Install the CLI, start a job, read its result.
+  </Card>
+  <Card title="Core concepts" icon="layers" href="/core-concepts">
+    How tasks, jobs, trials, agents and sandboxes fit together.
+  </Card>
+  <Card title="CLI reference" icon="terminal" href="/cli-reference">
+    Every verb and flag of the `evolve` command.
+  </Card>
+  <Card title="Dashboard" icon="layout-dashboard" href="/dashboard/datasets">
+    Datasets, jobs and the trial viewer in the browser.
+  </Card>
+</Columns>

package/docs-evals/sdk/python.mdx

@@ -0,0 +1,139 @@
+---
+title: "Python SDK"
+description: "Start a job, wait for it, read the result, and download it from Python."
+---
+
+```bash
+pip install evolvingmachines-evolve
+```
+
+The hosted surface is a set of clients, one per noun. Each one reads `EVOLVE_API_KEY` from the environment, or takes a `HostedClientConfig(api_key=..., base_url=...)`. Create a key on the dashboard's [API keys page](https://dashboard.evolvingmachines.ai/api-keys).
+
+Every call is a coroutine, and every client is an async context manager with `close()`. A request never follows a redirect: the client refuses every 3xx and raises it as an error, so the key only reaches the host you configured.
+
+```python
+import asyncio
+from evolve import jobs, trials, datasets
+
+evals = jobs()
+```
+
+## Start a job
+
+```python
+job = await evals.start(
+    datasets=[{"name": "terminal-bench-4", "version": "4.0", "n_tasks": 5}],
+    agents=[{"name": "codex", "model_name": "gpt-5.5"}],
+    n_attempts=1,
+    n_concurrent_trials=4,
+    max_trial_spend_usd=25,
+)
+
+print(job.id, job.status)   # "QUEUED"
+```
+
+`datasets` is a list of selectors, as dicts or `DatasetSelector` dataclasses. A bare `name` resolves to the active version; `task_names`, `exclude_task_names` and `n_tasks` narrow the task set. Every arm in `agents`, a dict or an `AgentArm`, names a `model_name`, and `version`, `reasoning_effort`, `kwargs`, `preset` and `skills` are optional.
+
+`sandbox_provider` picks `"e2b"`, `"daytona"` or `"modal"`. The retry policy is a `JobRetryConfigInput` dict and comes back resolved as `JobRetryConfig`. The fields are the same ones a `-c` config file takes on the CLI.
+
+Results are dataclasses with the wire's `snake_case` names and `Literal` vocabularies for every closed set. Four keys the wire spells in camelCase are mapped: `next_cursor`, `has_more`, `by_status` and `task_matrix`. A refused request raises `EvolveAPIError`; see [errors](/sdk-reference/errors).
+
+## Wait for it
+
+`watch()` returns a handle you either await, for the final job, or iterate, for each event. Pick one form per call.
+
+```python
+final = await evals.watch(job.id)
+print(final.status, final.trials.by_status, final.stats.get("cost_usd"))
+```
+
+```python
+async for event in evals.watch(job.id):
+    if event.type == "trial.settled":
+        print(event.data["task_name"], event.data["status"], event.data.get("reward"))
+```
+
+The stream replays from the beginning, so attaching late loses nothing, and it resumes on its own after a disconnect. `event.data` is a plain dict; branch on `event.type` and read it by key.
+
+## Read the result
+
+```python
+detail = await evals.get(job.id)
+
+async for trial in evals.trials(job.id):
+    print(trial.task_name, trial.agent_info.name, trial.status, trial.reward)
+
+failures = await evals.trials(
+    job.id,
+    status=["INFRASTRUCTURE_ERROR", "SCORING_ERROR"],
+)
+```
+
+Every list on this surface works the same way: await it for one page, iterate it to walk every page. `pass_at_k(job)` reads a finished job's pass@k out of its stats as sorted numbers.
+
+A trial id is global, so `trials()` reaches one without the job.
+
+```python
+t = trials()
+
+trial = await t.get(trial_id)
+print(trial.reward, trial.exception_info and trial.exception_info.exception_type)
+
+verifier_log = await t.artifact(trial_id, "verifier")   # str | None
+home = await t.artifact(trial_id, "agent-home")         # dict[str, str] | None
+
+async for event in t.trace_events(trial_id):
+    print(event.seq, event.type)
+```
+
+`artifact()` takes the same names as `evolve trial download --stream`. None means the trial never stored that artifact; it is a normal answer, not an error.
+
+## Download
+
+```python
+path = await evals.download(
+    job.id,
+    to="./results",
+)   # the saved file's path
+data = await evals.download(job.id)                    # bytes
+```
+
+The archive is the standard job directory, the same tree `evolve job download` unpacks.
+
+## Analyze
+
+```python
+await evals.analyze(
+    job.id,
+    failing=True,
+    n_trials=20,
+)
+analyzed = await evals.watch_analysis(job.id)
+```
+
+`analyze()` enqueues one analysis per trial and returns at once. `watch_analysis()` follows the batch of analyses to its end.
+
+## The catalog
+
+```python
+catalog = datasets()
+
+async for dataset in catalog.list():
+    print(dataset.name, dataset.active_version and dataset.active_version.version)
+
+version = await catalog.get("terminal-bench-4@4.0")
+```
+
+## Everything else
+
+`analyses()`, `checks()`, `skills()`, `agents()`, `auth()` and `orgs()` are built the same way. `meta()` fetches the platform's capability document, the live list of harnesses, models and limits, and needs no key. `hosted()` builds every client from one configuration.
+
+```python
+from evolve import hosted
+
+evolve = hosted()
+doc = await evolve.meta()
+job = await evolve.jobs.start(...)
+```
+
+Run any of the snippets above inside an `async def main()` and start it with `asyncio.run(main())`.

package/docs-evals/sdk/typescript.mdx

@@ -0,0 +1,141 @@
+---
+title: "TypeScript SDK"
+description: "Start a job, wait for it, read the result, and download it from TypeScript."
+---
+
+```bash
+npm install @evolvingmachines/evolve
+```
+
+The hosted surface is a set of clients, one per noun. Each one reads `EVOLVE_API_KEY` from the environment, or takes `{ apiKey, baseUrl }`. Create a key on the dashboard's [API keys page](https://dashboard.evolvingmachines.ai/api-keys).
+
+```ts
+import { jobs, trials, datasets } from "@evolvingmachines/evolve";
+
+const evals = jobs();
+```
+
+## Start a job
+
+```ts
+const job = await evals.start({
+  datasets: [{ name: "terminal-bench-4", version: "4.0", n_tasks: 5 }],
+  agents: [{ name: "codex", model_name: "gpt-5.5" }],
+  n_attempts: 1,
+  n_concurrent_trials: 4,
+  max_trial_spend_usd: 25,
+});
+
+console.log(job.id, job.status);   // "QUEUED"
+```
+
+`datasets` is a list of selectors. A bare `name` resolves to the active version; `task_names`, `exclude_task_names` and `n_tasks` narrow the task set.
+
+Every arm in `agents` names a `model_name`, and `version`, `reasoning_effort`, `kwargs`, `preset` and `skills` are optional. `sandbox_provider` picks `"e2b"`, `"daytona"` or `"modal"`. The fields are the same ones a `-c` config file takes on the CLI.
+
+## Wait for it
+
+`watch()` returns a handle you either await, for the final job, or iterate, for each event. Pick one form per call.
+
+```ts
+const final = await evals.watch(job.id);
+console.log(final.status, final.trials.byStatus, final.stats.cost_usd);
+```
+
+```ts
+for await (const event of evals.watch(job.id)) {
+  if (event.type === "trial.settled") {
+    console.log(event.data.task_name, event.data.status, event.data.reward);
+  }
+}
+```
+
+The stream replays from the beginning, so attaching late loses nothing, and it resumes on its own after a disconnect. `event.type` narrows `event.data`, so the fields above are typed.
+
+## Read the result
+
+```ts
+const detail = await evals.get(job.id);
+
+for await (const trial of evals.trials(job.id)) {
+  console.log(trial.task_name, trial.agent_info.name, trial.status, trial.reward);
+}
+
+const failures = await evals.trials(job.id, {
+  status: ["INFRASTRUCTURE_ERROR", "SCORING_ERROR"],
+});
+```
+
+Every list on this surface works the same way: await it for one page, iterate it to walk every page. `passAtK(job)` reads a finished job's pass@k out of its stats as sorted numbers. A refused request throws `EvolveApiError`; see [errors](/sdk-reference/errors).
+
+A trial id is global, so `trials()` reaches one without the job.
+
+```ts
+const t = trials();
+
+const trial = await t.get(trialId);
+console.log(trial.reward, trial.exception_info?.exception_type);
+
+const verifierLog = await t.artifact(trialId, "verifier");   // string | null
+const home = await t.artifact(
+  trialId,
+  "agent-home",
+);   // Record<string, string> | null
+
+for await (const event of t.traceEvents(trialId)) {
+  console.log(event.seq, event.type);
+}
+```
+
+`artifact()` takes the same names as `evolve trial download --stream`. Null means the trial never stored that artifact; it is a normal answer, not an error.
+
+## Download
+
+```ts
+const path = await evals.download(
+  job.id,
+  { to: "./results" },
+);   // the saved file's path
+const bytes = await evals.download(job.id);                        // a Buffer
+```
+
+The archive is the standard job directory, the same tree `evolve job download` unpacks.
+
+## Analyze
+
+```ts
+await evals.analyze(
+  job.id,
+  {
+    failing: true,
+    n_trials: 20,
+  },
+);
+const analyzed = await evals.watchAnalysis(job.id);
+```
+
+`analyze()` enqueues one analysis per trial and returns at once. `watchAnalysis()` follows the batch of analyses to its end.
+
+## The catalog
+
+```ts
+const catalog = datasets();
+
+for await (const dataset of catalog.list()) {
+  console.log(dataset.name, dataset.active_version?.version);
+}
+
+const version = await catalog.get("terminal-bench-4@4.0");
+```
+
+## Everything else
+
+`analyses()`, `checks()`, `skills()`, `agents()`, `auth()` and `orgs()` are built the same way. `meta()` fetches the platform's capability document, the live list of harnesses, models and limits, and needs no key. `hosted()` builds every client from one configuration.
+
+```ts
+import { hosted } from "@evolvingmachines/evolve";
+
+const evolve = hosted({ apiKey: process.env.EVOLVE_API_KEY });
+const doc = await evolve.meta();
+const job = await evolve.jobs.start({ /* … */ });
+```