@evolvingmachines/evolve 0.0.55-project-sable.20260916.db0bc0bb

package/docs-evals/sdk-reference/agents.mdx

@@ -0,0 +1,105 @@
+---
+title: "agents"
+description: "Register and manage your own agents."
+---
+
+`agents()` returns the registered-agents client. A registered agent's name works in a job's `agents[].name` like a built-in.
+
+## create
+
+<Tabs>
+  <Tab title="TypeScript">
+    ```ts
+    create(input: { name: string; run_command: string; env?: Record<string, string> } & ({ install_script: string } | { directory: string })): Promise<Agent>
+    ```
+
+    Register an agent from an install script's text, or from a local directory. `run_command` runs with `sh -c` at the task's working directory; what it can rely on, and the registration limit, are on [agents](/core-concepts/agents#register-your-own-agent).
+
+    ```ts
+    const agent = await agents().create({
+      name: "acme-cli",
+      install_script: "curl -fsSL https://acme.dev/install.sh | sh",
+      run_command: "acme-cli --headless",
+    });
+    ```
+  </Tab>
+  <Tab title="Python">
+    ```python
+    async def create(
+        *,
+        name: str,
+        install_script: Optional[str] = None,
+        directory: Optional[str] = None,
+        run_command: str,
+        env: Optional[Dict[str, str]] = None,
+    ) -> Agent
+    ```
+
+    Register an agent from an install script's text, or from a local directory. `run_command` runs with `sh -c` at the task's working directory; what it can rely on, and the registration limit, are on [agents](/core-concepts/agents#register-your-own-agent).
+
+    ```python
+    agent = await agents().create(
+        name="acme-cli",
+        install_script="curl -fsSL https://acme.dev/install.sh | sh",
+        run_command="acme-cli --headless",
+    )
+    ```
+  </Tab>
+</Tabs>
+
+## list, get, upsert, delete
+
+<Tabs>
+  <Tab title="TypeScript">
+    ```ts
+    list(options?: { limit?: number; cursor?: string }): AgentList
+    get(name: string): Promise<Agent>
+    upsert(
+      name: string,
+      input: { run_command: string; env?: Record<string, string> } & ({ install_script: string } | { directory: string }),
+    ): Promise<Agent>
+    delete(name: string): Promise<void>
+    ```
+
+    Your agents, one agent, replace one without a gap, or delete one. Past jobs keep the agent they recorded. `upsert` creates when the name is free and replaces when it is not, consuming no new slot; it is a full replacement, so an omitted `env` becomes empty.
+
+    ```ts
+    await agents().upsert(
+      "acme-cli",
+      {
+        install_script: "curl -fsSL https://acme.dev/install.sh | sh",
+        run_command: "acme-cli --headless --v2",
+      },
+    );
+    ```
+  </Tab>
+  <Tab title="Python">
+    ```python
+    def list(
+        *,
+        limit: Optional[int] = None,
+        cursor: Optional[str] = None,
+    )
+    async def get(name: str) -> Agent
+    async def upsert(
+        name: str,
+        *,
+        run_command: str,
+        install_script: Optional[str] = None,
+        directory: Optional[str] = None,
+        env: Optional[Dict[str, str]] = None,
+    ) -> Agent
+    async def delete(name: str) -> None
+    ```
+
+    Your agents, one agent, replace one without a gap, or delete one. Past jobs keep the agent they recorded. `upsert` creates when the name is free and replaces when it is not, consuming no new slot; it is a full replacement, so an omitted `env` becomes empty.
+
+    ```python
+    await agents().upsert(
+        "acme-cli",
+        install_script="curl -fsSL https://acme.dev/install.sh | sh",
+        run_command="acme-cli --headless --v2",
+    )
+    ```
+  </Tab>
+</Tabs>

package/docs-evals/sdk-reference/analyses.mdx

@@ -0,0 +1,125 @@
+---
+title: "analyses"
+description: "Read trace-analysis runs."
+---
+
+`analyses()` returns the analyses client. Analyses are started from a job, with `jobs().analyze()`; this client reads them back.
+
+## list
+
+<Tabs>
+  <Tab title="TypeScript">
+    ```ts
+    list(options?: { scope?: "my" | "shared"; job?: string; status?: AnalysisStatus[]; limit?: number; cursor?: string }): AnalysisList
+    ```
+
+    Analysis runs, newest first, each naming the trial, job and task it judged.
+
+    ```ts
+    for await (const a of analyses().list({
+      job: job.id,
+      status: ["failed"],
+    })) {
+      console.log(a.id, a.label);
+    }
+    ```
+  </Tab>
+  <Tab title="Python">
+    ```python
+    def list(
+        *,
+        scope: Optional[JobListScope] = None,
+        job: Optional[str] = None,
+        status: Optional[List[AnalysisStatus]] = None,
+        limit: Optional[int] = None,
+        cursor: Optional[str] = None,
+    )
+    ```
+
+    Analysis runs, newest first, each naming the trial, job and task it judged.
+
+    ```python
+    async for a in analyses().list(
+        job=job.id,
+        status=["failed"],
+    ):
+        print(a["id"], a["label"])
+    ```
+  </Tab>
+</Tabs>
+
+## get, transcript, artifact
+
+<Tabs>
+  <Tab title="TypeScript">
+    ```ts
+    get(analysisId: string): Promise<TrialAnalysis>
+    transcript(
+      analysisId: string,
+      options?: { since?: number },
+    ): Promise<AnalysisTranscript>
+    artifact(
+      analysisId: string,
+      stream: "trace-stdout" | "trace-stderr",
+    ): Promise<string | null>
+    artifact(
+      analysisId: string,
+      stream: "agent-home",
+    ): Promise<Record<string, string> | null>
+    ```
+
+    The verdict document for every analysis, failed ones included, the analyzer's own transcript (everything after `since` in one read), or one stored artifact; an analysis has no verifier log and no trajectory. A run opens to its job's creator and every member of the organization; any other id answers `analysis_not_found` (`trial_not_found` from `transcript`).
+
+    ```ts
+    const verdict = await analyses().get("a0a1b2c3-…");
+    ```
+  </Tab>
+  <Tab title="Python">
+    The Python client has `list` and `download` only. Read a verdict from the analyzed trial: `(await trials().get(trial_id)).analysis`.
+  </Tab>
+</Tabs>
+
+## download
+
+<Tabs>
+  <Tab title="TypeScript">
+    ```ts
+    download(analysisId: string): Promise<Buffer>
+    download(
+      analysisId: string,
+      options: { to: string },
+    ): Promise<string>
+    download(
+      analysisId: string,
+      options: { stream: true },
+    ): Promise<ReadableStream<Uint8Array>>
+    ```
+
+    The run as one `.tar.gz`, `analyze-<analyzed trial>__<7 chars>/`: a trial tree of the analyzer's own run, with `artifacts/analysis.json` on a completed run. The bytes, the saved file's path, or a stream. `analysis_not_terminal` while the run is queued or running.
+
+    ```ts
+    const path = await analyses().download(
+      "a0a1b2c3-…",
+      { to: "./analyses" },
+    );
+    ```
+  </Tab>
+  <Tab title="Python">
+    ```python
+    async def download(
+        analysis_id: str,
+        *,
+        to: Optional[str] = None,
+    )
+    ```
+
+    The run as one `.tar.gz`, `analyze-<analyzed trial>__<7 chars>/`: the bytes, or with `to` the saved file's path. `analysis_not_found` for an id you cannot read, `analysis_not_terminal` while the run is queued or running.
+
+    ```python
+    path = await analyses().download(
+        "a0a1b2c3-…",
+        to="./analyses",
+    )
+    ```
+  </Tab>
+</Tabs>

package/docs-evals/sdk-reference/auth.mdx

@@ -0,0 +1,76 @@
+---
+title: "auth and orgs"
+description: "Who you are, and your organizations."
+---
+
+`auth()` returns the identity client, `orgs()` the organizations client. Every job, dataset and analysis belongs to an organization, today your personal one; the API accepts an `org` on a job or a publish, but the SDK and CLI do not name one yet.
+
+## auth.status
+
+<Tabs>
+  <Tab title="TypeScript">
+    ```ts
+    status(): Promise<AuthStatus>
+    ```
+
+    The caller and the API key in use: `user_id`, `email`, and `key` with `id`, `label`, `created_at` and `last_used_at`. The key's secret is never returned, and `last_used_at` is always null today.
+
+    ```ts
+    const me = await auth().status();
+    ```
+  </Tab>
+  <Tab title="Python">
+    ```python
+    async def status() -> AuthStatus
+    ```
+
+    The caller and the API key in use. The key's secret is never returned.
+
+    ```python
+    me = await auth().status()
+    ```
+  </Tab>
+</Tabs>
+
+## orgs.list, orgs.get
+
+<Tabs>
+  <Tab title="TypeScript">
+    ```ts
+    list(): Promise<Organization[]>
+    get(org: string): Promise<OrganizationDetail>
+    ```
+
+    The organizations you belong to, or one organization with your role, its members, its quota and live usage.
+
+    ```ts
+    const org = await orgs().get("acme");
+    ```
+  </Tab>
+  <Tab title="Python">
+    ```python
+    async def list() -> List[Organization]
+    async def get(org: str) -> OrganizationDetail
+    ```
+
+    The organizations you belong to, or one organization with your role, its members, its quota and live usage.
+
+    ```python
+    org = await orgs().get("acme")
+    ```
+  </Tab>
+</Tabs>
+
+## Quota and usage
+
+`quota` is nine ceilings, each the value the platform administrator set or the fleet default; only the administrator's dashboard sets them, never a key or the SDK.
+
+- `max_queued_trials`: the one that refuses. A job whose trials would not fit is refused with `quota_exceeded`.
+- `max_concurrent_trials`, `max_concurrent_imports`, `max_concurrent_analyses` (trace analyses and task checks under one count): work beyond them waits.
+- `monthly_budget_usd`: model spend allowed per UTC calendar month; a trial it refuses settles `BUDGET` with `team:` in its message. Null means no monthly budget.
+- `max_concurrent_sandboxes_e2b`, `_daytona`, `_modal`: the organization's sandboxes in flight per provider, trials, analyses, regrade verifiers and managed sessions together; work beyond one waits, and an organization with no value of its own reads the platform's fleet ceiling.
+- `max_concurrent_sessions`: managed-agent sessions open at once; recorded, not yet enforced.
+
+A `0` pauses that kind of work: creates are refused at `max_queued_trials` 0, work waits at 0 on the concurrency ceilings, and `monthly_budget_usd` 0 pauses all model spend for the month.
+
+`usage` is the live load: `in_flight_trials`, `queued_trials`, `in_flight_imports`, `in_flight_analyses`, `active_sessions` (always 0 on a shared organization), and `month_spend_usd` with `month_spend_as_of`, the month-to-date model spend as of that time (UTC calendar month, reset on the 1st); it may lag by a few minutes. It is null, never 0, when no reading is held; `evolve auth org show` prints `unavailable` then.

package/docs-evals/sdk-reference/checks.mdx

@@ -0,0 +1,166 @@
+---
+title: "checks"
+description: "Start and read task quality checks."
+---
+
+`checks()` returns the checks client.
+
+## create
+
+<Tabs>
+  <Tab title="TypeScript">
+    ```ts
+    create(input: { source: { directory: string } | { dataset: string }; model_name?: string; rubric?: Rubric; prompt?: string; reasoning_effort?: string; sandbox_provider?: EvalSandboxProvider; n_concurrent?: number; include_task_names?: string[]; exclude_task_names?: string[]; n_tasks?: number; onUploadProgress?: (sentBytes: number, totalBytes: number) => void }): Promise<Check>
+    ```
+
+    Check a local task directory or a directory of them, uploaded as one archive (`upload_too_large` past the published ceiling), or a published dataset, nothing uploaded. Returns the accepted check with one `queued` result per task; an empty selection is refused with `no_checkable_tasks`, and a dataset version resolves as for a job (`dataset_not_found`, `no_active_version`, `version_not_ready`). A check on a dataset version also appears on every job spanning it, as `check` on `jobs().tasks()` rows.
+
+    ```ts
+    const check = await checks().create({ source: { dataset: "my-swe@1.0" } });
+    ```
+  </Tab>
+  <Tab title="Python">
+    ```python
+    async def create(
+        directory: Optional[str] = None,
+        *,
+        dataset: Optional[str] = None,
+        model_name=None,
+        rubric: Optional[Rubric] = None,
+        prompt=None,
+        reasoning_effort=None,
+        sandbox_provider=None,
+        n_concurrent=None,
+        include_task_names=None,
+        exclude_task_names=None,
+        n_tasks=None,
+        on_upload_progress=None,
+    ) -> Check
+    ```
+
+    Check a local task directory or a directory of them, uploaded as one archive (`upload_too_large` past the published ceiling), or a published dataset, nothing uploaded; one source, not both. Returns the accepted check with one `queued` result per task; an empty selection is refused with `no_checkable_tasks`, and a dataset version resolves as for a job. A check on a dataset version also appears on every job spanning it, as `check` on `jobs().tasks()` rows.
+
+    ```python
+    check = await checks().create(dataset="my-swe@1.0")
+    ```
+  </Tab>
+</Tabs>
+
+## get, list, watch
+
+<Tabs>
+  <Tab title="TypeScript">
+    ```ts
+    get(checkId: string): Promise<Check>
+    list(options?: { scope?: "my" | "shared"; status?: CheckStatus[]; limit?: number; cursor?: string }): CheckList
+    watch(checkId: string, options?: { onProgress?: (check: Check) => void; signal?: AbortSignal; pollIntervalMs?: number }): Promise<Check>
+    ```
+
+    One check with its per-task results, your checks, or a poll until every task settled.
+
+    ```ts
+    const done = await checks().watch(check.id);
+    ```
+  </Tab>
+  <Tab title="Python">
+    ```python
+    async def get(check_id: str) -> Check
+    def list(
+        *,
+        scope: Optional[JobListScope] = None,
+        status: Optional[List[CheckStatus]] = None,
+        limit: Optional[int] = None,
+        cursor: Optional[str] = None,
+    )
+    async def watch(
+        check_id: str,
+        *,
+        on_progress: Optional[Callable[[Check], None]] = None,
+        poll_interval_s: float = 2.0,
+        timeout_s: Optional[float] = None,
+    ) -> Check
+    ```
+
+    One check with its per-task results, your checks, or a poll until every task settled.
+
+    ```python
+    done = await checks().watch(check.id)
+    ```
+  </Tab>
+</Tabs>
+
+## task, transcript, artifact
+
+<Tabs>
+  <Tab title="TypeScript">
+    ```ts
+    task(taskCheckId: string): Promise<TaskCheck>
+    transcript(
+      taskCheckId: string,
+      options?: { since?: number },
+    ): Promise<TaskCheckTranscript>
+    artifact(
+      taskCheckId: string,
+      stream: "trace-stdout" | "trace-stderr",
+    ): Promise<string | null>
+    artifact(
+      taskCheckId: string,
+      stream: "agent-home",
+    ): Promise<Record<string, string> | null>
+    ```
+
+    One task's result by its task check id, the checker's own transcript (everything after `since` in one read), or one stored artifact; a task check has no verifier log and no trajectory. An id you may not read answers `trial_not_found`.
+
+    ```ts
+    const result = await checks().task(done.results[0].id);
+    ```
+  </Tab>
+  <Tab title="Python">
+    The Python client has no per-task reads. A task's result is in `Check.results`, returned by `get` and `watch`.
+  </Tab>
+</Tabs>
+
+## download
+
+<Tabs>
+  <Tab title="TypeScript">
+    ```ts
+    download(id: string): Promise<Buffer>
+    download(
+      id: string,
+      options: { to: string },
+    ): Promise<string>
+    download(
+      id: string,
+      options: { stream: true },
+    ): Promise<ReadableStream<Uint8Array>>
+    ```
+
+    The whole check by its check id (`check-<id>/` with `check_report.json` and one folder per task), or one task's folder by its task check id (`check-<task>__<7 chars>/`), as one `.tar.gz`. Each task folder is a trial tree of the checker's own run, with `artifacts/check-result.json` on a completed run. An id that is neither answers `check_not_found`; an unsettled check or task check answers `check_not_terminal`.
+
+    ```ts
+    const path = await checks().download(
+      check.id,
+      { to: "./checks" },
+    );
+    ```
+  </Tab>
+  <Tab title="Python">
+    ```python
+    async def download(
+        id: str,
+        *,
+        to: Optional[str] = None,
+    )
+    ```
+
+    The whole check by its check id (`check-<id>/` with `check_report.json` and one folder per task), or one task's folder by its task check id (`check-<task>__<7 chars>/`), as one `.tar.gz`. An id that is neither answers `check_not_found`; an unsettled check or task check answers `check_not_terminal`.
+
+    ```python
+    path = await checks().download(
+        check.id,
+        to="./checks",
+    )
+    ```
+  </Tab>
+</Tabs>