@evolvingmachines/evolve 0.0.55-project-sable.20260916.db0bc0bb
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +25 -0
- package/dist/chunk-5GVMBEYI.js +11 -0
- package/dist/chunk-HQJI3HHU.js +1 -0
- package/dist/chunk-JS2UTK2I.js +427 -0
- package/dist/cli/index.cjs +73 -0
- package/dist/cli/index.d.cts +270 -0
- package/dist/cli/index.d.ts +270 -0
- package/dist/cli/index.js +54 -0
- package/dist/index.cjs +669 -0
- package/dist/index.d.cts +5023 -0
- package/dist/index.d.ts +5023 -0
- package/dist/index.js +230 -0
- package/dist/managed-modal-IJW62SSK.js +4 -0
- package/dist/resumable-HFSICSH3.js +1 -0
- package/dist/tar-GVC4KE52.js +1 -0
- package/dist/types-DlpTxdR_.d.cts +4777 -0
- package/dist/types-DlpTxdR_.d.ts +4777 -0
- package/dist/upload-KPYDMWFJ.js +1 -0
- package/docs-agents/SKILL.md +138 -0
- package/docs-agents/index.md +81 -0
- package/docs-agents/python/01-getting-started.md +511 -0
- package/docs-agents/python/02-configuration.md +855 -0
- package/docs-agents/python/03-runtime.md +899 -0
- package/docs-agents/python/04-streaming.md +451 -0
- package/docs-agents/python/05-swarm-pipeline.md +953 -0
- package/docs-agents/python/index.md +122 -0
- package/docs-agents/typescript/01-getting-started.md +494 -0
- package/docs-agents/typescript/02-configuration.md +841 -0
- package/docs-agents/typescript/03-runtime.md +864 -0
- package/docs-agents/typescript/04-streaming.md +478 -0
- package/docs-agents/typescript/05-swarm-pipeline.md +957 -0
- package/docs-agents/typescript/index.md +121 -0
- package/docs-evals/SKILL.md +112 -0
- package/docs-evals/cli-reference/agent.mdx +67 -0
- package/docs-evals/cli-reference/analysis.mdx +91 -0
- package/docs-evals/cli-reference/analyze.mdx +74 -0
- package/docs-evals/cli-reference/auth.mdx +60 -0
- package/docs-evals/cli-reference/check.mdx +146 -0
- package/docs-evals/cli-reference/dataset.mdx +133 -0
- package/docs-evals/cli-reference/index.mdx +108 -0
- package/docs-evals/cli-reference/job.mdx +239 -0
- package/docs-evals/cli-reference/run.mdx +196 -0
- package/docs-evals/cli-reference/secrets.mdx +95 -0
- package/docs-evals/cli-reference/skill.mdx +51 -0
- package/docs-evals/cli-reference/skills.mdx +81 -0
- package/docs-evals/cli-reference/trial.mdx +112 -0
- package/docs-evals/cli-reference/upload.mdx +36 -0
- package/docs-evals/core-concepts/agents.mdx +106 -0
- package/docs-evals/core-concepts/analyze.mdx +86 -0
- package/docs-evals/core-concepts/check.mdx +51 -0
- package/docs-evals/core-concepts/datasets.mdx +138 -0
- package/docs-evals/core-concepts/index.mdx +42 -0
- package/docs-evals/core-concepts/jobs.mdx +136 -0
- package/docs-evals/core-concepts/models.mdx +68 -0
- package/docs-evals/core-concepts/sandboxes.mdx +52 -0
- package/docs-evals/core-concepts/secrets.mdx +41 -0
- package/docs-evals/core-concepts/skills.mdx +42 -0
- package/docs-evals/core-concepts/tasks.mdx +216 -0
- package/docs-evals/core-concepts/trial-outputs.mdx +103 -0
- package/docs-evals/core-concepts/trials.mdx +103 -0
- package/docs-evals/core-concepts/upload.mdx +45 -0
- package/docs-evals/dashboard/checks.mdx +15 -0
- package/docs-evals/dashboard/datasets.mdx +24 -0
- package/docs-evals/dashboard/jobs.mdx +65 -0
- package/docs-evals/dashboard/traces.mdx +15 -0
- package/docs-evals/dashboard/trial-viewer.mdx +30 -0
- package/docs-evals/docs.json +180 -0
- package/docs-evals/getting-started/installation.mdx +56 -0
- package/docs-evals/getting-started/quick-start.mdx +85 -0
- package/docs-evals/index.mdx +36 -0
- package/docs-evals/sdk/python.mdx +139 -0
- package/docs-evals/sdk/typescript.mdx +141 -0
- package/docs-evals/sdk-reference/agents.mdx +105 -0
- package/docs-evals/sdk-reference/analyses.mdx +125 -0
- package/docs-evals/sdk-reference/auth.mdx +76 -0
- package/docs-evals/sdk-reference/checks.mdx +166 -0
- package/docs-evals/sdk-reference/datasets.mdx +305 -0
- package/docs-evals/sdk-reference/errors.mdx +199 -0
- package/docs-evals/sdk-reference/index.mdx +58 -0
- package/docs-evals/sdk-reference/jobs.mdx +731 -0
- package/docs-evals/sdk-reference/meta.mdx +52 -0
- package/docs-evals/sdk-reference/skills.mdx +68 -0
- package/docs-evals/sdk-reference/trials.mdx +232 -0
- package/docs-evals/sdk-reference/types.mdx +272 -0
- package/docs-evals/snippets/global-options.mdx +13 -0
- package/docs-evals/snippets/list-flags.mdx +23 -0
- package/harness-capabilities.json +430 -0
- package/hosted-error-codes.json +120 -0
- package/package.json +163 -0
- package/skills/create-adapter/SKILL.md +287 -0
- package/skills/create-task/SKILL.md +491 -0
- package/skills/evolve/SKILL.md +39 -0
- package/skills/publish/SKILL.md +142 -0
- package/skills/rewardkit/SKILL.md +330 -0
- package/spec/atif/README.md +45 -0
- package/spec/atif/trajectory.schema.json +865 -0
- package/spec/openapi.yaml +13447 -0
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: "datasets"
|
|
3
|
+
description: "The catalog: list, read, publish, download, activate, delete."
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
`datasets()` returns the catalog client. A `ref` is `name` or `name@version`.
|
|
7
|
+
|
|
8
|
+
## list, get, getActive
|
|
9
|
+
|
|
10
|
+
<Tabs>
|
|
11
|
+
<Tab title="TypeScript">
|
|
12
|
+
```ts
|
|
13
|
+
list(options?: { search?: string; limit?: number; cursor?: string }): DatasetList
|
|
14
|
+
get(
|
|
15
|
+
ref: string,
|
|
16
|
+
options?: { limit?: number; cursor?: string },
|
|
17
|
+
): Promise<Dataset>
|
|
18
|
+
getActive(
|
|
19
|
+
name: string,
|
|
20
|
+
options?: { limit?: number; cursor?: string },
|
|
21
|
+
): Promise<ActiveDataset>
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
The catalog, one dataset with one page of its tasks (default 200, at most 500), or the active version alone. `getActive` throws `NoActiveVersionError` when nothing is runnable. A task row carries public fields only; see [Task](/sdk-reference/types#task).
|
|
25
|
+
|
|
26
|
+
```ts
|
|
27
|
+
const dataset = await datasets().get("terminal-bench-4@4.0");
|
|
28
|
+
```
|
|
29
|
+
</Tab>
|
|
30
|
+
<Tab title="Python">
|
|
31
|
+
```python
|
|
32
|
+
def list(
|
|
33
|
+
*,
|
|
34
|
+
search: Optional[str] = None,
|
|
35
|
+
limit: Optional[int] = None,
|
|
36
|
+
cursor: Optional[str] = None,
|
|
37
|
+
)
|
|
38
|
+
async def get(
|
|
39
|
+
ref: str,
|
|
40
|
+
*,
|
|
41
|
+
limit: Optional[int] = None,
|
|
42
|
+
cursor: Optional[str] = None,
|
|
43
|
+
) -> Dataset
|
|
44
|
+
async def get_active(
|
|
45
|
+
name: str,
|
|
46
|
+
*,
|
|
47
|
+
limit: Optional[int] = None,
|
|
48
|
+
cursor: Optional[str] = None,
|
|
49
|
+
) -> ActiveDataset
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
The catalog, one dataset with one page of its tasks, or the active version alone. `get_active` raises `NoActiveVersionError` when nothing is runnable.
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
dataset = await datasets().get("terminal-bench-4@4.0")
|
|
56
|
+
```
|
|
57
|
+
</Tab>
|
|
58
|
+
</Tabs>
|
|
59
|
+
|
|
60
|
+
## getTaskBuild
|
|
61
|
+
|
|
62
|
+
<Tabs>
|
|
63
|
+
<Tab title="TypeScript">
|
|
64
|
+
```ts
|
|
65
|
+
getTaskBuild(
|
|
66
|
+
ref: string,
|
|
67
|
+
taskName: string,
|
|
68
|
+
): Promise<TaskBuild>
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
One task's build record: its outcome and, on failure, the failing step and log.
|
|
72
|
+
|
|
73
|
+
```ts
|
|
74
|
+
const build = await datasets().getTaskBuild("my-swe@1.0", "broken-task");
|
|
75
|
+
```
|
|
76
|
+
</Tab>
|
|
77
|
+
<Tab title="Python">
|
|
78
|
+
```python
|
|
79
|
+
async def get_task_build(
|
|
80
|
+
ref: str,
|
|
81
|
+
task_name: str,
|
|
82
|
+
) -> TaskBuild
|
|
83
|
+
```
|
|
84
|
+
|
|
85
|
+
One task's build record: its outcome and, on failure, the failing step and log.
|
|
86
|
+
|
|
87
|
+
```python
|
|
88
|
+
build = await datasets().get_task_build("my-swe@1.0", "broken-task")
|
|
89
|
+
```
|
|
90
|
+
</Tab>
|
|
91
|
+
</Tabs>
|
|
92
|
+
|
|
93
|
+
## preflight
|
|
94
|
+
|
|
95
|
+
<Tabs>
|
|
96
|
+
<Tab title="TypeScript">
|
|
97
|
+
```ts
|
|
98
|
+
preflight(input: { source: { directory: string } }): Promise<DatasetPreflight>
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
Dry-run a local folder of tasks: only each task's metadata is sent, nothing is uploaded or written.
|
|
102
|
+
|
|
103
|
+
```ts
|
|
104
|
+
const verdict = await datasets().preflight({ source: { directory: "./my-swe" } });
|
|
105
|
+
```
|
|
106
|
+
</Tab>
|
|
107
|
+
<Tab title="Python">
|
|
108
|
+
```python
|
|
109
|
+
async def preflight(
|
|
110
|
+
*,
|
|
111
|
+
directory: str,
|
|
112
|
+
) -> DatasetPreflight
|
|
113
|
+
```
|
|
114
|
+
|
|
115
|
+
Dry-run a local folder of tasks: only each task's metadata is sent, nothing is uploaded or written.
|
|
116
|
+
|
|
117
|
+
```python
|
|
118
|
+
verdict = await datasets().preflight(directory="./my-swe")
|
|
119
|
+
```
|
|
120
|
+
</Tab>
|
|
121
|
+
</Tabs>
|
|
122
|
+
|
|
123
|
+
## publish
|
|
124
|
+
|
|
125
|
+
<Tabs>
|
|
126
|
+
<Tab title="TypeScript">
|
|
127
|
+
```ts
|
|
128
|
+
publish(input: { source: DatasetSource; name?: string; version?: string }, options?: { onUploadProgress?: (sentBytes: number, totalBytes: number) => void; onRegistered?: (importId: string) => void }): Promise<DatasetImport>
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
Publish a version from one `source`: `{ git_url, git_ref, git_path? }` (an `https://` URL, and a subfolder that exists at the ref), `{ directory }`, `{ archive_url }` or `{ hub_package }`. Returns the import record; follow it with `watchImport`.
|
|
132
|
+
|
|
133
|
+
`onUploadProgress` reports sent bytes for a directory source, and `onRegistered` fires once with the import id when a large upload begins. What is packed is on [datasets](/core-concepts/datasets#publish-your-own).
|
|
134
|
+
|
|
135
|
+
```ts
|
|
136
|
+
const imp = await datasets().publish({
|
|
137
|
+
source: { directory: "./my-swe" },
|
|
138
|
+
name: "my-swe",
|
|
139
|
+
version: "1.0",
|
|
140
|
+
});
|
|
141
|
+
```
|
|
142
|
+
</Tab>
|
|
143
|
+
<Tab title="Python">
|
|
144
|
+
```python
|
|
145
|
+
async def publish(
|
|
146
|
+
*,
|
|
147
|
+
git_url=None,
|
|
148
|
+
git_ref=None,
|
|
149
|
+
git_path=None,
|
|
150
|
+
directory=None,
|
|
151
|
+
archive_url=None,
|
|
152
|
+
hub_package=None,
|
|
153
|
+
name=None,
|
|
154
|
+
version=None,
|
|
155
|
+
on_upload_progress=None,
|
|
156
|
+
on_registered=None,
|
|
157
|
+
) -> DatasetImport
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
Publish a version from one source: `git_url` with `git_ref` (and `git_path`), `directory`, `archive_url`, or `hub_package`. Returns the import record; follow it with `watch_import`.
|
|
161
|
+
|
|
162
|
+
```python
|
|
163
|
+
imp = await datasets().publish(
|
|
164
|
+
directory="./my-swe",
|
|
165
|
+
name="my-swe",
|
|
166
|
+
version="1.0",
|
|
167
|
+
)
|
|
168
|
+
```
|
|
169
|
+
</Tab>
|
|
170
|
+
</Tabs>
|
|
171
|
+
|
|
172
|
+
## getImport, watchImport, listImports
|
|
173
|
+
|
|
174
|
+
<Tabs>
|
|
175
|
+
<Tab title="TypeScript">
|
|
176
|
+
```ts
|
|
177
|
+
getImport(id: string): Promise<DatasetImport>
|
|
178
|
+
watchImport(id: string, options?: { onStatus?: (datasetImport: DatasetImport) => void; onProgress?: (progress: DatasetImportProgress, datasetImport: DatasetImport) => void; onVersion?: (version: DatasetVersion, dataset: Dataset) => void; signal?: AbortSignal; pollIntervalMs?: number; settleTimeoutMs?: number }): Promise<DatasetImport>
|
|
179
|
+
listImports(options?: { status?: DatasetImportStatus; dataset?: string; limit?: number; cursor?: string }): DatasetImportList
|
|
180
|
+
```
|
|
181
|
+
|
|
182
|
+
Read one publish, follow it until the version is READY or FAILED, or list your publishes. `watchImport` polls every `pollIntervalMs` (default 2000): `onStatus` fires on every status change, `onProgress` on every change of the five-phase `progress` (`extracting`, `parsing`, `building`, `copying`, `verifying`), `onVersion` on the version's state. `settleTimeoutMs` (default 30 minutes) bounds the wait; past it the watch throws `ImportSettleError` with the last observed state. `listImports` filters by `status` and `dataset`.
|
|
183
|
+
|
|
184
|
+
```ts
|
|
185
|
+
const done = await datasets().watchImport(imp.id);
|
|
186
|
+
```
|
|
187
|
+
</Tab>
|
|
188
|
+
<Tab title="Python">
|
|
189
|
+
```python
|
|
190
|
+
async def get_import(id: str) -> DatasetImport
|
|
191
|
+
async def watch_import(
|
|
192
|
+
id: str,
|
|
193
|
+
*,
|
|
194
|
+
on_status=None,
|
|
195
|
+
on_progress=None,
|
|
196
|
+
on_version=None,
|
|
197
|
+
poll_interval_s: float = 2.0,
|
|
198
|
+
timeout_s: Optional[float] = None,
|
|
199
|
+
settle_timeout_s: float = ...,
|
|
200
|
+
) -> DatasetImport
|
|
201
|
+
def list_imports(
|
|
202
|
+
*,
|
|
203
|
+
status: Optional[str] = None,
|
|
204
|
+
dataset: Optional[str] = None,
|
|
205
|
+
limit: Optional[int] = None,
|
|
206
|
+
cursor: Optional[str] = None,
|
|
207
|
+
)
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
Read one publish, follow it until the version is READY or FAILED, or list your publishes. `watch_import` polls every `poll_interval_s`: `on_status` fires on every status change, `on_progress` on every change of the five-phase `progress` (`extracting`, `parsing`, `building`, `copying`, `verifying`), `on_version` on the version's state. `settle_timeout_s` (default 30 minutes) bounds the wait; past it the watch raises `ImportSettleError` with the last observed state. `list_imports` filters by `status` and `dataset`.
|
|
211
|
+
|
|
212
|
+
```python
|
|
213
|
+
done = await datasets().watch_import(imp.id)
|
|
214
|
+
```
|
|
215
|
+
</Tab>
|
|
216
|
+
</Tabs>
|
|
217
|
+
|
|
218
|
+
## download
|
|
219
|
+
|
|
220
|
+
<Tabs>
|
|
221
|
+
<Tab title="TypeScript">
|
|
222
|
+
```ts
|
|
223
|
+
download(ref: string): Promise<Buffer>
|
|
224
|
+
download(
|
|
225
|
+
ref: string,
|
|
226
|
+
options: { to: string },
|
|
227
|
+
): Promise<string>
|
|
228
|
+
download(
|
|
229
|
+
ref: string,
|
|
230
|
+
options: { stream: true },
|
|
231
|
+
): Promise<ReadableStream<Uint8Array>>
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
The original package of a dataset you own: the bytes, the saved file's path, or a stream. The Buffer and `{ to }` shapes verify the download against the digest the server states and throw `EvolveDigestMismatchError` on a mismatch; `{ stream: true }` is unverified. A version published before packages were retained answers `package_not_retained`. Prefer `{ to }` for anything sizeable.
|
|
235
|
+
|
|
236
|
+
```ts
|
|
237
|
+
const path = await datasets().download(
|
|
238
|
+
"my-swe@1.0",
|
|
239
|
+
{ to: "./corpora" },
|
|
240
|
+
);
|
|
241
|
+
```
|
|
242
|
+
</Tab>
|
|
243
|
+
<Tab title="Python">
|
|
244
|
+
```python
|
|
245
|
+
async def download(
|
|
246
|
+
ref: str,
|
|
247
|
+
*,
|
|
248
|
+
to: Optional[str] = None,
|
|
249
|
+
)
|
|
250
|
+
```
|
|
251
|
+
|
|
252
|
+
The original package of a dataset you own: the bytes, or with `to` the saved file's path. Both shapes verify the download against the digest the server states, raising `EvolveDigestMismatchError` on a mismatch. A version published before packages were retained answers `package_not_retained`. Prefer `to` for anything sizeable.
|
|
253
|
+
|
|
254
|
+
```python
|
|
255
|
+
path = await datasets().download(
|
|
256
|
+
"my-swe@1.0",
|
|
257
|
+
to="./corpora",
|
|
258
|
+
)
|
|
259
|
+
```
|
|
260
|
+
</Tab>
|
|
261
|
+
</Tabs>
|
|
262
|
+
|
|
263
|
+
## activate, update, delete
|
|
264
|
+
|
|
265
|
+
<Tabs>
|
|
266
|
+
<Tab title="TypeScript">
|
|
267
|
+
```ts
|
|
268
|
+
activate(
|
|
269
|
+
name: string,
|
|
270
|
+
version: string,
|
|
271
|
+
): Promise<Dataset>
|
|
272
|
+
update(
|
|
273
|
+
name: string,
|
|
274
|
+
patch: { upstream_auto_import: boolean },
|
|
275
|
+
): Promise<Dataset>
|
|
276
|
+
delete(name: string): Promise<void>
|
|
277
|
+
```
|
|
278
|
+
|
|
279
|
+
Make a READY version the active one (`version_not_ready` while building, `version_not_activatable` for FAILED or ARCHIVED), switch automatic import of new upstream versions (`upstream_not_watchable` without a moving git ref), or delete a dataset you own (`dataset_not_owned` on a platform dataset, `dataset_in_use` naming the jobs that reference it).
|
|
280
|
+
|
|
281
|
+
```ts
|
|
282
|
+
await datasets().activate("my-swe", "1.0");
|
|
283
|
+
```
|
|
284
|
+
</Tab>
|
|
285
|
+
<Tab title="Python">
|
|
286
|
+
```python
|
|
287
|
+
async def activate(
|
|
288
|
+
name: str,
|
|
289
|
+
version: str,
|
|
290
|
+
) -> Dataset
|
|
291
|
+
async def update(
|
|
292
|
+
name: str,
|
|
293
|
+
*,
|
|
294
|
+
upstream_auto_import: bool,
|
|
295
|
+
) -> Dataset
|
|
296
|
+
async def delete(name: str) -> None
|
|
297
|
+
```
|
|
298
|
+
|
|
299
|
+
Make a READY version the active one (`version_not_ready` while building, `version_not_activatable` for FAILED or ARCHIVED), switch automatic import of new upstream versions (`upstream_not_watchable` without a moving git ref), or delete a dataset you own (`dataset_not_owned` on a platform dataset, `dataset_in_use` naming the jobs that reference it).
|
|
300
|
+
|
|
301
|
+
```python
|
|
302
|
+
await datasets().activate("my-swe", "1.0")
|
|
303
|
+
```
|
|
304
|
+
</Tab>
|
|
305
|
+
</Tabs>
|
|
@@ -0,0 +1,199 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: "Errors"
|
|
3
|
+
description: "The error class every refused request raises, and every code it can carry."
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
A refused request raises one class. Branch on its `code`, never on the message.
|
|
7
|
+
|
|
8
|
+
<Tabs>
|
|
9
|
+
<Tab title="TypeScript">
|
|
10
|
+
```ts
|
|
11
|
+
import { EvolveApiError, HOSTED_ERROR_CODES, isHostedErrorCode } from "@evolvingmachines/evolve";
|
|
12
|
+
|
|
13
|
+
class EvolveApiError extends Error {
|
|
14
|
+
status: number; // the HTTP status
|
|
15
|
+
code: HostedErrorCode | string; // the stable code below; "unknown_error" when the server sent none, or one a newer server added
|
|
16
|
+
message: string; // the server's sentence; may be shortened when a list is long
|
|
17
|
+
param?: string; // the input the refusal is about: a body path, a query parameter, or a multipart part
|
|
18
|
+
details?: Record<string, unknown>; // the complete machine-readable data; never shortened
|
|
19
|
+
retryAfterSec?: number; // on 429 and 503, from the body first, the Retry-After header second
|
|
20
|
+
requestId?: string; // the server's id for this failure, from x-request-id; quote it in support
|
|
21
|
+
isKnownCode(): boolean;
|
|
22
|
+
}
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
```ts
|
|
26
|
+
try {
|
|
27
|
+
await jobs().start({ datasets: [{ name: "terminal-bench-4" }], agents: [{ name: "codex", model_name: "gpt-5.5" }], sandbox_provider: "modal" });
|
|
28
|
+
} catch (err) {
|
|
29
|
+
if (err instanceof EvolveApiError && err.code === "provider_unsupported") {
|
|
30
|
+
const { refused_tasks } = err.details as { refused_tasks: { task_name: string; reason: string }[] };
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
`HOSTED_ERROR_CODES` is the closed list as a runtime array and `isHostedErrorCode(value)` narrows a string to it. The same list is published as `error_codes` in the [capability document](/sdk-reference/meta).
|
|
36
|
+
</Tab>
|
|
37
|
+
<Tab title="Python">
|
|
38
|
+
```python
|
|
39
|
+
from evolve import EvolveAPIError, HOSTED_ERROR_CODES, is_hosted_error_code
|
|
40
|
+
|
|
41
|
+
class EvolveAPIError(Exception):
|
|
42
|
+
status: int # the HTTP status
|
|
43
|
+
code: str # the stable code below; "unknown_error" when the server sent none
|
|
44
|
+
param: Optional[str] # the input the refusal is about
|
|
45
|
+
details: Optional[Dict[str, Any]] # the complete machine-readable data; never shortened
|
|
46
|
+
retry_after_sec: Optional[float] # on 429 and 503
|
|
47
|
+
request_id: Optional[str] # the server's id for this failure; quote it in support
|
|
48
|
+
def is_known_code(self) -> bool: ...
|
|
49
|
+
```
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
try:
|
|
53
|
+
await jobs().start(datasets=[{"name": "terminal-bench-4"}], agents=[{"name": "codex", "model_name": "gpt-5.5"}], sandbox_provider="modal")
|
|
54
|
+
except EvolveAPIError as err:
|
|
55
|
+
if err.code == "provider_unsupported":
|
|
56
|
+
refused = (err.details or {}).get("refused_tasks", [])
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
`str(err)` is the server's sentence, which may be shortened when a list is long. `HOSTED_ERROR_CODES` is the closed list and `is_hosted_error_code(value)` tests a string against it.
|
|
60
|
+
</Tab>
|
|
61
|
+
</Tabs>
|
|
62
|
+
|
|
63
|
+
Every response, success or failure, carries the same id in its `x-request-id` header. Of the 429 codes only `rate_limited` carries a retry delay; `quota_exceeded` and every `too_many_concurrent_*` code carry none.
|
|
64
|
+
|
|
65
|
+
## Errors that are not API refusals
|
|
66
|
+
|
|
67
|
+
These mean the request succeeded, or never happened, and something else went wrong. They are separate classes, not codes.
|
|
68
|
+
|
|
69
|
+
- `NoActiveVersionError`: `datasets().getActive()` (Python `get_active`) on a dataset with no active version. Carries `dataset`.
|
|
70
|
+
- `ImportSettleError`: `watchImport()` (Python `watch_import`) reached its settle timeout before the version settled. `code` is `settle_timeout`; carries `importId` (Python `import_id`), `dataset`, `version` and the last observed `state`. When `state` is `FAILED` the version did settle; read the failure with `getImport()` (Python `get_import`).
|
|
71
|
+
- `EvolveDigestMismatchError`: a download's bytes do not match the digest the server stated. Carries `expected` and `actual`. Do not trust the stored object.
|
|
72
|
+
- `EvolveIncompleteDownloadError`: fewer bytes arrived than `Content-Length` promised. Carries `expectedBytes` and `receivedBytes` (Python `expected_bytes`, `received_bytes`). Retry the download.
|
|
73
|
+
- `EvolveUploadTimeoutError`, TypeScript only: an upload saw no socket activity for 600 seconds. Retry the upload.
|
|
74
|
+
|
|
75
|
+
## The codes
|
|
76
|
+
|
|
77
|
+
Grouped by the door that returns them. The HTTP status is stated where it is fixed.
|
|
78
|
+
|
|
79
|
+
### Keys and limits
|
|
80
|
+
|
|
81
|
+
| Code | When |
|
|
82
|
+
| --- | --- |
|
|
83
|
+
| `missing_authorization` | No API key on the request. |
|
|
84
|
+
| `invalid_api_key` | The key is not valid. |
|
|
85
|
+
| `read_only_key` | 403. A read-only key on a mutating route. |
|
|
86
|
+
| `credential_service_unavailable` | The key could not be checked; retry. |
|
|
87
|
+
| `rate_limited` | 429. Wait `retryAfterSec`; every SDK watch loop does so on its own. |
|
|
88
|
+
| `insufficient_credits` | 402. The account's credit balance is zero at job create, resume or retry. |
|
|
89
|
+
| `quota_exceeded` | 429, no retry delay. The organization's `max_queued_trials` would be crossed. `details` carry `quota`, `limit`, `used`, `requested` and `org`. The CLI prints `Launch quota exceeded:` and exits 2. |
|
|
90
|
+
|
|
91
|
+
### Input
|
|
92
|
+
|
|
93
|
+
| Code | When |
|
|
94
|
+
| --- | --- |
|
|
95
|
+
| `invalid_json` | The body is not JSON. |
|
|
96
|
+
| `invalid_input` | A field is malformed; the message names it and `param` names it when the server can. Also the 413 for a trial home too large to serve as one text view (`details`: `size_bytes`, `max_unranged_bytes`). |
|
|
97
|
+
| `invalid_limit`, `invalid_cursor`, `invalid_after` | A paging parameter is malformed. |
|
|
98
|
+
| `invalid_status`, `invalid_visibility`, `invalid_format` | A filter or selector names a value outside its closed set. |
|
|
99
|
+
| `invalid_ids` | 400. A stop batch that is empty or over 100 ids. |
|
|
100
|
+
| `invalid_multipart` | 400. An upload that is not `multipart/form-data`, or is malformed. |
|
|
101
|
+
| `idempotency_key_reused` | 409. The key already stands for a different request. |
|
|
102
|
+
| `concurrent_update` | The row changed under the request; read it again. |
|
|
103
|
+
|
|
104
|
+
### Datasets and publishing
|
|
105
|
+
|
|
106
|
+
| Code | When |
|
|
107
|
+
| --- | --- |
|
|
108
|
+
| `dataset_not_found` | The name does not exist, or belongs to someone else; the two are never told apart. |
|
|
109
|
+
| `dataset_version_not_found` | The version does not exist. |
|
|
110
|
+
| `dataset_name_taken` | 409. The name belongs to another publisher. |
|
|
111
|
+
| `dataset_in_use` | 409. Jobs reference the dataset; `details` name them. |
|
|
112
|
+
| `dataset_not_owned` | The caller does not own it; a platform dataset has no owner. |
|
|
113
|
+
| `dataset_import_in_progress` | 409. A visibility change while a version is importing; `details` name the version. |
|
|
114
|
+
| `upstream_not_watchable` | Auto-import on a dataset with no moving git ref. |
|
|
115
|
+
| `no_active_version` | A bare name on a dataset with no active version. |
|
|
116
|
+
| `version_not_ready` | 409. A job or an activation on a version that is not READY. |
|
|
117
|
+
| `version_not_activatable` | A FAILED or ARCHIVED version. |
|
|
118
|
+
| `unknown_task_names`, `no_tasks` | A selector names tasks the version lacks, or filters every task away. |
|
|
119
|
+
| `task_not_found` | 404. `getTaskBuild()` on a name with no recorded outcome. |
|
|
120
|
+
| `task_failed_to_build` | A job create that names a task whose build FAILED; `details.failed_tasks` quote each reason. |
|
|
121
|
+
| `unpinned_git_ref` | A git publish on a branch; `details.commit` is the sha to pin. |
|
|
122
|
+
| `hub_package_not_found` | 400. The hub package does not exist or is private; `details.hub_package`. |
|
|
123
|
+
| `hub_unreachable` | 502. The hub could not be asked; retry the publish. |
|
|
124
|
+
| `invalid_archive` | The tarball is unreadable or unsafe. |
|
|
125
|
+
| `import_too_large` | 413. A dataset archive over the published ceiling. |
|
|
126
|
+
| `import_not_found` | The import id does not exist or is not yours. |
|
|
127
|
+
| `too_many_concurrent_imports` | 429, no retry delay. Too many uploads in flight; retry when one finishes. `details.max_concurrent`. |
|
|
128
|
+
| `package_not_retained`, `package_corrupt`, `package_missing` | 409. The original package was never retained, is corrupt, or is gone. Re-publish. |
|
|
129
|
+
| `too_many_concurrent_package_downloads` | 429, no retry delay. `details.max_concurrent`. |
|
|
130
|
+
| `upload_session_not_found`, `upload_offset_mismatch`, `upload_chunk_digest_mismatch`, `upload_incomplete`, `upload_archive_digest_mismatch`, `upload_session_failed`, `too_many_concurrent_upload_chunks` | A large upload's session failed; the SDK surfaces the refusal as is. Retry the publish or upload. |
|
|
131
|
+
|
|
132
|
+
### Jobs, trials, regrades
|
|
133
|
+
|
|
134
|
+
| Code | When |
|
|
135
|
+
| --- | --- |
|
|
136
|
+
| `provider_unsupported` | 422. A selected task is refused on the chosen provider; `details.refused_tasks` carry each reason. A GPU task that moves to `modal` is not refused; the trial records the move as `sandbox_provider_degrade`. |
|
|
137
|
+
| `job_not_found` | The job does not exist or is not visible to you. A regrade job id on delete. |
|
|
138
|
+
| `job_not_terminal` | 409. Resume, retry, regrade, analyze or delete on a live job; on delete also a live regrade, listed in `details.regrade_job_ids`. |
|
|
139
|
+
| `no_failed_trials` | 409. Resume, or retry with `failed_only`, found nothing. |
|
|
140
|
+
| `trial_not_found` | 404. A retry names a trial the job does not own; a trace feed id no run owns. |
|
|
141
|
+
| `trial_not_settled` | 409. A retry names a trial still running. |
|
|
142
|
+
| `regrade_source_ineligible` | 409. The trial recorded no verifier inputs; the message says why. |
|
|
143
|
+
| `no_regradable_trials` | 409. A whole-job regrade found nothing eligible. |
|
|
144
|
+
| `org_forbidden` | 403. A member who did not create the job tried to delete it. |
|
|
145
|
+
| `job_uploaded` | 409. Resume, retry or regrade on an uploaded job. |
|
|
146
|
+
| `secret_not_found`, `secret_ambiguous`, `secret_brokered_unsupported`, `secret_exists`, `secret_not_attached` | See [secrets](/core-concepts/secrets). |
|
|
147
|
+
| `agent_version_not_found` | A version pin that does not resolve. |
|
|
148
|
+
| `agent_version_unresolvable` | 502. The latest version could not be looked up; retry the create. |
|
|
149
|
+
| `agent_kwarg_unsupported`, `agent_config_unsupported`, `agent_config_key_refused`, `agent_preset_unsupported` | See [agents](/core-concepts/agents). |
|
|
150
|
+
|
|
151
|
+
### Analyze and check
|
|
152
|
+
|
|
153
|
+
| Code | When |
|
|
154
|
+
| --- | --- |
|
|
155
|
+
| `invalid_rubric` | 400. Unknown keys, empty or duplicate criteria, or a length bound exceeded. |
|
|
156
|
+
| `analysis_already_running` | 409. One analysis wave at a time; also blocks delete. |
|
|
157
|
+
| `no_analyzable_trials` | 409. Every trial is CANCELLED, or the filter selects nothing. |
|
|
158
|
+
| `analysis_not_found` | 404. An analysis you cannot read or that never existed. |
|
|
159
|
+
| `analysis_not_terminal` | 409. Download of an analysis still queued or running. |
|
|
160
|
+
| `check_not_found` | 404. Neither a check nor a task check you can read. |
|
|
161
|
+
| `check_not_terminal` | 409. Download of a check or task check not yet settled. |
|
|
162
|
+
| `no_checkable_tasks` | 400. No task directory in the archive, or the globs and the cap left nothing. |
|
|
163
|
+
| `too_many_concurrent_check_uploads` | 429, no retry delay. `details.max_concurrent`. |
|
|
164
|
+
|
|
165
|
+
### Job upload
|
|
166
|
+
|
|
167
|
+
The upload door itself answers four codes: `invalid_multipart` (400), `invalid_input` (400), `upload_too_large` (413) and `too_many_concurrent_job_uploads` (429). Everything else arrives on the import's `failure`, with no HTTP status.
|
|
168
|
+
|
|
169
|
+
| Code | When |
|
|
170
|
+
| --- | --- |
|
|
171
|
+
| `not_a_job_dir` | No `result.json` and `config.json` at the root, or they do not parse. |
|
|
172
|
+
| `invalid_archive` | The tarball is unreadable or unsafe. |
|
|
173
|
+
| `invalid_trial` | One trial's `result.json` cannot be ingested; the failure names the trial. |
|
|
174
|
+
| `trial_too_large` | Not a failure: the trial is skipped and listed on `skipped_trials`; `details` carry `file`, `bytes`, `max_bytes`. |
|
|
175
|
+
| `upload_too_large` | The archive is over the published ceiling, at the door or when extracted. |
|
|
176
|
+
| `job_already_uploaded` | You already uploaded this archive's job; `details.existing_job_id`. Delete it to replace it. |
|
|
177
|
+
| `too_many_concurrent_job_uploads` | 429, no retry delay. `details.max_concurrent`. |
|
|
178
|
+
| `job_import_not_found` | 404. An import that is not yours or does not exist. |
|
|
179
|
+
|
|
180
|
+
### Registered agents and skills
|
|
181
|
+
|
|
182
|
+
| Code | When |
|
|
183
|
+
| --- | --- |
|
|
184
|
+
| `agent_not_found` | Another owner's name reads the same as none. |
|
|
185
|
+
| `agent_name_taken`, `agent_name_reserved`, `agent_invalid_name` | The name is used, collides with a built-in, or breaks the published pattern. |
|
|
186
|
+
| `agent_source_required`, `agent_source_conflict` | Neither an install script nor a directory, or both. |
|
|
187
|
+
| `agent_invalid_env` | The declared env overrides a run-contract key, or looks like a credential. |
|
|
188
|
+
| `agent_too_large`, `agent_limit_reached` | The tarball or the per-user count is over the published limit. |
|
|
189
|
+
| `skill_not_found`, `skill_name_not_found` | An `upload:<id>` or a `name:<skill-name>` that does not resolve. |
|
|
190
|
+
| `skill_ref_invalid`, `skill_unresolvable`, `skill_invalid` | A reference that does not parse, cannot be fetched, or holds no valid skill. |
|
|
191
|
+
| `skill_in_use` | 409. A running job references the upload. |
|
|
192
|
+
| `skill_too_large`, `skill_limit_reached` | The tarball or the per-user count is over the published limit. |
|
|
193
|
+
| `too_many_concurrent_skill_uploads` | 429, no retry delay. `details.max_concurrent`. |
|
|
194
|
+
|
|
195
|
+
### Organizations
|
|
196
|
+
|
|
197
|
+
`org_not_found`, `org_slug_taken`, `org_forbidden`, `org_personal_immutable`, `org_last_owner`, `org_in_use`, `org_member_not_found`, `invite_not_found` and `invite_invalid` belong to team accounts. The SDK reads organizations only, so only `org_not_found` and `org_forbidden` can reach it today.
|
|
198
|
+
|
|
199
|
+
`internal_error` is the server's own failure; quote `requestId`.
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
---
|
|
2
|
+
title: "Clients"
|
|
3
|
+
description: "How the hosted clients are built and configured."
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
One client per noun, each built by a factory. Every factory reads `EVOLVE_API_KEY` from the environment unless a config names the key. Create a key on the dashboard's [API keys page](https://dashboard.evolvingmachines.ai/api-keys).
|
|
7
|
+
|
|
8
|
+
<Tabs>
|
|
9
|
+
<Tab title="TypeScript">
|
|
10
|
+
```ts
|
|
11
|
+
import { jobs, trials, datasets, analyses, checks, skills, agents, auth, orgs, hosted, meta } from "@evolvingmachines/evolve";
|
|
12
|
+
|
|
13
|
+
interface HostedClientConfig {
|
|
14
|
+
apiKey?: string; // default: process.env.EVOLVE_API_KEY
|
|
15
|
+
baseUrl?: string; // default: the Evolve dashboard API
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function jobs(config?: HostedClientConfig): JobsClient
|
|
19
|
+
function hosted(config?: HostedClientConfig): HostedEvolve
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
`datasets()`, `trials()`, `analyses()`, `checks()`, `skills()`, `agents()`, `auth()` and `orgs()` take the same config. `hosted()` builds every client from one config and exposes them as properties: `datasets`, `agents`, `jobs`, `skills`, `trials`, `analyses`, `checks`, `orgs`, plus `meta()`.
|
|
23
|
+
|
|
24
|
+
```ts
|
|
25
|
+
const evolve = hosted({ apiKey: process.env.EVOLVE_API_KEY });
|
|
26
|
+
const job = await evolve.jobs.get("3e1f9a2c-…");
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Every list method returns a handle you either await for one page (`{ items, nextCursor, hasMore }`) or iterate with `for await` to walk every page.
|
|
30
|
+
</Tab>
|
|
31
|
+
<Tab title="Python">
|
|
32
|
+
```python
|
|
33
|
+
from evolve import jobs, trials, datasets, analyses, checks, skills, agents, auth, orgs, hosted, meta
|
|
34
|
+
from evolve import HostedClientConfig
|
|
35
|
+
|
|
36
|
+
@dataclass
|
|
37
|
+
class HostedClientConfig:
|
|
38
|
+
api_key: Optional[str] = None # default: EVOLVE_API_KEY
|
|
39
|
+
base_url: Optional[str] = None # default: the Evolve dashboard API
|
|
40
|
+
|
|
41
|
+
def jobs(config: Optional[HostedClientConfig] = None) -> JobsClient
|
|
42
|
+
def hosted(config: Optional[HostedClientConfig] = None) -> HostedEvolve
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
`datasets()`, `trials()`, `analyses()`, `checks()`, `skills()`, `agents()`, `auth()` and `orgs()` take the same config. `hosted()` builds every client from one config and exposes them as properties: `datasets`, `agents`, `jobs`, `skills`, `trials`, `analyses`, `checks`, `orgs`, plus `await meta()`.
|
|
46
|
+
|
|
47
|
+
```python
|
|
48
|
+
evolve = hosted(HostedClientConfig(api_key=os.environ["EVOLVE_API_KEY"]))
|
|
49
|
+
job = await evolve.jobs.get("3e1f9a2c-…")
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Every method is a coroutine. Each client is an async context manager and has `close()`. Every list method returns a handle you either await for one page (`items`, `next_cursor`, `has_more`) or iterate with `async for` to walk every page.
|
|
53
|
+
</Tab>
|
|
54
|
+
</Tabs>
|
|
55
|
+
|
|
56
|
+
The config has no timeout setting. The clients behind `hosted()` are built on first access, so `meta()` works before a key is set. Neither client follows a redirect with your key: Python raises the 3xx, TypeScript drops the `Authorization` header on a cross-origin redirect.
|
|
57
|
+
|
|
58
|
+
Every object the clients return is catalogued on [types](/sdk-reference/types), and every refusal on [errors](/sdk-reference/errors).
|