@evolvingmachines/evolve 0.0.55-project-sable.20260916.db0bc0bb

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. package/LICENSE +201 -0
  2. package/README.md +25 -0
  3. package/dist/chunk-5GVMBEYI.js +11 -0
  4. package/dist/chunk-HQJI3HHU.js +1 -0
  5. package/dist/chunk-JS2UTK2I.js +427 -0
  6. package/dist/cli/index.cjs +73 -0
  7. package/dist/cli/index.d.cts +270 -0
  8. package/dist/cli/index.d.ts +270 -0
  9. package/dist/cli/index.js +54 -0
  10. package/dist/index.cjs +669 -0
  11. package/dist/index.d.cts +5023 -0
  12. package/dist/index.d.ts +5023 -0
  13. package/dist/index.js +230 -0
  14. package/dist/managed-modal-IJW62SSK.js +4 -0
  15. package/dist/resumable-HFSICSH3.js +1 -0
  16. package/dist/tar-GVC4KE52.js +1 -0
  17. package/dist/types-DlpTxdR_.d.cts +4777 -0
  18. package/dist/types-DlpTxdR_.d.ts +4777 -0
  19. package/dist/upload-KPYDMWFJ.js +1 -0
  20. package/docs-agents/SKILL.md +138 -0
  21. package/docs-agents/index.md +81 -0
  22. package/docs-agents/python/01-getting-started.md +511 -0
  23. package/docs-agents/python/02-configuration.md +855 -0
  24. package/docs-agents/python/03-runtime.md +899 -0
  25. package/docs-agents/python/04-streaming.md +451 -0
  26. package/docs-agents/python/05-swarm-pipeline.md +953 -0
  27. package/docs-agents/python/index.md +122 -0
  28. package/docs-agents/typescript/01-getting-started.md +494 -0
  29. package/docs-agents/typescript/02-configuration.md +841 -0
  30. package/docs-agents/typescript/03-runtime.md +864 -0
  31. package/docs-agents/typescript/04-streaming.md +478 -0
  32. package/docs-agents/typescript/05-swarm-pipeline.md +957 -0
  33. package/docs-agents/typescript/index.md +121 -0
  34. package/docs-evals/SKILL.md +112 -0
  35. package/docs-evals/cli-reference/agent.mdx +67 -0
  36. package/docs-evals/cli-reference/analysis.mdx +91 -0
  37. package/docs-evals/cli-reference/analyze.mdx +74 -0
  38. package/docs-evals/cli-reference/auth.mdx +60 -0
  39. package/docs-evals/cli-reference/check.mdx +146 -0
  40. package/docs-evals/cli-reference/dataset.mdx +133 -0
  41. package/docs-evals/cli-reference/index.mdx +108 -0
  42. package/docs-evals/cli-reference/job.mdx +239 -0
  43. package/docs-evals/cli-reference/run.mdx +196 -0
  44. package/docs-evals/cli-reference/secrets.mdx +95 -0
  45. package/docs-evals/cli-reference/skill.mdx +51 -0
  46. package/docs-evals/cli-reference/skills.mdx +81 -0
  47. package/docs-evals/cli-reference/trial.mdx +112 -0
  48. package/docs-evals/cli-reference/upload.mdx +36 -0
  49. package/docs-evals/core-concepts/agents.mdx +106 -0
  50. package/docs-evals/core-concepts/analyze.mdx +86 -0
  51. package/docs-evals/core-concepts/check.mdx +51 -0
  52. package/docs-evals/core-concepts/datasets.mdx +138 -0
  53. package/docs-evals/core-concepts/index.mdx +42 -0
  54. package/docs-evals/core-concepts/jobs.mdx +136 -0
  55. package/docs-evals/core-concepts/models.mdx +68 -0
  56. package/docs-evals/core-concepts/sandboxes.mdx +52 -0
  57. package/docs-evals/core-concepts/secrets.mdx +41 -0
  58. package/docs-evals/core-concepts/skills.mdx +42 -0
  59. package/docs-evals/core-concepts/tasks.mdx +216 -0
  60. package/docs-evals/core-concepts/trial-outputs.mdx +103 -0
  61. package/docs-evals/core-concepts/trials.mdx +103 -0
  62. package/docs-evals/core-concepts/upload.mdx +45 -0
  63. package/docs-evals/dashboard/checks.mdx +15 -0
  64. package/docs-evals/dashboard/datasets.mdx +24 -0
  65. package/docs-evals/dashboard/jobs.mdx +65 -0
  66. package/docs-evals/dashboard/traces.mdx +15 -0
  67. package/docs-evals/dashboard/trial-viewer.mdx +30 -0
  68. package/docs-evals/docs.json +180 -0
  69. package/docs-evals/getting-started/installation.mdx +56 -0
  70. package/docs-evals/getting-started/quick-start.mdx +85 -0
  71. package/docs-evals/index.mdx +36 -0
  72. package/docs-evals/sdk/python.mdx +139 -0
  73. package/docs-evals/sdk/typescript.mdx +141 -0
  74. package/docs-evals/sdk-reference/agents.mdx +105 -0
  75. package/docs-evals/sdk-reference/analyses.mdx +125 -0
  76. package/docs-evals/sdk-reference/auth.mdx +76 -0
  77. package/docs-evals/sdk-reference/checks.mdx +166 -0
  78. package/docs-evals/sdk-reference/datasets.mdx +305 -0
  79. package/docs-evals/sdk-reference/errors.mdx +199 -0
  80. package/docs-evals/sdk-reference/index.mdx +58 -0
  81. package/docs-evals/sdk-reference/jobs.mdx +731 -0
  82. package/docs-evals/sdk-reference/meta.mdx +52 -0
  83. package/docs-evals/sdk-reference/skills.mdx +68 -0
  84. package/docs-evals/sdk-reference/trials.mdx +232 -0
  85. package/docs-evals/sdk-reference/types.mdx +272 -0
  86. package/docs-evals/snippets/global-options.mdx +13 -0
  87. package/docs-evals/snippets/list-flags.mdx +23 -0
  88. package/harness-capabilities.json +430 -0
  89. package/hosted-error-codes.json +120 -0
  90. package/package.json +163 -0
  91. package/skills/create-adapter/SKILL.md +287 -0
  92. package/skills/create-task/SKILL.md +491 -0
  93. package/skills/evolve/SKILL.md +39 -0
  94. package/skills/publish/SKILL.md +142 -0
  95. package/skills/rewardkit/SKILL.md +330 -0
  96. package/spec/atif/README.md +45 -0
  97. package/spec/atif/trajectory.schema.json +865 -0
  98. package/spec/openapi.yaml +13447 -0
@@ -0,0 +1,133 @@
1
+ ---
2
+ title: "evolve dataset"
3
+ description: "Browse and publish the dataset catalog."
4
+ ---
5
+
6
+ import GlobalOptions from '/snippets/global-options.mdx';
7
+ import ListFlags from '/snippets/list-flags.mdx';
8
+
9
+ ```bash
10
+ evolve dataset <command> [options]
11
+ ```
12
+
13
+ ## dataset list
14
+
15
+ ```bash
16
+ evolve dataset list [options]
17
+ ```
18
+
19
+ List the dataset catalog.
20
+
21
+ <ListFlags />
22
+
23
+ <ParamField path="--search" type="text">
24
+ Free-text filter over name and description.
25
+ </ParamField>
26
+
27
+ ## dataset show
28
+
29
+ ```bash
30
+ evolve dataset show <name[@version]> [options]
31
+ ```
32
+
33
+ Show one dataset: its versions, its manifest identity, its tasks with a NOTES column, the providers each task can run on, and the failed tasks of the shown version with their reasons. One line reports a git dataset whose upstream ref has moved; it never appears in `--json`, where the same fact is the `upstream` field.
34
+
35
+ <ParamField path="-l, --limit" type="n">
36
+ Task-list page size.
37
+ </ParamField>
38
+
39
+ <ParamField path="--cursor" type="c">
40
+ Resume the task list from a cursor.
41
+ </ParamField>
42
+
43
+ ```bash
44
+ evolve dataset show terminal-bench-4@4.0
45
+ ```
46
+
47
+ ## dataset check
48
+
49
+ ```bash
50
+ evolve dataset check <dir>
51
+ ```
52
+
53
+ Pre-flight a local corpus. A dry run: nothing is uploaded, nothing is written. Prints a verdict per task and exits 1 on any refusal.
54
+
55
+ ## dataset publish
56
+
57
+ ```bash
58
+ evolve dataset publish [options]
59
+ ```
60
+
61
+ Publish a dataset version from a git source, a local directory, or a fetchable source: a public tarball URL, or a public hub package. A name owned by someone else is refused before the bytes move, and an archive over the published ceiling is refused with `import_too_large`.
62
+
63
+ A directory upload prints `upload <sent>/<total> (P%)` once per 10% of the archive; with `--json --watch` the same steps are `upload.progress` events (`sent_bytes`, `total_bytes`, `elapsed_sec`). A large upload prints `Registered import <id>` early and reads `QUEUED (receiving)` until the corpus has arrived. Once the build settles, `--watch` prints every task's outcome and ends with `built N of M tasks`.
64
+
65
+ <ParamField path="--git" type="url">
66
+ Git repository URL. With `--ref`.
67
+ </ParamField>
68
+
69
+ <ParamField path="--ref" type="ref">
70
+ Pinned git ref: a full 40-hex commit sha, or a tag, resolved to its commit at publish and verified at import. Branch names are refused as `unpinned_git_ref`. With `--git`.
71
+ </ParamField>
72
+
73
+ <ParamField path="--path" type="subfolder">
74
+ Repository subfolder holding the corpus. With `--git`; only that folder is imported.
75
+ </ParamField>
76
+
77
+ <ParamField path="--dir" type="path">
78
+ Local corpus directory, tarred and uploaded. Pre-flighted first; see `--skip-preflight`.
79
+ </ParamField>
80
+
81
+ <ParamField path="--from" type="url | hub:org/name[@ref]">
82
+ Fetchable source the server pulls itself, with no local bytes: a public https tarball URL, or `hub:org/name[@ref]`, a public Harbor hub package. The ref is the latest tag by default, a revision number, or `sha256:<digest>`; it is resolved and digest-pinned when the publish is accepted.
83
+ </ParamField>
84
+
85
+ <ParamField path="--name" type="dataset">
86
+ Catalog dataset name to create or extend. Optional with `--dir` when the corpus carries a `dataset.toml` manifest, and with `--from hub:…`, which defaults to the package's short name. Required with `--git` and `--from <url>`.
87
+ </ParamField>
88
+
89
+ <ParamField path="--version" type="v">
90
+ Version label for the published version. Optional with `--dir` when `dataset.toml` declares one, and with `--from hub:…`, which defaults to the resolved revision. Required with `--git` and `--from <url>`.
91
+ </ParamField>
92
+
93
+ <ParamField path="--watch">
94
+ Poll until the publish settles: the version `READY` (built and active) or `FAILED`.
95
+ </ParamField>
96
+
97
+ <ParamField path="--skip-preflight">
98
+ Upload without the pre-flight check. A refused task then lands `FAILED` at import instead of being caught here.
99
+ </ParamField>
100
+
101
+ ```bash
102
+ evolve dataset publish --from hub:cookbook/hello-world --watch
103
+ ```
104
+
105
+ ## dataset watch
106
+
107
+ ```bash
108
+ evolve dataset watch <name | import-id>
109
+ ```
110
+
111
+ Re-attach to a publish and follow it to `READY` or `FAILED`, the same follow `dataset publish --watch` renders. Works after the CLI exited, or from another machine. A name with no live publish is refused, naming the newest settled import. With `--json` the same events stream, opened with `import.attached`; the transfer's own `upload.progress` lines belong to the publishing process and never appear here.
112
+
113
+ ## dataset download
114
+
115
+ ```bash
116
+ evolve dataset download <name[@version]> [options]
117
+ ```
118
+
119
+ Download the original corpus package. Owner only.
120
+
121
+ <ParamField path="-o, --output-dir" type="dir">
122
+ Directory to save into. Defaults to the current directory.
123
+ </ParamField>
124
+
125
+ ## dataset activate
126
+
127
+ ```bash
128
+ evolve dataset activate <name> <version>
129
+ ```
130
+
131
+ Make a `READY` version the dataset's active version.
132
+
133
+ <GlobalOptions />
@@ -0,0 +1,108 @@
1
+ ---
2
+ title: "CLI reference"
3
+ description: "Every verb of the evolve command, from its own help output."
4
+ ---
5
+
6
+ ```bash
7
+ evolve <command> [options]
8
+ ```
9
+
10
+ The `evolve` command starts jobs and reads everything back. Every verb accepts `--json` for machine-readable output, and `-h` or `--help` prints its flags under short headings, with defaults, and one to three examples.
11
+
12
+ ## Verbs
13
+
14
+ | Verb | What it does |
15
+ | --- | --- |
16
+ | [`run`](/cli-reference/run) | Start a job. Add `--watch` to follow it. The short form of `job start`. |
17
+ | [`analyze`](/cli-reference/analyze) | Analyze a finished job's trial traces against a rubric. |
18
+ | [`check`](/cli-reference/check) | Check task quality against a rubric. |
19
+ | [`upload`](/cli-reference/upload) | Upload a job directory as a finished job. |
20
+
21
+ ## Command groups
22
+
23
+ | Group | Commands |
24
+ | --- | --- |
25
+ | [`skills`](/cli-reference/skills) | `list`, `get`, `path`, `install` |
26
+ | [`job`](/cli-reference/job) | `start`, `list`, `show`, `trials`, `tasks`, `compare`, `cancel`, `delete`, `stop`, `resume`, `retry`, `regrade`, `imports`, `import`, `download`, `grep` |
27
+ | [`trial`](/cli-reference/trial) | `show`, `trace`, `download`, `retry`, `regrade`, `stop` |
28
+ | [`analysis`](/cli-reference/analysis) | `list`, `show`, `trace`, `download` |
29
+ | [`check`](/cli-reference/check) | `list`, `show`, `trace`, `download` |
30
+ | [`dataset`](/cli-reference/dataset) | `list`, `show`, `check`, `publish`, `watch`, `download`, `activate` |
31
+ | [`skill`](/cli-reference/skill) | `list`, `upload`, `show`, `delete` |
32
+ | [`agent`](/cli-reference/agent) | `list`, `show`, `add`, `remove` |
33
+ | [`auth`](/cli-reference/auth) | `status`, `org list`, `org show` |
34
+ | [`secrets`](/cli-reference/secrets) | `set`, `list`, `delete` |
35
+
36
+ The singular noun is canonical; `jobs`, `trials`, `analyses` and `datasets` are hidden aliases, as are `secret` for `secrets` and `ls` for `list`. `skills` is not the plural of `skill`: it is the group that serves the bundled skills to your coding agent. `agents` is not an alias: it is reserved for the managed-agents CLI and refuses, pointing at `evolve agent`.
37
+
38
+ ## Global options
39
+
40
+ These work on every command.
41
+
42
+ <ParamField path="--json">
43
+ Machine-readable JSON output. A refusal then prints one `{"error": {...}}` object on stdout with the server's `code`, `message`, `param`, `details`, `retryAfterSec` and `request_id` when present.
44
+ </ParamField>
45
+
46
+ <ParamField path="--api-key" type="key">
47
+ API key. Defaults to `$EVOLVE_API_KEY`.
48
+ </ParamField>
49
+
50
+ <ParamField path="--base-url" type="url">
51
+ API base URL. Defaults to the Evolve dashboard API.
52
+ </ParamField>
53
+
54
+ <ParamField path="-v, --version">
55
+ Print the CLI version.
56
+ </ParamField>
57
+
58
+ ## Ids
59
+
60
+ Every verb that takes an id (`job`, `trial`, `analysis`, `check` with a check id or a task check id, `skill`) also takes an unambiguous prefix of at least 8 characters. A prefix that matches nothing or more than one row is refused naming the candidates. A full id is never prefix-matched, `name:<skill-name>` is a name, and a job import id is taken as printed.
61
+
62
+ A trial prefix is resolved against every trial of every job in scope, so on an account with many jobs the lookup can hit the rate limit; the full id, which `evolve job trials <job>` prints, always works.
63
+
64
+ ## Output
65
+
66
+ Human tables on a TTY, tab-separated rows when piped, `--json` for the machine shape (NDJSON under `--watch`), and `-q` for ids only. `--no-trunc` disables cell truncation and `--no-headers` drops the header row from piped output. A money cell carries its state: `$0.06` is final, `at least $0.06` is still being written, `-` means not measured yet.
67
+
68
+ `--columns` chooses and orders list columns, comma-separated; `--columns help` prints the keys. For `job list` they are `id`, `name`, `status`, `datasets`, `agents`, `trials`, `spent`, `started`; for `job trials` `task`, `agent`, `attempt`, `status`, `reward`, `spent`, `tokens`, `gpu`, `id`; for `analysis list` `id`, `status`, `task`, `job`, `trial`, `model`, `attempts`, `spent`, `created`, `finished`.
69
+
70
+ ## Listing flags
71
+
72
+ Every `list` command, and `job trials`, `job tasks` and `job imports`, share one set of paging and formatting flags. Two take only the formatting flags, not `-l` or `--cursor`: `auth org list` and `secrets list`.
73
+
74
+ <ParamField path="-l, --limit" type="n">
75
+ Page size.
76
+ </ParamField>
77
+
78
+ <ParamField path="--cursor" type="c">
79
+ Resume from a page cursor.
80
+ </ParamField>
81
+
82
+ <ParamField path="--columns" type="keys | all | help">
83
+ Choose and order columns, comma-separated. `help` lists the keys.
84
+ </ParamField>
85
+
86
+ <ParamField path="-q, --quiet">
87
+ Print only ids, one per line, for piping.
88
+ </ParamField>
89
+
90
+ <ParamField path="--no-trunc">
91
+ Full cell content instead of one-line truncation.
92
+ </ParamField>
93
+
94
+ <ParamField path="--no-headers">
95
+ Omit the header row in piped (TSV) output.
96
+ </ParamField>
97
+
98
+ ## Scope
99
+
100
+ `--scope` on `job list`, `analysis list` and `check list` is `my`, what you created and the default, or `shared`, your organizations' rows that teammates created. `all` is refused. Every id printed under either scope resolves on its `show`, `trace` and `download`; an id outside your organizations answers the noun's not-found code, exactly like an id that does not exist.
101
+
102
+ ## Values
103
+
104
+ A flag's value may begin with `-`, such as `-x '-*'` or a negative number. Only a token that spells another flag of the same command is refused, and the refusal shows the `--flag=value` form. A typo in `--stream`, `--status`, `--scope` or `run`'s `-e` is a usage error naming the legal values, before any request.
105
+
106
+ ## Exit codes
107
+
108
+ `0` on success (with `--watch`: the job `COMPLETED`, a publish settled `READY`, every analysis completed, no task check errored). `1` on a runtime or API failure (with `--watch`: `FAILED` or `CANCELLED`, a failed analysis, an errored task check). `2` on a usage error, and on a quota refusal, which prints `Launch quota exceeded:` with the server's sentence. A rate limit prints one line with the wait and exits 1; the SDK's watch loops wait it out on their own.
@@ -0,0 +1,239 @@
1
+ ---
2
+ title: "evolve job"
3
+ description: "Start, follow, and derive jobs."
4
+ ---
5
+
6
+ import GlobalOptions from '/snippets/global-options.mdx';
7
+ import ListFlags from '/snippets/list-flags.mdx';
8
+
9
+ ```bash
10
+ evolve job <command> [options]
11
+ ```
12
+
13
+ Every command below takes the [global options](/cli-reference#global-options).
14
+
15
+ ## job start
16
+
17
+ ```bash
18
+ evolve job start [options]
19
+ ```
20
+
21
+ Start a job; add `--watch` to follow it. Exactly the options of [`evolve run`](/cli-reference/run).
22
+
23
+ ## job list
24
+
25
+ ```bash
26
+ evolve job list [options]
27
+ ```
28
+
29
+ List jobs, newest first: yours, or `--scope shared` for your organizations'.
30
+
31
+ <ListFlags />
32
+
33
+ <ParamField path="--search" type="text">
34
+ Free-text filter over job name and dataset names.
35
+ </ParamField>
36
+
37
+ <ParamField path="--scope" type="my | shared">
38
+ Visibility scope: `my`, what you created (the default), or `shared`, your organizations' rows that teammates created.
39
+ </ParamField>
40
+
41
+ ```bash
42
+ evolve job list --limit 20 -q
43
+ ```
44
+
45
+ ## job show
46
+
47
+ ```bash
48
+ evolve job show <id> [id...]
49
+ ```
50
+
51
+ Show one or more jobs in full. It ends with a pass@k block, one line per arm and dataset that has numbers; `--json` carries the raw `stats.evals[].pass_at_k`. An `analyze` row holds the embedded policy and an `analysis` row the tally with the analyzer's spend. An uploaded job prints its provider as `ported` and its spend as `reported $X.XX (N/M trials reporting)`.
52
+
53
+ ## job trials
54
+
55
+ ```bash
56
+ evolve job trials <id> [options]
57
+ ```
58
+
59
+ List a job's trials.
60
+
61
+ <ListFlags />
62
+
63
+ <ParamField path="--status" type="s1,s2,...">
64
+ Filter by trial status, for example `INFRASTRUCTURE_ERROR`.
65
+ </ParamField>
66
+
67
+ <ParamField path="--dataset" type="name">
68
+ Filter to one dataset's trials.
69
+ </ParamField>
70
+
71
+ ```bash
72
+ evolve job trials 3e1f9a2c-… --status INFRASTRUCTURE_ERROR,SCORING_ERROR
73
+ ```
74
+
75
+ ## job tasks
76
+
77
+ ```bash
78
+ evolve job tasks <id> [options]
79
+ ```
80
+
81
+ Per-task rollup of a job: one row per task.
82
+
83
+ <ListFlags />
84
+
85
+ ## job compare
86
+
87
+ ```bash
88
+ evolve job compare <id> <id> [...]
89
+ ```
90
+
91
+ Compare two to ten jobs side by side.
92
+
93
+ ## job cancel
94
+
95
+ ```bash
96
+ evolve job cancel <id>
97
+ ```
98
+
99
+ Request cancellation of a job.
100
+
101
+ ## job delete
102
+
103
+ ```bash
104
+ evolve job delete <id> [options]
105
+ ```
106
+
107
+ Permanently delete a job you created: its trials, traces, analyses and stored files. It names the job, then asks. Another member of the organization is refused with `org_forbidden`; a live job, a running analysis wave or a live regrade refuses as on [jobs](/core-concepts/jobs#stop-cancel-delete).
108
+
109
+ <ParamField path="-y, --yes">
110
+ Delete without a confirmation prompt. On a non-interactive stdin the command refuses without it, rather than guessing. With `--json` the receipt is `{"job_id", "trials_deleted", "analyses_deleted"}`.
111
+ </ParamField>
112
+
113
+ ## job stop
114
+
115
+ ```bash
116
+ evolve job stop <id> --dataset <name>
117
+ ```
118
+
119
+ Stop one dataset's live trials without cancelling the job. Trials already settled are reported as already terminal, and a dataset the job never spanned is refused. If the command fails part way (a rate limit, say), the report prints first, marked `PARTIAL` with the trials no answer came back for (`partial: true` and `unreported` under `--json`), then the error, exit 1; re-running the same command finishes the rest.
120
+
121
+ <ParamField path="--dataset" type="name" required>
122
+ The dataset whose live trials to stop.
123
+ </ParamField>
124
+
125
+ ## job resume
126
+
127
+ ```bash
128
+ evolve job resume <id> [options]
129
+ ```
130
+
131
+ A new linked job over a terminal job's failed or stopped trials.
132
+
133
+ <ParamField path="-f, --filter-error-type" type="type">
134
+ Failure types to resume, matched on the trial's exception type. Repeatable. Defaults to the standard failure set plus stopped trials.
135
+ </ParamField>
136
+
137
+ ```bash
138
+ evolve job resume 3e1f9a2c-… -f InfrastructureError
139
+ ```
140
+
141
+ ## job retry
142
+
143
+ ```bash
144
+ evolve job retry <id> [options]
145
+ ```
146
+
147
+ A new linked job re-running selected trials: all of them, the failed ones only, or named ids.
148
+
149
+ <ParamField path="--failed-only">
150
+ Only retry failed trials: `SCORING_ERROR`, `INFRASTRUCTURE_ERROR`, `BUDGET`, `INDETERMINATE`. Stopped and scored trials are not failures.
151
+ </ParamField>
152
+
153
+ <ParamField path="-t, --trial" type="trial-id">
154
+ Retry exactly this trial. Repeatable, all or nothing: each must be settled, and the job may still be running. Not combinable with `--failed-only`.
155
+ </ParamField>
156
+
157
+ ## job regrade
158
+
159
+ ```bash
160
+ evolve job regrade <id> [options]
161
+ ```
162
+
163
+ Verifier-only re-run of a terminal job. The result is a job.
164
+
165
+ <ParamField path="--status" type="s1,s2,...">
166
+ Only regrade source trials in these statuses.
167
+ </ParamField>
168
+
169
+ <ParamField path="--task" type="name">
170
+ Only regrade source trials of this task.
171
+ </ParamField>
172
+
173
+ ## job imports
174
+
175
+ ```bash
176
+ evolve job imports [options]
177
+ ```
178
+
179
+ List your job imports (uploads), newest first. This is how an import id is found again.
180
+
181
+ <ListFlags />
182
+
183
+ <ParamField path="--status" type="QUEUED | RUNNING | COMPLETED | FAILED">
184
+ Filter by import status.
185
+ </ParamField>
186
+
187
+ ## job import
188
+
189
+ ```bash
190
+ evolve job import <import-id> [options]
191
+ ```
192
+
193
+ Show one job import, an upload's record, with a `skipped` count and one line per skipped trial when the ingest left any out.
194
+
195
+ <ParamField path="--watch">
196
+ Poll until the import settles: `COMPLETED`, and the job is printed, or `FAILED`.
197
+ </ParamField>
198
+
199
+ ## job download
200
+
201
+ ```bash
202
+ evolve job download <id> [options]
203
+ ```
204
+
205
+ Download the job's results, unpacked as the standard job-directory tree plus `evolve.json` records.
206
+
207
+ <ParamField path="-o, --output-dir" type="dir">
208
+ Directory to unpack into. Defaults to the current directory. The tree lands in `<dir>/job-<id>/`.
209
+ </ParamField>
210
+
211
+ <ParamField path="--overwrite">
212
+ Replace an existing `<dir>/job-<id>/`.
213
+ </ParamField>
214
+
215
+ ## job grep
216
+
217
+ ```bash
218
+ evolve job grep <id> <pattern> [options]
219
+ ```
220
+
221
+ Search every trial's parsed trace in one server-side pass and print one line per matching trial with its sampled events. The pattern grammar and the page semantics are on [`jobs().grep()`](/sdk-reference/jobs#grep).
222
+
223
+ <ParamField path="--type" type="event-type">
224
+ Only search events of exactly this type.
225
+ </ParamField>
226
+
227
+ <ParamField path="-l, --limit" type="n" default="50">
228
+ Per-trial match groups per page. Max 200.
229
+ </ParamField>
230
+
231
+ <ParamField path="--cursor" type="c">
232
+ Resume after this trial id, the previous page's `nextCursor`.
233
+ </ParamField>
234
+
235
+ ```bash
236
+ evolve job grep 3e1f9a2c-… 'permission denied'
237
+ ```
238
+
239
+ <GlobalOptions />
@@ -0,0 +1,196 @@
1
+ ---
2
+ title: "evolve run"
3
+ description: "Start a job, and optionally follow it to the end."
4
+ ---
5
+
6
+ import GlobalOptions from '/snippets/global-options.mdx';
7
+
8
+ ```bash
9
+ evolve run [options]
10
+ ```
11
+
12
+ Start a job. `evolve run` is the short form of `evolve job start`, and both take the same options.
13
+
14
+ ```bash
15
+ evolve run \
16
+ -d terminal-bench-4@4.0 \
17
+ -a codex \
18
+ -m gpt-5.5 \
19
+ -k 2 \
20
+ -l 5 \
21
+ --watch
22
+ ```
23
+
24
+ ## What to run
25
+
26
+ <ParamField path="-d, --dataset" type="name[@version]">
27
+ Dataset to run. Repeatable. A bare name means the active version.
28
+ </ParamField>
29
+
30
+ <ParamField path="-i, --include-task-name" type="glob">
31
+ Include filter over task names, applied to every dataset. Repeatable.
32
+ </ParamField>
33
+
34
+ <ParamField path="-x, --exclude-task-name" type="glob">
35
+ Exclude filter over task names, applied to every dataset. Repeatable.
36
+ </ParamField>
37
+
38
+ <ParamField path="-l, --n-tasks" type="n">
39
+ Cap the task count of each dataset after the filters.
40
+ </ParamField>
41
+
42
+ <ParamField path="-a, --agent" type="name[@version]">
43
+ Agent, built-in or registered.
44
+ </ParamField>
45
+
46
+ <ParamField path="-m, --model" type="name">
47
+ Model. Repeatable; each model is one arm.
48
+ </ParamField>
49
+
50
+ <ParamField path="-k, --n-attempts" type="n" default="1">
51
+ Attempts per task and arm.
52
+ </ParamField>
53
+
54
+ <ParamField path="-n, --n-concurrent" type="n" default="4">
55
+ Parallel trials, 1 to 150.
56
+ </ParamField>
57
+
58
+ <ParamField path="-e, --env" type="provider" default="daytona">
59
+ Sandbox provider: `e2b`, `daytona` or `modal`.
60
+ </ParamField>
61
+
62
+ <ParamField path="--job-name" type="name">
63
+ User-facing label. Generated by the server when omitted.
64
+ </ParamField>
65
+
66
+ ## The arm
67
+
68
+ Each of these applies to every arm of the job.
69
+
70
+ <ParamField path="--effort" type="value">
71
+ Reasoning effort for every arm, applied verbatim. The values each harness accepts are on the [models](/core-concepts/models#reasoning-effort) page and in the [capability document](/sdk-reference/meta). An agent that cannot honor the value is refused, never silently skipped.
72
+ </ParamField>
73
+
74
+ <ParamField path="--preset" type="name">
75
+ Named settings preset for every arm: `no-internet` turns the vendor's server-side web tools off, `pinned-context` fixes the context window at 200000 tokens. Only `claude` and `codex` take one; a preset wins over `--ak config` where they disagree. An unknown name is refused with `invalid_input`, a preset the agent cannot guarantee with `agent_preset_unsupported`.
76
+ </ParamField>
77
+
78
+ <ParamField path="--agent-kwarg, --ak" type="key=value">
79
+ Agent kwarg for every arm, as `key=value`, repeatable. The delivered key is `config`: `--ak 'config=<path|inline JSON>'` becomes the harness's native settings file, your document as the base and the platform's routing on top; a path is read locally, as JSON or as TOML for a Codex config. Only `claude` and `codex` take a config. A key other than `config` is refused with `agent_kwarg_unsupported`, a config on another harness with `agent_config_unsupported`, and a key touching billing, base URL, routing or env with `agent_config_key_refused`.
80
+ </ParamField>
81
+
82
+ <ParamField path="--skills, --skill" type="ref | path">
83
+ Skill for every agent arm. Repeatable. Accepts `skills.sh/<owner>/<repo>[/<skill>]`, `org/repo[@ref]`, an https git URL, `upload:<id>`, `name:<skill-name>` (your moving name pointer, resolved server-side), or a local folder, which is uploaded first and then referenced; `--print-config` still shows the path you typed.
84
+ </ParamField>
85
+
86
+ <ParamField path="--ae, --agent-env" type="KEY=VALUE">
87
+ Sent as the job's `agent_env`, which the server refuses on this deployment: the create fails naming the two `--ve` keys it honors instead. Agent environment comes from the task's `[environment.env]` table or an attached `--secret`.
88
+ </ParamField>
89
+
90
+ <ParamField path="--ve, --verifier-env" type="KEY=VALUE">
91
+ Environment for every verifier run. Repeatable. The server honors exactly `REWARDKIT_JUDGE` and `REWARDKIT_MODEL`, rewardkit's judge override, and refuses any other key.
92
+ </ParamField>
93
+
94
+ <ParamField path="--secret" type="NAME[@LABEL][=ENVNAME]">
95
+ Attach one of your stored env secrets to every agent run. Repeatable. `NAME` is the stored secret's name.
96
+
97
+ `@LABEL` picks a labeled row; omitted, the `default` row or the only row, and several labels with no `default` is refused as ambiguous. `=ENVNAME` renames the env var inside the sandbox. References only: the value never rides the command line or the wire.
98
+ </ParamField>
99
+
100
+ <ParamField path="--secret-inline" type="NAME[@LABEL]:DELIVERY=VALUE">
101
+ Save `VALUE` into your vault as an env secret and attach it to this job in one step. Repeatable. `DELIVERY` is `brokered` or `direct` and sits before `=`, so everything after the first `=` is the value, byte for byte. `@LABEL` defaults to `default`.
102
+
103
+ Restating an existing secret exactly attaches it, so re-running the same command converges; a different value or delivery is refused as `secret_exists`. The job stores only the reference.
104
+ </ParamField>
105
+
106
+ ## Spend and retries
107
+
108
+ <ParamField path="--max-trial-spend" type="usd">
109
+ Model-spend cap for each trial. Defaults to the server's, $200.
110
+ </ParamField>
111
+
112
+ <ParamField path="-r, --max-retries" type="n">
113
+ Max automatic retries per trial on infrastructure errors. Defaults to 2; `0` turns retries off. Each attempt carries the full trial cap. Over a `-c` file it overrides only its own field of `retry`.
114
+ </ParamField>
115
+
116
+ <ParamField path="--retry-include" type="exception">
117
+ Exception types to retry on. Repeatable. The names are `AgentAuthenticationError` (a gateway credential refusal), `ModelNotFoundError` (a model that never served) and `InfrastructureError` (everything else). Defaults to everything `--retry-exclude` admits.
118
+ </ParamField>
119
+
120
+ <ParamField path="--retry-exclude" type="exception">
121
+ Exception types not to retry on. Repeatable, and it wins over `--retry-include`. Defaults to the platform's non-retryable set; an explicit `null` in a `-c` file's `retry` turns exclusions off entirely.
122
+ </ParamField>
123
+
124
+ ## Embedded analysis
125
+
126
+ <ParamField path="--analyze">
127
+ Analyze each trial's trace server-side as it settles. Cancelled trials are skipped. Bare, it uses the platform's analyze model, rubric and prompt.
128
+ </ParamField>
129
+
130
+ <ParamField path="--analyze-model" type="name">
131
+ Model the analyzer runs. Must be on the `claude` roster. Implies `--analyze`.
132
+ </ParamField>
133
+
134
+ <ParamField path="--analyze-rubric" type="path">
135
+ Rubric file for the analyzer: TOML, YAML or JSON in the `{criteria}` shape. Implies `--analyze`.
136
+ </ParamField>
137
+
138
+ <ParamField path="--analyze-prompt" type="path">
139
+ Prompt file for the analyzer. Replaces the built-in prompt. Implies `--analyze`.
140
+ </ParamField>
141
+
142
+ <ParamField path="--analyze-provider" type="provider">
143
+ Sandbox provider the analyzer runs on. Defaults to the platform's analysis default. Implies `--analyze`.
144
+ </ParamField>
145
+
146
+ <ParamField path="--analyze-effort" type="value">
147
+ Reasoning effort the analyzer runs at. Defaults to the per-model default: `high` on `openrouter/deepseek/deepseek-v4.1-flash`, `max` on `glm-5.3-flash`. Implies `--analyze`.
148
+ </ParamField>
149
+
150
+ ## Timeouts
151
+
152
+ <ParamField path="--timeout-multiplier" type="x" default="1.0">
153
+ Multiplier for task timeouts, any finite number above 0; below 1 shrinks. It multiplies each task's declared timeouts for this job only; the task is never rewritten. A product past 2,147,483,647 ms is refused at create. Resumed and retried jobs inherit the multipliers.
154
+ </ParamField>
155
+
156
+ <ParamField path="--agent-timeout-multiplier" type="x">
157
+ Multiplier for the agent execution timeout. Overrides `--timeout-multiplier`.
158
+ </ParamField>
159
+
160
+ <ParamField path="--verifier-timeout-multiplier" type="x">
161
+ Multiplier for the verifier timeout. Overrides `--timeout-multiplier`.
162
+ </ParamField>
163
+
164
+ <ParamField path="--agent-setup-timeout-multiplier" type="x">
165
+ Multiplier for the agent setup timeout. Overrides `--timeout-multiplier`.
166
+ </ParamField>
167
+
168
+ <ParamField path="--environment-build-timeout-multiplier" type="x">
169
+ Multiplier for the environment build timeout. Overrides `--timeout-multiplier`.
170
+ </ParamField>
171
+
172
+ ## Config file and output
173
+
174
+ <ParamField path="-c, --config" type="path">
175
+ Job config file, YAML or JSON with the same field names `jobs().start()` takes. Explicit flags override its fields one at a time, inside the `retry` and `analyze` objects too. The file may be partial: `-d` and `-a`/`-m` can supply what it omits, but a selector inside it needs `name` and an arm needs `name` and `model_name`.
176
+
177
+ YAML is read with the 1.1 schema (`yes` and `on` are booleans, `012` is octal), and a duplicate key, a second document or an unknown tag refuse with a line number. The body is checked before any request: an unknown key is refused by name, a string must be a string (quote `version: "1.10"`), `n_attempts` and `n_tasks` are at least 1, `n_concurrent_trials` is 1 to 150, `job_name` is at most 200 characters. A bare date or `.inf` is refused rather than rewritten.
178
+ </ParamField>
179
+
180
+ <ParamField path="--print-config">
181
+ Print the resolved job body as JSON and exit without running.
182
+ </ParamField>
183
+
184
+ <ParamField path="--watch">
185
+ Stream events until the job finishes.
186
+ </ParamField>
187
+
188
+ <ParamField path="-q, --quiet">
189
+ With `--watch`: suppress the event log and print the final block only.
190
+ </ParamField>
191
+
192
+ <ParamField path="-y, --yes">
193
+ Accepted for compatibility. Hosted runs have no prompt to confirm.
194
+ </ParamField>
195
+
196
+ <GlobalOptions />