@evolvingmachines/evolve 0.0.65 → 0.0.67-train-harness-wave-2026-09-25.20260926.e00a8a6d

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (153) hide show
  1. package/dist/{chunk-DK2MGUPB.js → chunk-DI6LPW2B.js} +2 -2
  2. package/dist/cli/index.cjs +29 -29
  3. package/dist/cli/index.d.cts +3 -3
  4. package/dist/cli/index.d.ts +3 -3
  5. package/dist/cli/index.js +18 -18
  6. package/dist/index.cjs +64 -60
  7. package/dist/index.d.cts +297 -18
  8. package/dist/index.d.ts +297 -18
  9. package/dist/index.js +48 -44
  10. package/dist/{types-B_qmzlMk.d.cts → types-Dj0iMl5V.d.cts} +70 -90
  11. package/dist/{types-B_qmzlMk.d.ts → types-Dj0iMl5V.d.ts} +70 -90
  12. package/harness-capabilities.json +285 -0
  13. package/package.json +9 -6
  14. package/skills/create-adapter/SKILL.md +71 -26
  15. package/skills/create-task/SKILL.md +124 -66
  16. package/skills/evolve/SKILL.md +37 -20
  17. package/skills/evolve-evals/SKILL.md +142 -69
  18. package/skills/evolve-evals/references/cli-reference/agent.md +76 -0
  19. package/skills/evolve-evals/references/cli-reference/analysis.md +78 -0
  20. package/skills/evolve-evals/references/cli-reference/analyze.md +97 -0
  21. package/skills/evolve-evals/references/cli-reference/auth.md +96 -0
  22. package/skills/evolve-evals/references/cli-reference/check.md +157 -0
  23. package/skills/evolve-evals/references/cli-reference/dataset.md +154 -0
  24. package/skills/evolve-evals/references/cli-reference/filesystem.md +168 -0
  25. package/skills/evolve-evals/references/cli-reference/index.md +149 -0
  26. package/skills/evolve-evals/references/cli-reference/job.md +198 -0
  27. package/skills/evolve-evals/references/cli-reference/run.md +188 -0
  28. package/skills/evolve-evals/references/cli-reference/secrets.md +77 -0
  29. package/skills/evolve-evals/references/cli-reference/session.md +56 -0
  30. package/skills/evolve-evals/references/cli-reference/skill.md +52 -0
  31. package/skills/evolve-evals/references/cli-reference/skills.md +85 -0
  32. package/skills/evolve-evals/references/cli-reference/trial.md +115 -0
  33. package/skills/evolve-evals/references/cli-reference/upload.md +72 -0
  34. package/skills/evolve-evals/references/core-concepts/agents.md +101 -0
  35. package/skills/evolve-evals/references/core-concepts/analyze.md +143 -0
  36. package/skills/evolve-evals/references/core-concepts/check.md +116 -0
  37. package/skills/evolve-evals/references/core-concepts/compatibility.md +39 -0
  38. package/skills/evolve-evals/references/core-concepts/compose.md +103 -0
  39. package/skills/evolve-evals/references/core-concepts/datasets.md +234 -0
  40. package/skills/evolve-evals/references/core-concepts/index.md +52 -0
  41. package/skills/evolve-evals/references/core-concepts/jobs.md +223 -0
  42. package/skills/evolve-evals/references/core-concepts/models.md +73 -0
  43. package/skills/evolve-evals/references/core-concepts/multi-step.md +93 -0
  44. package/skills/evolve-evals/references/core-concepts/rewardkit.md +341 -0
  45. package/skills/evolve-evals/references/core-concepts/sandboxes.md +73 -0
  46. package/skills/evolve-evals/references/core-concepts/secrets.md +86 -0
  47. package/skills/evolve-evals/references/core-concepts/sharing.md +81 -0
  48. package/skills/evolve-evals/references/core-concepts/skills.md +100 -0
  49. package/skills/evolve-evals/references/core-concepts/task-config.md +169 -0
  50. package/skills/evolve-evals/references/core-concepts/task-environment.md +127 -0
  51. package/skills/evolve-evals/references/core-concepts/task-reference.md +191 -0
  52. package/skills/evolve-evals/references/core-concepts/task-verifiers.md +209 -0
  53. package/skills/evolve-evals/references/core-concepts/tasks.md +90 -0
  54. package/skills/evolve-evals/references/core-concepts/teams.md +128 -0
  55. package/skills/evolve-evals/references/core-concepts/trial-outputs.md +225 -0
  56. package/skills/evolve-evals/references/core-concepts/trials.md +129 -0
  57. package/skills/evolve-evals/references/core-concepts/upload-sdk-session.md +240 -0
  58. package/skills/evolve-evals/references/core-concepts/upload.md +166 -0
  59. package/skills/evolve-evals/references/dashboard/checks.md +54 -0
  60. package/skills/evolve-evals/references/dashboard/datasets.md +65 -0
  61. package/skills/evolve-evals/references/dashboard/jobs.md +106 -0
  62. package/skills/evolve-evals/references/dashboard/traces.md +54 -0
  63. package/skills/evolve-evals/references/dashboard/trial-viewer.md +77 -0
  64. package/skills/evolve-evals/references/getting-started/first-task.md +228 -0
  65. package/skills/evolve-evals/references/getting-started/installation.md +79 -0
  66. package/skills/evolve-evals/references/getting-started/quick-start.md +100 -0
  67. package/skills/evolve-evals/references/index.md +58 -0
  68. package/skills/evolve-evals/references/sdk/http.md +110 -0
  69. package/skills/evolve-evals/references/sdk/python.md +108 -0
  70. package/skills/evolve-evals/references/sdk/typescript.md +104 -0
  71. package/skills/evolve-evals/references/sdk-reference/agents.md +75 -0
  72. package/skills/evolve-evals/references/sdk-reference/analyses.md +125 -0
  73. package/skills/evolve-evals/references/sdk-reference/auth.md +171 -0
  74. package/skills/evolve-evals/references/sdk-reference/checks.md +146 -0
  75. package/skills/evolve-evals/references/sdk-reference/datasets.md +170 -0
  76. package/skills/evolve-evals/references/sdk-reference/error-codes.md +231 -0
  77. package/skills/evolve-evals/references/sdk-reference/errors.md +98 -0
  78. package/skills/evolve-evals/references/sdk-reference/filesystem.md +166 -0
  79. package/skills/evolve-evals/references/sdk-reference/index.md +161 -0
  80. package/skills/evolve-evals/references/sdk-reference/jobs.md +250 -0
  81. package/skills/evolve-evals/references/sdk-reference/meta.md +54 -0
  82. package/skills/evolve-evals/references/sdk-reference/methods/agents.md +154 -0
  83. package/skills/evolve-evals/references/sdk-reference/methods/analyses.md +211 -0
  84. package/skills/evolve-evals/references/sdk-reference/methods/auth.md +257 -0
  85. package/skills/evolve-evals/references/sdk-reference/methods/checks.md +462 -0
  86. package/skills/evolve-evals/references/sdk-reference/methods/datasets.md +687 -0
  87. package/skills/evolve-evals/references/sdk-reference/methods/filesystem.md +527 -0
  88. package/skills/evolve-evals/references/sdk-reference/methods/jobs.md +1023 -0
  89. package/skills/evolve-evals/references/sdk-reference/methods/meta.md +213 -0
  90. package/skills/evolve-evals/references/sdk-reference/methods/secrets.md +157 -0
  91. package/skills/evolve-evals/references/sdk-reference/methods/skills.md +125 -0
  92. package/skills/evolve-evals/references/sdk-reference/methods/trials.md +371 -0
  93. package/skills/evolve-evals/references/sdk-reference/secrets.md +104 -0
  94. package/skills/evolve-evals/references/sdk-reference/skills.md +66 -0
  95. package/skills/evolve-evals/references/sdk-reference/trials.md +164 -0
  96. package/skills/evolve-evals/references/sdk-reference/types.md +1255 -0
  97. package/skills/publish/SKILL.md +103 -79
  98. package/skills/rewardkit/SKILL.md +78 -23
  99. package/spec/openapi.yaml +164 -160
  100. package/skills/evolve-evals/references/cli-reference/agent.mdx +0 -79
  101. package/skills/evolve-evals/references/cli-reference/analysis.mdx +0 -160
  102. package/skills/evolve-evals/references/cli-reference/analyze.mdx +0 -84
  103. package/skills/evolve-evals/references/cli-reference/auth.mdx +0 -133
  104. package/skills/evolve-evals/references/cli-reference/check.mdx +0 -263
  105. package/skills/evolve-evals/references/cli-reference/dataset.mdx +0 -151
  106. package/skills/evolve-evals/references/cli-reference/index.mdx +0 -108
  107. package/skills/evolve-evals/references/cli-reference/job.mdx +0 -278
  108. package/skills/evolve-evals/references/cli-reference/run.mdx +0 -204
  109. package/skills/evolve-evals/references/cli-reference/secrets.mdx +0 -95
  110. package/skills/evolve-evals/references/cli-reference/skill.mdx +0 -61
  111. package/skills/evolve-evals/references/cli-reference/skills.mdx +0 -81
  112. package/skills/evolve-evals/references/cli-reference/trial.mdx +0 -181
  113. package/skills/evolve-evals/references/cli-reference/upload.mdx +0 -36
  114. package/skills/evolve-evals/references/core-concepts/agents.mdx +0 -106
  115. package/skills/evolve-evals/references/core-concepts/analyze.mdx +0 -86
  116. package/skills/evolve-evals/references/core-concepts/check.mdx +0 -64
  117. package/skills/evolve-evals/references/core-concepts/datasets.mdx +0 -138
  118. package/skills/evolve-evals/references/core-concepts/index.mdx +0 -42
  119. package/skills/evolve-evals/references/core-concepts/jobs.mdx +0 -151
  120. package/skills/evolve-evals/references/core-concepts/models.mdx +0 -68
  121. package/skills/evolve-evals/references/core-concepts/sandboxes.mdx +0 -52
  122. package/skills/evolve-evals/references/core-concepts/secrets.mdx +0 -41
  123. package/skills/evolve-evals/references/core-concepts/sharing.mdx +0 -35
  124. package/skills/evolve-evals/references/core-concepts/skills.mdx +0 -42
  125. package/skills/evolve-evals/references/core-concepts/tasks.mdx +0 -216
  126. package/skills/evolve-evals/references/core-concepts/teams.mdx +0 -36
  127. package/skills/evolve-evals/references/core-concepts/trial-outputs.mdx +0 -117
  128. package/skills/evolve-evals/references/core-concepts/trials.mdx +0 -103
  129. package/skills/evolve-evals/references/core-concepts/upload.mdx +0 -93
  130. package/skills/evolve-evals/references/dashboard/checks.mdx +0 -21
  131. package/skills/evolve-evals/references/dashboard/datasets.mdx +0 -24
  132. package/skills/evolve-evals/references/dashboard/jobs.mdx +0 -65
  133. package/skills/evolve-evals/references/dashboard/traces.mdx +0 -17
  134. package/skills/evolve-evals/references/dashboard/trial-viewer.mdx +0 -30
  135. package/skills/evolve-evals/references/getting-started/installation.mdx +0 -56
  136. package/skills/evolve-evals/references/getting-started/quick-start.mdx +0 -94
  137. package/skills/evolve-evals/references/index.mdx +0 -48
  138. package/skills/evolve-evals/references/sdk/python.mdx +0 -161
  139. package/skills/evolve-evals/references/sdk/typescript.mdx +0 -163
  140. package/skills/evolve-evals/references/sdk-reference/agents.mdx +0 -108
  141. package/skills/evolve-evals/references/sdk-reference/analyses.mdx +0 -222
  142. package/skills/evolve-evals/references/sdk-reference/auth.mdx +0 -116
  143. package/skills/evolve-evals/references/sdk-reference/checks.mdx +0 -300
  144. package/skills/evolve-evals/references/sdk-reference/datasets.mdx +0 -346
  145. package/skills/evolve-evals/references/sdk-reference/errors.mdx +0 -210
  146. package/skills/evolve-evals/references/sdk-reference/index.mdx +0 -60
  147. package/skills/evolve-evals/references/sdk-reference/jobs.mdx +0 -822
  148. package/skills/evolve-evals/references/sdk-reference/meta.mdx +0 -52
  149. package/skills/evolve-evals/references/sdk-reference/skills.mdx +0 -69
  150. package/skills/evolve-evals/references/sdk-reference/trials.mdx +0 -302
  151. package/skills/evolve-evals/references/sdk-reference/types.mdx +0 -274
  152. package/skills/evolve-evals/references/snippets/global-options.mdx +0 -13
  153. package/skills/evolve-evals/references/snippets/list-flags.mdx +0 -23
@@ -0,0 +1,1023 @@
1
+ ---
2
+ title: "Jobs methods"
3
+ description: "Every job method: inputs, results, and a small example in each SDK."
4
+ ---
5
+
6
+ Examples use a configured `client`. See [client setup](/sdk-reference/index) for credentials and [returned objects](/sdk-reference/types) for result fields. Optional values are omitted unless shown. Python examples run inside an async function.
7
+
8
+ Collection handles support two forms: `await` reads one page; `for await` / `async for` visits all pages. A page has `items`, `nextCursor`, and `hasMore` in TypeScript; `items`, `next_cursor`, and `has_more` in Python.
9
+
10
+ | Work | Methods |
11
+ | --- | --- |
12
+ | Submit and follow | [start](#start), [get](#get), [list](#list), [watch](#watch) |
13
+ | Read results | [trials](#trials), [tasks](#tasks), [compare](#compare), [grep](#grep), [download](#download) |
14
+ | Change execution | [cancel](#cancel), [resume](#resume), [retry](#retry), [regrade](#regrade) |
15
+ | Analyze | [analyze](#analyze), [watchAnalysis](#watchanalysis) |
16
+ | Import | [upload](#upload), [getImport](#getimport), [watchImport](#watchimport), [listImports](#listimports) |
17
+ | Share or remove | [share](#share), [unshare](#unshare), [shares](#shares), [delete](#delete) |
18
+
19
+ ## start
20
+
21
+ Submit tasks and agent/model arms. Returns the accepted `Job`; execution continues in the cloud.
22
+
23
+ ### Signature
24
+
25
+ ```ts TypeScript signature
26
+ start(
27
+ input: JobCreate,
28
+ options?: StartJobOptions
29
+ ): Promise<Job>;
30
+ ```
31
+
32
+ ```python Python signature
33
+ async def start(
34
+ *,
35
+ datasets: List[Union[DatasetSelector, Dict[str, Any]]],
36
+ agents: List[Union[AgentArm, Dict[str, Any]]],
37
+ job_name: Optional[str] = None,
38
+ org: Optional[str] = None,
39
+ n_attempts: Optional[int] = None,
40
+ n_concurrent_trials: Optional[int] = None,
41
+ max_trial_spend_usd: Optional[float] = None,
42
+ sandbox_provider: Optional[str] = None,
43
+ retry: Optional[JobRetryConfigInput] = None,
44
+ analyze: Optional[AnalyzeConfigInput] = None,
45
+ system_log: Optional[bool] = None,
46
+ timeout_multiplier: Optional[float] = None,
47
+ agent_timeout_multiplier: Optional[float] = None,
48
+ verifier_timeout_multiplier: Optional[float] = None,
49
+ agent_setup_timeout_multiplier: Optional[float] = None,
50
+ environment_build_timeout_multiplier: Optional[float] = None,
51
+ agent_env: Optional[Dict[str, str]] = None,
52
+ verifier_env: Optional[Dict[str, str]] = None,
53
+ secrets: Optional[List[Union['JobSecretRef', 'JobSecretInline', Dict[str, Any]]]] = None,
54
+ idempotency_key: Optional[str] = None,
55
+ ) -> Job: ...
56
+ ```
57
+
58
+ ```ts TypeScript
59
+ const job = await client.start({
60
+ datasets: [
61
+ {
62
+ name: "harbor-examples",
63
+ version: "1.0",
64
+ task_names: ["hello-world"]
65
+ }
66
+ ],
67
+ agents: [
68
+ {
69
+ name: "codex",
70
+ model_name: "gpt-5.6-luna"
71
+ }
72
+ ],
73
+ max_trial_spend_usd: 1,
74
+ retry: {
75
+ max_retries: 0
76
+ },
77
+ });
78
+ ```
79
+
80
+ ```python Python
81
+ job = await client.start(
82
+ datasets=[
83
+ {
84
+ 'name': 'harbor-examples',
85
+ 'version': '1.0',
86
+ 'task_names': ['hello-world'],
87
+ },
88
+ ],
89
+ agents=[
90
+ {
91
+ 'name': 'codex',
92
+ 'model_name': 'gpt-5.6-luna',
93
+ },
94
+ ],
95
+ max_trial_spend_usd=1,
96
+ retry={
97
+ 'max_retries': 0,
98
+ },
99
+ )
100
+ ```
101
+
102
+ Required: nonempty `datasets` and `agents`; each selector needs `name`, and each arm needs `name` plus `model_name`. Python accepts dictionaries or the matching dataclasses.
103
+
104
+ TypeScript takes one input object, then optional `{ idempotencyKey }`. Python takes keyword arguments, including `idempotency_key`.
105
+
106
+ [Run settings](/sdk-reference/jobs#run-settings) explains defaults, timeout multipliers, retry policy, secrets, and accepted environment overrides. An omitted SDK option leaves the decision to the server; the returned job records the resolved settings. `agent_env` is a pass-through input that the managed platform currently refuses. `verifier_env` accepts only `REWARDKIT_JUDGE` and `REWARDKIT_MODEL`.
107
+
108
+ ### Input fields
109
+
110
+ ```ts
111
+ interface JobCreate {
112
+ job_name?: string;
113
+ org?: string;
114
+ datasets: DatasetSelector[];
115
+ agents: AgentArmInput[];
116
+ n_attempts?: number;
117
+ n_concurrent_trials?: number;
118
+ max_trial_spend_usd?: number;
119
+ sandbox_provider?: EvalSandboxProvider;
120
+ retry?: RetryConfigInput;
121
+ analyze?: AnalyzeConfigInput;
122
+ system_log?: boolean;
123
+ timeout_multiplier?: number;
124
+ agent_timeout_multiplier?: number;
125
+ verifier_timeout_multiplier?: number;
126
+ agent_setup_timeout_multiplier?: number;
127
+ environment_build_timeout_multiplier?: number;
128
+ agent_env?: Record<string, string>;
129
+ verifier_env?: Record<string, string>;
130
+ secrets?: Array<JobSecretRef | JobSecretInline>;
131
+ }
132
+
133
+ interface DatasetSelector {
134
+ name: string;
135
+ version?: string;
136
+ task_names?: string[];
137
+ exclude_task_names?: string[];
138
+ n_tasks?: number;
139
+ }
140
+
141
+ interface AgentArmInput {
142
+ name: string;
143
+ model_name: string;
144
+ version?: string | null;
145
+ reasoning_effort?: string | null;
146
+ kwargs?: Record<string, unknown> | null;
147
+ preset?: string | null;
148
+ skills?: string[] | null;
149
+ }
150
+
151
+ interface RetryConfigInput {
152
+ max_retries?: number;
153
+ include_exceptions?: string[] | null;
154
+ exclude_exceptions?: string[] | null;
155
+ wait_multiplier?: number;
156
+ min_wait_sec?: number;
157
+ max_wait_sec?: number;
158
+ }
159
+
160
+ interface JobSecretRef {
161
+ name: string;
162
+ label?: string;
163
+ as?: string;
164
+ }
165
+
166
+ interface JobSecretInline {
167
+ name: string;
168
+ value: string;
169
+ delivery: "brokered" | "direct";
170
+ label?: string;
171
+ as?: string;
172
+ }
173
+ ```
174
+
175
+ ## get
176
+
177
+ Read one job by id. Returns `Job`, including resolved configuration, trial counts, statistics, and provenance.
178
+
179
+ ### Signature
180
+
181
+ ```ts TypeScript signature
182
+ get(id: string): Promise<Job>;
183
+ ```
184
+
185
+ ```python Python signature
186
+ async def get(id: str) -> Job: ...
187
+ ```
188
+
189
+ ```ts TypeScript
190
+ const job = await client.get(jobId);
191
+ ```
192
+
193
+ ```python Python
194
+ job = await client.get(job_id)
195
+ ```
196
+
197
+ ## list
198
+
199
+ List jobs, check summaries, or both. Returns a page or an iterable of matching rows.
200
+
201
+ ### Signature
202
+
203
+ ```ts TypeScript signature
204
+ list(
205
+ options?: ListJobsOptions & { kind?: "job" }
206
+ ): JobList;
207
+ list(
208
+ options: ListJobsOptions & { kind: "check" }
209
+ ): CheckRowList;
210
+ list(
211
+ options: ListJobsOptions & { kind: "all" }
212
+ ): JobListItemList;
213
+ list(options: ListJobsOptions): JobListItemList;
214
+ ```
215
+
216
+ ```python Python signature
217
+ def list(
218
+ *,
219
+ search: Optional[str] = None,
220
+ scope: Optional[JobListScope] = None,
221
+ kind: Optional[JobListKind] = None,
222
+ limit: Optional[int] = None,
223
+ cursor: Optional[str] = None,
224
+ ) -> _PaginatedList: ...
225
+ ```
226
+
227
+ ```ts TypeScript
228
+ const page = await client.list({
229
+ kind: "all",
230
+ scope: "org",
231
+ limit: 20
232
+ });
233
+ ```
234
+
235
+ ```python Python
236
+ page = await client.list(
237
+ kind='all',
238
+ scope='org',
239
+ limit=20,
240
+ )
241
+ ```
242
+
243
+ | Option | Type and default |
244
+ | --- | --- |
245
+ | `search` | Optional string; filters job and dataset names |
246
+ | `scope` | `my` (default), `shared`, or `org` |
247
+ | `kind` | `job` (default), `check`, or `all` |
248
+ | `limit`, `cursor` | Optional page size and cursor; default 50, maximum 200 |
249
+
250
+ Read `row.kind` before using a mixed row. A `check` row has task-check counts, not trial counts. See [row fields](/sdk-reference/types#job-or-check-row).
251
+
252
+ ## trials
253
+
254
+ List a job’s attempts. Returns a page or an iterable of `Trial`.
255
+
256
+ ### Signature
257
+
258
+ ```ts TypeScript signature
259
+ trials(
260
+ id: string,
261
+ options?: ListTrialsOptions
262
+ ): TrialList;
263
+ ```
264
+
265
+ ```python Python signature
266
+ def trials(
267
+ id: str,
268
+ *,
269
+ status: Optional[List[str]] = None,
270
+ dataset: Optional[str] = None,
271
+ limit: Optional[int] = None,
272
+ cursor: Optional[str] = None,
273
+ ) -> _PaginatedList: ...
274
+ ```
275
+
276
+ ```ts TypeScript
277
+ const page = await client.trials(jobId, {
278
+ status: ["SCORING_ERROR"]
279
+ });
280
+ ```
281
+
282
+ ```python Python
283
+ page = await client.trials(
284
+ job_id,
285
+ status=['SCORING_ERROR'],
286
+ )
287
+ ```
288
+
289
+ Optional `status` is a list of [trial statuses](/sdk-reference/types#status-vocabulary). `dataset` matches the trial’s `source` exactly. `limit` defaults to 50 (maximum 200); `cursor` continues a page. A list row may truncate an exception message; `trials().get(id)` reads its full detail.
290
+
291
+ ## tasks
292
+
293
+ Read one row per task and dataset source: trial tally, mean reward, measured cost, and latest task quality check. Returns a page or iterable of `JobTaskRollup`.
294
+
295
+ ### Signature
296
+
297
+ ```ts TypeScript signature
298
+ tasks(
299
+ id: string,
300
+ options?: ListJobTasksOptions
301
+ ): JobTaskRollupList;
302
+ ```
303
+
304
+ ```python Python signature
305
+ def tasks(
306
+ id: str,
307
+ *,
308
+ limit: Optional[int] = None,
309
+ cursor: Optional[str] = None,
310
+ ) -> _PaginatedList: ...
311
+ ```
312
+
313
+ ```ts TypeScript
314
+ const page = await client.tasks(jobId, {
315
+ limit: 20
316
+ });
317
+ ```
318
+
319
+ ```python Python
320
+ page = await client.tasks(
321
+ job_id,
322
+ limit=20,
323
+ )
324
+ ```
325
+
326
+ Optional `limit` defaults to 50 (maximum 200); `cursor` continues a page. See [task rollup fields](/sdk-reference/types#task-rollups).
327
+
328
+ ## watch
329
+
330
+ Follow job events, or wait for the final `Job`. Use one form per handle.
331
+
332
+ ### Signature
333
+
334
+ ```ts TypeScript signature
335
+ watch(id: string, options?: WatchJobOptions): JobWatch;
336
+ ```
337
+
338
+ ```python Python signature
339
+ def watch(
340
+ id: str,
341
+ *,
342
+ on_event: Optional[Callable[[JobEvent], None]] = None,
343
+ timeout_s: Optional[float] = None,
344
+ reconnect_delay_s: float = 1.0,
345
+ max_reconnect_delay_s: float = 30.0,
346
+ ) -> _JobWatch: ...
347
+ ```
348
+
349
+ ```ts TypeScript
350
+ const final = await client.watch(jobId, {
351
+ onEvent: e => console.log(e.type)
352
+ });
353
+ ```
354
+
355
+ ```python Python
356
+ final = await client.watch(
357
+ job_id,
358
+ on_event=lambda e: print(e.type),
359
+ )
360
+ ```
361
+
362
+ | TypeScript option | Python option | Default |
363
+ | --- | --- | --- |
364
+ | `onEvent(event)` | `on_event(event)` | No callback |
365
+ | `reconnectDelayMs` | `reconnect_delay_s` | 1,000 ms / 1 second |
366
+ | `maxReconnectDelayMs` | `max_reconnect_delay_s` | 30,000 ms / 30 seconds |
367
+ | `signal` | `timeout_s` | No client cancellation signal / no timeout |
368
+
369
+ Iteration returns `JobEvent`, with `seq`, `type`, and `data`. The stream replays from the beginning and resumes on reconnect. [Event payloads](/sdk-reference/types#job-events) document every event type. A `trial.settled` event can be followed by `trial.retrying`; it does not mean the job is finished.
370
+
371
+ ## cancel
372
+
373
+ Request cancellation. Returns `Job`. Repeating it on a terminal job leaves the job unchanged.
374
+
375
+ ### Signature
376
+
377
+ ```ts TypeScript signature
378
+ cancel(id: string): Promise<Job>;
379
+ ```
380
+
381
+ ```python Python signature
382
+ async def cancel(id: str) -> Job: ...
383
+ ```
384
+
385
+ ```ts TypeScript
386
+ const job = await client.cancel(jobId);
387
+ ```
388
+
389
+ ```python Python
390
+ job = await client.cancel(job_id)
391
+ ```
392
+
393
+ ## resume
394
+
395
+ Create a new job for a terminal source’s failed and stopped work. Returns `Job`; the source remains unchanged.
396
+
397
+ ### Signature
398
+
399
+ ```ts TypeScript signature
400
+ resume(
401
+ id: string,
402
+ request?: ResumeRequest,
403
+ options?: StartJobOptions
404
+ ): Promise<Job>;
405
+ ```
406
+
407
+ ```python Python signature
408
+ async def resume(
409
+ id: str,
410
+ *,
411
+ filter_error_types: Optional[List[str]] = None,
412
+ idempotency_key: Optional[str] = None,
413
+ ) -> Job: ...
414
+ ```
415
+
416
+ ```ts TypeScript
417
+ const next = await client.resume(jobId, {}, {
418
+ idempotencyKey: requestId
419
+ });
420
+ ```
421
+
422
+ ```python Python
423
+ next_job = await client.resume(
424
+ job_id,
425
+ idempotency_key=request_id,
426
+ )
427
+ ```
428
+
429
+ Optional `filter_error_types: string[]` selects failures by `exception_info.exception_type`. Omitted, the default set includes stopped trials. The new job records `source_jobs[].action: "resume"`. Imported jobs cannot resume.
430
+
431
+ ## retry
432
+
433
+ Create a new job for selected settled trials, including scored trials. Returns `Job`.
434
+
435
+ ### Signature
436
+
437
+ ```ts TypeScript signature
438
+ retry(
439
+ id: string,
440
+ request?: RetryRequest,
441
+ options?: StartJobOptions
442
+ ): Promise<Job>;
443
+ ```
444
+
445
+ ```python Python signature
446
+ async def retry(
447
+ id: str,
448
+ *,
449
+ trial_ids: Optional[List[str]] = None,
450
+ failed_only: Optional[bool] = None,
451
+ idempotency_key: Optional[str] = None,
452
+ ) -> Job: ...
453
+ ```
454
+
455
+ ```ts TypeScript
456
+ const next = await client.retry(jobId, {
457
+ trial_ids: [trialId]
458
+ });
459
+ ```
460
+
461
+ ```python Python
462
+ next_job = await client.retry(
463
+ job_id,
464
+ trial_ids=[trial_id],
465
+ )
466
+ ```
467
+
468
+ | Option | Meaning |
469
+ | --- | --- |
470
+ | `trial_ids` | Explicit settled trials; the source job may still be running |
471
+ | `failed_only` | Select the terminal job’s failed trials |
472
+ | Neither selection | Repeat every trial of the terminal source job |
473
+ | `idempotencyKey` / `idempotency_key` | Optional stable request key; TypeScript puts it in the third argument |
474
+
475
+ Do not combine `trial_ids` with `failed_only`. The new job records `action: "retry"`. Imported jobs cannot retry.
476
+
477
+ ## regrade
478
+
479
+ Run verifiers again against eligible recorded inputs. Returns a new `Job` with `is_regrade: true`; it does not rerun the agent.
480
+
481
+ ### Signature
482
+
483
+ ```ts TypeScript signature
484
+ regrade(
485
+ id: string,
486
+ request?: RegradeRequest
487
+ ): Promise<Job>;
488
+ ```
489
+
490
+ ```python Python signature
491
+ async def regrade(
492
+ id: str,
493
+ *,
494
+ statuses: Optional[List[str]] = None,
495
+ task_name: Optional[str] = None,
496
+ ) -> Job: ...
497
+ ```
498
+
499
+ ```ts TypeScript
500
+ const job = await client.regrade(jobId, {
501
+ task_name: "hello-world"
502
+ });
503
+ ```
504
+
505
+ ```python Python
506
+ job = await client.regrade(
507
+ job_id,
508
+ task_name='hello-world',
509
+ )
510
+ ```
511
+
512
+ The source job must be terminal. Optional `statuses: TrialStatus[]` and `task_name: string` narrow selection. Shared-verifier, older unretained, and imported inputs may be ineligible. [Regrade requirements](/sdk-reference/jobs) explain the boundary.
513
+
514
+ ## analyze
515
+
516
+ Queue analysis of a terminal job’s traces. Returns the `Job` immediately, with its analysis wave pending.
517
+
518
+ ### Signature
519
+
520
+ ```ts TypeScript signature
521
+ analyze(
522
+ id: string,
523
+ request?: AnalyzeConfigInput
524
+ ): Promise<Job>;
525
+ ```
526
+
527
+ ```python Python signature
528
+ async def analyze(
529
+ id: str,
530
+ *,
531
+ agent: Optional[str] = None,
532
+ model_name: Optional[str] = None,
533
+ rubric: Optional[Rubric] = None,
534
+ prompt: Optional[str] = None,
535
+ sandbox_provider: Optional[EvalSandboxProvider] = None,
536
+ reasoning_effort: Optional[str] = None,
537
+ n_concurrent: Optional[int] = None,
538
+ passing: Optional[bool] = None,
539
+ failing: Optional[bool] = None,
540
+ n_trials: Optional[int] = None,
541
+ trial_ids: Optional[List[str]] = None,
542
+ ) -> Job: ...
543
+ ```
544
+
545
+ ```ts TypeScript
546
+ const job = await client.analyze(jobId, {
547
+ failing: true,
548
+ n_trials: 10
549
+ });
550
+ ```
551
+
552
+ ```python Python
553
+ job = await client.analyze(
554
+ job_id,
555
+ failing=True,
556
+ n_trials=10,
557
+ )
558
+ ```
559
+
560
+ All options are optional. `analyses().defaults()` reads the current agent, model, effort, provider, rubric, and prompt defaults.
561
+
562
+ | Selection | Meaning |
563
+ | --- | --- |
564
+ | `passing` | Only `SCORED` trials with primary reward exactly `1` |
565
+ | `failing` | Other analyzable trials; mutually exclusive with `passing` |
566
+ | `trial_ids` | Explicit trials from this job; combines with the reward filter |
567
+ | `n_trials` | Positive cap after selection |
568
+ | `n_concurrent` | Requested parallel analyses, further bounded by the organization |
569
+
570
+ Cancelled trials are excluded. `trial_ids` is accepted here, but refused inside `start({ analyze })`. The [analysis guide](/sdk-reference/analyses) covers custom rubric and prompt semantics.
571
+
572
+ ### Input fields
573
+
574
+ ```ts
575
+ interface AnalyzeConfigInput {
576
+ agent?: string;
577
+ model_name?: string;
578
+ rubric?: Rubric;
579
+ prompt?: string;
580
+ reasoning_effort?: string;
581
+ sandbox_provider?: EvalSandboxProvider;
582
+ n_concurrent?: number;
583
+ passing?: boolean;
584
+ failing?: boolean;
585
+ n_trials?: number;
586
+ trial_ids?: string[];
587
+ }
588
+ ```
589
+
590
+ ## watchAnalysis
591
+
592
+ Python: `watch_analysis`. Wait for a manually queued analysis wave to settle. Returns the final `Job`.
593
+
594
+ ### Signature
595
+
596
+ ```ts TypeScript signature
597
+ watchAnalysis(
598
+ id: string,
599
+ options?: WatchAnalysisOptions
600
+ ): Promise<Job>;
601
+ ```
602
+
603
+ ```python Python signature
604
+ async def watch_analysis(
605
+ id: str,
606
+ *,
607
+ on_stats: Optional[Callable[[Job], None]] = None,
608
+ poll_interval_s: float = 2.0,
609
+ timeout_s: Optional[float] = None,
610
+ ) -> Job: ...
611
+ ```
612
+
613
+ ```ts TypeScript
614
+ const job = await client.watchAnalysis(jobId, {
615
+ pollIntervalMs: 2000
616
+ });
617
+ ```
618
+
619
+ ```python Python
620
+ job = await client.watch_analysis(
621
+ job_id,
622
+ poll_interval_s=2,
623
+ )
624
+ ```
625
+
626
+ Optional `onStats(job)` / `on_stats(job)` runs when the analysis tally changes. Polling starts at 2 seconds, doubles while unchanged to 30 seconds, and resets on a change. TypeScript accepts `signal`; Python accepts `timeout_s`.
627
+
628
+ Call after `analyze()`. A job with no analysis tally can wait indefinitely. On a running job with embedded analysis, a temporarily empty queue can end this watch before the remaining trials settle.
629
+
630
+ ## compare
631
+
632
+ Compare 2–10 jobs you can read. Returns per-job aggregates and a task matrix, with disagreements first.
633
+
634
+ ### Signature
635
+
636
+ ```ts TypeScript signature
637
+ compare(ids: string[]): Promise<CompareResponse>;
638
+ ```
639
+
640
+ ```python Python signature
641
+ async def compare(ids: List[str]) -> CompareResponse: ...
642
+ ```
643
+
644
+ ```ts TypeScript
645
+ const comparison = await client.compare(
646
+ [firstId, secondId]
647
+ );
648
+ ```
649
+
650
+ ```python Python
651
+ comparison = await client.compare(
652
+ [first_id, second_id],
653
+ )
654
+ ```
655
+
656
+ Read `comparison.jobs` and `comparison.taskMatrix` / `comparison.task_matrix`. Means use scored trials with non-null rewards; `coverage.scored` counts all scored trials. See [all comparison fields](/sdk-reference/types#comparison-results).
657
+
658
+ ## grep
659
+
660
+ Search every trial’s parsed trace in one job. Returns one page of matching trial groups.
661
+
662
+ ### Signature
663
+
664
+ ```ts TypeScript signature
665
+ grep(
666
+ id: string,
667
+ q: string,
668
+ options?: GrepJobOptions
669
+ ): Promise<JobGrepPage>;
670
+ ```
671
+
672
+ ```python Python signature
673
+ async def grep(
674
+ id: str,
675
+ q: str,
676
+ *,
677
+ type: Optional[str] = None,
678
+ cursor: Optional[str] = None,
679
+ limit: Optional[int] = None,
680
+ ) -> JobGrepPage: ...
681
+ ```
682
+
683
+ ```ts TypeScript
684
+ const page = await client.grep(
685
+ jobId,
686
+ "permission denied",
687
+ {
688
+ limit: 20
689
+ }
690
+ );
691
+ ```
692
+
693
+ ```python Python
694
+ page = await client.grep(
695
+ job_id,
696
+ 'permission denied',
697
+ limit=20,
698
+ )
699
+ ```
700
+
701
+ Required `q` is a case-insensitive POSIX regular expression. Optional `type` matches an event type exactly. `limit` defaults to 50 (maximum 200); `cursor` continues the trial groups.
702
+
703
+ Each group has `trial_id`, nullable `task_name`, exact `match_count`, and up to five sampled `events`. Use `trials().trace(trialId, { grep: q })` / `trace(trial_id, grep=q)` to read all matching events for one trial.
704
+
705
+ ## download
706
+
707
+ Download a terminal job’s results as a `.tar.gz` in the [job-directory format](/core-concepts/trial-outputs).
708
+
709
+ ### Signature
710
+
711
+ ```ts TypeScript signature
712
+ download(id: string): Promise<Buffer>;
713
+ download(
714
+ id: string,
715
+ options: { to: string }
716
+ ): Promise<string>;
717
+ download(
718
+ id: string,
719
+ options: { stream: true }
720
+ ): Promise<ReadableStream<Uint8Array>>;
721
+ download(
722
+ id: string,
723
+ options?: DownloadJobOptions
724
+ ): Promise<Buffer | string | ReadableStream<Uint8Array>>;
725
+ ```
726
+
727
+ ```python Python signature
728
+ async def download(
729
+ id: str,
730
+ *,
731
+ to: Optional[str] = None,
732
+ ) -> bytes | str: ...
733
+ ```
734
+
735
+ ```ts TypeScript
736
+ const path = await client.download(jobId, {
737
+ to: "./results"
738
+ });
739
+ ```
740
+
741
+ ```python Python
742
+ path = await client.download(
743
+ job_id,
744
+ to='./results',
745
+ )
746
+ ```
747
+
748
+ | Delivery | TypeScript | Python |
749
+ | --- | --- | --- |
750
+ | Omit options | `Buffer` | `bytes` |
751
+ | `to` directory | Saved file path (`string`) | Saved file path (`str`) |
752
+ | `stream: true` | `ReadableStream<Uint8Array>` | Not exposed |
753
+
754
+ In-memory and saved downloads verify length and any server digest. For a raw TypeScript stream, the caller owns verification. `to` saves an archive; it does not extract it.
755
+
756
+ ## upload
757
+
758
+ Import an existing job directory or archive. Returns `JobImport`, not `Job`.
759
+
760
+ ### Signature
761
+
762
+ ```ts TypeScript signature
763
+ upload(
764
+ source: string | { archive_url: string },
765
+ options?: UploadJobOptions,
766
+ ): Promise<JobImport>;
767
+ ```
768
+
769
+ ```python Python signature
770
+ async def upload(
771
+ dir_or_archive: Optional[str] = None,
772
+ *,
773
+ archive_url: Optional[str] = None,
774
+ dataset: Optional[str] = None,
775
+ on_upload_progress: Optional[Callable[[int, int], None]] = None,
776
+ on_registered: Optional[Callable[[str], None]] = None,
777
+ ) -> JobImport: ...
778
+ ```
779
+
780
+ ```ts TypeScript
781
+ const record = await client.upload("./recorded-job", {
782
+ dataset: "harbor-examples@1.0"
783
+ });
784
+ ```
785
+
786
+ ```python Python
787
+ record = await client.upload(
788
+ './recorded-job',
789
+ dataset='harbor-examples@1.0',
790
+ )
791
+ ```
792
+
793
+ | Input | TypeScript | Python |
794
+ | --- | --- | --- |
795
+ | Directory or `.tar.gz` | First argument: path string | First argument: `dir_or_archive` |
796
+ | Public HTTPS archive | `{ archive_url: "https://…" }` | `archive_url="https://…"` |
797
+ | Dataset link | `options.dataset` | `dataset=` |
798
+ | Byte progress | `onUploadProgress(sent, total)` | `on_upload_progress(sent, total)` |
799
+ | Early import id | `onRegistered(id)` | `on_registered(id)` |
800
+
801
+ Supply one source. URL imports have no client upload progress. `onRegistered` fires only on the resumable upload path, before transfer; the single-request path returns the id when accepted. Follow the import with `watchImport` / `watch_import`, then read `job_id`.
802
+
803
+ For a local agent session, use [Upload an SDK run](/core-concepts/upload-sdk-session).
804
+
805
+ ## getImport
806
+
807
+ Python: `get_import`. Read one import owned by the caller. Returns `JobImport`, including status, progress, skipped trials, failure, and eventual `job_id`.
808
+
809
+ ### Signature
810
+
811
+ ```ts TypeScript signature
812
+ getImport(id: string): Promise<JobImport>;
813
+ ```
814
+
815
+ ```python Python signature
816
+ async def get_import(id: str) -> JobImport: ...
817
+ ```
818
+
819
+ ```ts TypeScript
820
+ const record = await client.getImport(importId);
821
+ ```
822
+
823
+ ```python Python
824
+ record = await client.get_import(import_id)
825
+ ```
826
+
827
+ ## watchImport
828
+
829
+ Python: `watch_import`. Wait for an import to reach `COMPLETED` or `FAILED`. Returns `JobImport` in either case.
830
+
831
+ ### Signature
832
+
833
+ ```ts TypeScript signature
834
+ watchImport(
835
+ id: string,
836
+ options?: WatchJobImportOptions
837
+ ): Promise<JobImport>;
838
+ ```
839
+
840
+ ```python Python signature
841
+ async def watch_import(
842
+ id: str,
843
+ *,
844
+ on_status: Optional[Callable[[JobImport], None]] = None,
845
+ on_progress: Optional[Callable[[JobImportProgress, JobImport], None]] = None,
846
+ poll_interval_s: float = 2.0,
847
+ timeout_s: Optional[float] = None,
848
+ ) -> JobImport: ...
849
+ ```
850
+
851
+ ```ts TypeScript
852
+ const record = await client.watchImport(importId, {
853
+ pollIntervalMs: 2000
854
+ });
855
+ ```
856
+
857
+ ```python Python
858
+ record = await client.watch_import(
859
+ import_id,
860
+ poll_interval_s=2,
861
+ )
862
+ ```
863
+
864
+ Optional callbacks: `onStatus(record)` / `on_status(record)` on status changes; `onProgress(progress, record)` / `on_progress(progress, record)` on progress changes. Polling defaults to 2 seconds. TypeScript accepts `signal`; Python accepts `timeout_s`.
865
+
866
+ Transient 429/503 replies delay the watch. A returned `FAILED` import carries `failure`; success supplies `job_id` while the imported job still exists.
867
+
868
+ ## listImports
869
+
870
+ Python: `list_imports`. Recover previous import ids. Returns a page or iterable of the caller’s `JobImport` records, newest first.
871
+
872
+ ### Signature
873
+
874
+ ```ts TypeScript signature
875
+ listImports(
876
+ options?: ListJobImportsOptions
877
+ ): JobImportList;
878
+ ```
879
+
880
+ ```python Python signature
881
+ def list_imports(
882
+ *,
883
+ status: Optional[str] = None,
884
+ limit: Optional[int] = None,
885
+ cursor: Optional[str] = None,
886
+ ) -> _PaginatedList: ...
887
+ ```
888
+
889
+ ```ts TypeScript
890
+ const page = await client.listImports({
891
+ status: "FAILED",
892
+ limit: 20
893
+ });
894
+ ```
895
+
896
+ ```python Python
897
+ page = await client.list_imports(
898
+ status='FAILED',
899
+ limit=20,
900
+ )
901
+ ```
902
+
903
+ Optional `status` uses `QUEUED`, `RUNNING`, `COMPLETED`, or `FAILED`; `limit` defaults to 50 (maximum 200); `cursor` continues results.
904
+
905
+ ## share
906
+
907
+ Grant read access to a job you created. Returns the whole `JobShares` state.
908
+
909
+ ### Signature
910
+
911
+ ```ts TypeScript signature
912
+ share(
913
+ id: string,
914
+ request: JobShareRequest
915
+ ): Promise<JobShares>;
916
+ ```
917
+
918
+ ```python Python signature
919
+ async def share(
920
+ id: str,
921
+ *,
922
+ link: bool = False,
923
+ emails: Optional[List[str]] = None,
924
+ ) -> JobShares: ...
925
+ ```
926
+
927
+ ```ts TypeScript
928
+ const shares = await client.share(jobId, {
929
+ link: true
930
+ });
931
+ ```
932
+
933
+ ```python Python
934
+ shares = await client.share(
935
+ job_id,
936
+ link=True,
937
+ )
938
+ ```
939
+
940
+ Provide `link: true` / `link=True`, `emails: string[]` / `emails=[...]`, or both. A link is unlisted. Email sharing sends an invitation to each new address; recipients can read, not operate, the job. Up to 50 email shares per job. See [share-state fields](/sdk-reference/types#sharing-and-deletion).
941
+
942
+ ## unshare
943
+
944
+ Revoke a link or email grants. Returns `JobShares`. Creator-only and idempotent.
945
+
946
+ ### Signature
947
+
948
+ ```ts TypeScript signature
949
+ unshare(
950
+ id: string,
951
+ request: JobShareRequest
952
+ ): Promise<JobShares>;
953
+ ```
954
+
955
+ ```python Python signature
956
+ async def unshare(
957
+ id: str,
958
+ *,
959
+ link: bool = False,
960
+ emails: Optional[List[str]] = None,
961
+ ) -> JobShares: ...
962
+ ```
963
+
964
+ ```ts TypeScript
965
+ const shares = await client.unshare(jobId, {
966
+ link: true
967
+ });
968
+ ```
969
+
970
+ ```python Python
971
+ shares = await client.unshare(
972
+ job_id,
973
+ link=True,
974
+ )
975
+ ```
976
+
977
+ Accepts the same `link` and `emails` fields as `share`. A revoked link stops working; sharing again creates a new link.
978
+
979
+ ## shares
980
+
981
+ Read current visibility, link state, and email grants. Returns `JobShares`. Creator-only.
982
+
983
+ ### Signature
984
+
985
+ ```ts TypeScript signature
986
+ shares(id: string): Promise<JobShares>;
987
+ ```
988
+
989
+ ```python Python signature
990
+ async def shares(id: str) -> JobShares: ...
991
+ ```
992
+
993
+ ```ts TypeScript
994
+ const shares = await client.shares(jobId);
995
+ ```
996
+
997
+ ```python Python
998
+ shares = await client.shares(job_id)
999
+ ```
1000
+
1001
+ ## delete
1002
+
1003
+ Permanently remove a terminal job you created and its stored trial and analysis results. Returns `JobDeleteResult`.
1004
+
1005
+ ### Signature
1006
+
1007
+ ```ts TypeScript signature
1008
+ delete(id: string): Promise<JobDeleteResult>;
1009
+ ```
1010
+
1011
+ ```python Python signature
1012
+ async def delete(id: str) -> JobDeleteResult: ...
1013
+ ```
1014
+
1015
+ ```ts TypeScript
1016
+ const receipt = await client.delete(jobId);
1017
+ ```
1018
+
1019
+ ```python Python
1020
+ receipt = await client.delete(job_id)
1021
+ ```
1022
+
1023
+ The receipt has `job_id`, `trials_deleted`, and `analyses_deleted`. Active analyses or a live derived regrade prevent deletion. Regrade job ids are not deleted through this method. Deleting an imported job releases its duplicate-import lock.