@evolvingmachines/evolve 0.0.55-project-sable.20260916.db0bc0bb
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +25 -0
- package/dist/chunk-5GVMBEYI.js +11 -0
- package/dist/chunk-HQJI3HHU.js +1 -0
- package/dist/chunk-JS2UTK2I.js +427 -0
- package/dist/cli/index.cjs +73 -0
- package/dist/cli/index.d.cts +270 -0
- package/dist/cli/index.d.ts +270 -0
- package/dist/cli/index.js +54 -0
- package/dist/index.cjs +669 -0
- package/dist/index.d.cts +5023 -0
- package/dist/index.d.ts +5023 -0
- package/dist/index.js +230 -0
- package/dist/managed-modal-IJW62SSK.js +4 -0
- package/dist/resumable-HFSICSH3.js +1 -0
- package/dist/tar-GVC4KE52.js +1 -0
- package/dist/types-DlpTxdR_.d.cts +4777 -0
- package/dist/types-DlpTxdR_.d.ts +4777 -0
- package/dist/upload-KPYDMWFJ.js +1 -0
- package/docs-agents/SKILL.md +138 -0
- package/docs-agents/index.md +81 -0
- package/docs-agents/python/01-getting-started.md +511 -0
- package/docs-agents/python/02-configuration.md +855 -0
- package/docs-agents/python/03-runtime.md +899 -0
- package/docs-agents/python/04-streaming.md +451 -0
- package/docs-agents/python/05-swarm-pipeline.md +953 -0
- package/docs-agents/python/index.md +122 -0
- package/docs-agents/typescript/01-getting-started.md +494 -0
- package/docs-agents/typescript/02-configuration.md +841 -0
- package/docs-agents/typescript/03-runtime.md +864 -0
- package/docs-agents/typescript/04-streaming.md +478 -0
- package/docs-agents/typescript/05-swarm-pipeline.md +957 -0
- package/docs-agents/typescript/index.md +121 -0
- package/docs-evals/SKILL.md +112 -0
- package/docs-evals/cli-reference/agent.mdx +67 -0
- package/docs-evals/cli-reference/analysis.mdx +91 -0
- package/docs-evals/cli-reference/analyze.mdx +74 -0
- package/docs-evals/cli-reference/auth.mdx +60 -0
- package/docs-evals/cli-reference/check.mdx +146 -0
- package/docs-evals/cli-reference/dataset.mdx +133 -0
- package/docs-evals/cli-reference/index.mdx +108 -0
- package/docs-evals/cli-reference/job.mdx +239 -0
- package/docs-evals/cli-reference/run.mdx +196 -0
- package/docs-evals/cli-reference/secrets.mdx +95 -0
- package/docs-evals/cli-reference/skill.mdx +51 -0
- package/docs-evals/cli-reference/skills.mdx +81 -0
- package/docs-evals/cli-reference/trial.mdx +112 -0
- package/docs-evals/cli-reference/upload.mdx +36 -0
- package/docs-evals/core-concepts/agents.mdx +106 -0
- package/docs-evals/core-concepts/analyze.mdx +86 -0
- package/docs-evals/core-concepts/check.mdx +51 -0
- package/docs-evals/core-concepts/datasets.mdx +138 -0
- package/docs-evals/core-concepts/index.mdx +42 -0
- package/docs-evals/core-concepts/jobs.mdx +136 -0
- package/docs-evals/core-concepts/models.mdx +68 -0
- package/docs-evals/core-concepts/sandboxes.mdx +52 -0
- package/docs-evals/core-concepts/secrets.mdx +41 -0
- package/docs-evals/core-concepts/skills.mdx +42 -0
- package/docs-evals/core-concepts/tasks.mdx +216 -0
- package/docs-evals/core-concepts/trial-outputs.mdx +103 -0
- package/docs-evals/core-concepts/trials.mdx +103 -0
- package/docs-evals/core-concepts/upload.mdx +45 -0
- package/docs-evals/dashboard/checks.mdx +15 -0
- package/docs-evals/dashboard/datasets.mdx +24 -0
- package/docs-evals/dashboard/jobs.mdx +65 -0
- package/docs-evals/dashboard/traces.mdx +15 -0
- package/docs-evals/dashboard/trial-viewer.mdx +30 -0
- package/docs-evals/docs.json +180 -0
- package/docs-evals/getting-started/installation.mdx +56 -0
- package/docs-evals/getting-started/quick-start.mdx +85 -0
- package/docs-evals/index.mdx +36 -0
- package/docs-evals/sdk/python.mdx +139 -0
- package/docs-evals/sdk/typescript.mdx +141 -0
- package/docs-evals/sdk-reference/agents.mdx +105 -0
- package/docs-evals/sdk-reference/analyses.mdx +125 -0
- package/docs-evals/sdk-reference/auth.mdx +76 -0
- package/docs-evals/sdk-reference/checks.mdx +166 -0
- package/docs-evals/sdk-reference/datasets.mdx +305 -0
- package/docs-evals/sdk-reference/errors.mdx +199 -0
- package/docs-evals/sdk-reference/index.mdx +58 -0
- package/docs-evals/sdk-reference/jobs.mdx +731 -0
- package/docs-evals/sdk-reference/meta.mdx +52 -0
- package/docs-evals/sdk-reference/skills.mdx +68 -0
- package/docs-evals/sdk-reference/trials.mdx +232 -0
- package/docs-evals/sdk-reference/types.mdx +272 -0
- package/docs-evals/snippets/global-options.mdx +13 -0
- package/docs-evals/snippets/list-flags.mdx +23 -0
- package/harness-capabilities.json +430 -0
- package/hosted-error-codes.json +120 -0
- package/package.json +163 -0
- package/skills/create-adapter/SKILL.md +287 -0
- package/skills/create-task/SKILL.md +491 -0
- package/skills/evolve/SKILL.md +39 -0
- package/skills/publish/SKILL.md +142 -0
- package/skills/rewardkit/SKILL.md +330 -0
- package/spec/atif/README.md +45 -0
- package/spec/atif/trajectory.schema.json +865 -0
- package/spec/openapi.yaml +13447 -0
|
@@ -0,0 +1,4777 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Public types for the hosted evaluations API — datasets, jobs, trials, agents.
|
|
3
|
+
*
|
|
4
|
+
* THE VOCABULARY IS THE WIRE'S. Every field on a wire-shaped object below is
|
|
5
|
+
* spelled exactly as spec/openapi.yaml spells it (snake_case), so the spec
|
|
6
|
+
* reads as the SDK's own field reference and nothing is ever lost in a casing
|
|
7
|
+
* translation. The only camelCase keys are the four frozen historical spots
|
|
8
|
+
* the spec names: the page envelope (`items`/`nextCursor`/`hasMore`), the job
|
|
9
|
+
* body's `trials.byStatus`, the compare response's `taskMatrix`, and the error
|
|
10
|
+
* envelope (`retryAfterSec`/`requestId`). That freeze is a WIRE law, not a
|
|
11
|
+
* property-name law: both SDKs send and receive those keys camelCase, this SDK
|
|
12
|
+
* also exposes them verbatim, and the Python SDK maps them to snake_case
|
|
13
|
+
* attributes (`next_cursor`/`has_more`/`by_status`/`task_matrix`,
|
|
14
|
+
* `retry_after_sec`/`request_id`). SDK-side controls that never touch the
|
|
15
|
+
* wire — client config, delivery options, callbacks — stay
|
|
16
|
+
* TypeScript-idiomatic camelCase.
|
|
17
|
+
*/
|
|
18
|
+
/** Configuration for the datasets() / agents() / jobs() / trials() factories */
|
|
19
|
+
interface HostedClientConfig {
|
|
20
|
+
/** API key (default: process.env.EVOLVE_API_KEY) */
|
|
21
|
+
apiKey?: string;
|
|
22
|
+
/** API base URL override (default: the Evolve dashboard API) */
|
|
23
|
+
baseUrl?: string;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* ONE page shape for every collection on this surface — top level or nested.
|
|
27
|
+
*
|
|
28
|
+
* `nextCursor` means one thing everywhere: pass it back as the next call's
|
|
29
|
+
* `cursor` for the next page, and `null` means there is no next page. It never
|
|
30
|
+
* echoes where you already are, so a poller can always tell it has caught up.
|
|
31
|
+
* The three envelope keys are frozen verbatim on the wire.
|
|
32
|
+
*/
|
|
33
|
+
interface Page<T> {
|
|
34
|
+
items: T[];
|
|
35
|
+
nextCursor: string | null;
|
|
36
|
+
hasMore: boolean;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* A value you can `await`, with the rest of the promise surface attached.
|
|
40
|
+
*
|
|
41
|
+
* The dual-use handles below were `PromiseLike` alone, which is enough for
|
|
42
|
+
* `await` and nothing else — so `client.list().catch(...)` was a compile error
|
|
43
|
+
* two lines after `await client.list()` compiled fine, and `.finally()` for a
|
|
44
|
+
* spinner was unavailable. A handle that is 90% of a promise is worse than one
|
|
45
|
+
* that is none of it, because the missing 10% is only discovered at the call
|
|
46
|
+
* site that needed it.
|
|
47
|
+
*
|
|
48
|
+
* `then`/`catch`/`finally` all return real Promises, so anything chained off a
|
|
49
|
+
* handle behaves exactly like promise code from that point on.
|
|
50
|
+
*/
|
|
51
|
+
interface Awaitable<T> extends PromiseLike<T> {
|
|
52
|
+
catch<TResult = never>(onrejected?: ((reason: unknown) => TResult | PromiseLike<TResult>) | null): Promise<T | TResult>;
|
|
53
|
+
finally(onfinally?: (() => void) | null): Promise<T>;
|
|
54
|
+
}
|
|
55
|
+
/** Cursor + page-size options, accepted by every paged call */
|
|
56
|
+
interface PageOptions {
|
|
57
|
+
/** Max items per page */
|
|
58
|
+
limit?: number;
|
|
59
|
+
/** Cursor from a previous page's nextCursor */
|
|
60
|
+
cursor?: string;
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Job lifecycle status (wire values, as the API emits them).
|
|
64
|
+
* Terminal: COMPLETED, CANCELLED, FAILED.
|
|
65
|
+
*/
|
|
66
|
+
type JobStatus = "QUEUED" | "RUNNING" | "CANCELLING" | "COMPLETED" | "CANCELLED" | "FAILED";
|
|
67
|
+
/**
|
|
68
|
+
* Trial status law: a valid reward (including 0) = SCORED; verifier crash or
|
|
69
|
+
* out-of-domain reward = SCORING_ERROR (never a fabricated zero);
|
|
70
|
+
* INFRASTRUCTURE_ERROR: the trial was lost before a result was recorded;
|
|
71
|
+
* BUDGET: a budget above the trial's own cap refused it — the account's
|
|
72
|
+
* credits, the organization's monthly budget, or the platform's global stop
|
|
73
|
+
* — at the platform's pre-boot wallet check (nothing was started) or mid-run;
|
|
74
|
+
* `exception_info.exception_type` is `ApiUsageLimitError`, the message
|
|
75
|
+
* carries the subject (`user:`, `team:`, `other:`) right after the
|
|
76
|
+
* `[agent-phase:budget_exhausted]` stage prefix every agent-phase failure
|
|
77
|
+
* detail carries; never retried
|
|
78
|
+
* automatically, resume once the budget is raised (a hosted extension —
|
|
79
|
+
* Harbor has no wallet); INDETERMINATE: the platform cannot tell whether the
|
|
80
|
+
* trial completed.
|
|
81
|
+
*
|
|
82
|
+
* A runtime value (not only a type), like TRIAL_ARTIFACT_STREAMS, so the CLI
|
|
83
|
+
* can validate a `--status` filter against this list instead of a second copy.
|
|
84
|
+
*/
|
|
85
|
+
declare const TRIAL_STATUSES: readonly ["QUEUED", "RUNNING", "SCORING", "SCORED", "SCORING_ERROR", "INFRASTRUCTURE_ERROR", "BUDGET", "INDETERMINATE", "CANCELLED"];
|
|
86
|
+
/** One trial lifecycle status — see TRIAL_STATUSES for the law. */
|
|
87
|
+
type TrialStatus = (typeof TRIAL_STATUSES)[number];
|
|
88
|
+
/**
|
|
89
|
+
* Sandbox provider a hosted job runs on. Named `EvalSandboxProvider` to avoid
|
|
90
|
+
* colliding with the core SDK's `SandboxProvider` (the sandbox-abstraction
|
|
91
|
+
* interface). A runtime value for the same reason as TRIAL_STATUSES: the CLI
|
|
92
|
+
* validates `-e/--env` against it.
|
|
93
|
+
*/
|
|
94
|
+
declare const EVAL_SANDBOX_PROVIDERS: readonly ["e2b", "daytona", "modal"];
|
|
95
|
+
/** One sandbox provider — see EVAL_SANDBOX_PROVIDERS. */
|
|
96
|
+
type EvalSandboxProvider = (typeof EVAL_SANDBOX_PROVIDERS)[number];
|
|
97
|
+
/**
|
|
98
|
+
* The list scopes — Harbor's `--scope` on `harbor hub job list` (their
|
|
99
|
+
* cli/hub.py list_jobs_cmd: my | shared | all). `my` is what you created;
|
|
100
|
+
* `shared` is what your organizations' other members created — every row the
|
|
101
|
+
* per-id doors already open for you that is not your own. Harbor's `all`
|
|
102
|
+
* adds public rows; nothing hosted is public, so the server refuses it and
|
|
103
|
+
* the CLI refuses it at the keyboard. A runtime value for the same reason as
|
|
104
|
+
* TRIAL_STATUSES: the CLI validates `--scope` against it.
|
|
105
|
+
*/
|
|
106
|
+
declare const JOB_LIST_SCOPES: readonly ["my", "shared"];
|
|
107
|
+
/** One list scope — see JOB_LIST_SCOPES. */
|
|
108
|
+
type JobListScope = (typeof JOB_LIST_SCOPES)[number];
|
|
109
|
+
/**
|
|
110
|
+
* An analysis's own lifecycle ladder — lowercase, the object's Harbor
|
|
111
|
+
* dialect (spec TrialAnalysis.status). A runtime value so the CLI validates
|
|
112
|
+
* `analysis list --status` against it instead of a second copy.
|
|
113
|
+
*/
|
|
114
|
+
declare const ANALYSIS_STATUSES: readonly ["queued", "running", "completed", "failed"];
|
|
115
|
+
/** One analysis status — see ANALYSIS_STATUSES. */
|
|
116
|
+
type AnalysisStatus = (typeof ANALYSIS_STATUSES)[number];
|
|
117
|
+
/**
|
|
118
|
+
* A task quality check's own lifecycle ladder — derived from its tasks
|
|
119
|
+
* (spec Check.status): `queued` while no task started, `running` while any
|
|
120
|
+
* task is still queued or running and one has started, `completed` once
|
|
121
|
+
* every task settled. A check never fails as a whole (each task carries its
|
|
122
|
+
* own typed failure). A runtime value so the CLI validates `check list
|
|
123
|
+
* --status` against it instead of a second copy.
|
|
124
|
+
*/
|
|
125
|
+
declare const CHECK_STATUSES: readonly ["queued", "running", "completed"];
|
|
126
|
+
/** One check status — see CHECK_STATUSES. */
|
|
127
|
+
type CheckStatus = (typeof CHECK_STATUSES)[number];
|
|
128
|
+
/**
|
|
129
|
+
* Which lane a settled trial's `agent_result.cost_usd` came from. Only
|
|
130
|
+
* `"measured"` is final. `"measured_provisional"` is a real gateway reading
|
|
131
|
+
* taken inside its asynchronous spend flush — an honest floor a deferred pass
|
|
132
|
+
* later confirms or raises into `"measured"`. `"assumed_cap"` means nobody
|
|
133
|
+
* measured this trial: the figure it carries is zero, a placeholder and never
|
|
134
|
+
* the cap (the platform under-bills rather than publish an invented number),
|
|
135
|
+
* replaced when a real reading lands. Read anything but `"measured"` as not
|
|
136
|
+
* yet final.
|
|
137
|
+
*/
|
|
138
|
+
type SpendSource = "measured" | "measured_provisional" | "assumed_cap";
|
|
139
|
+
/**
|
|
140
|
+
* THE ONE-HOME USAGE READING — "what has this run's meter said so far", money
|
|
141
|
+
* and tokens from the SAME gateway spend-log records so the two can never
|
|
142
|
+
* describe different sets of requests. Served under the one key `usage`, with
|
|
143
|
+
* these exact keys, by the trial surfaces and the managed-agents session
|
|
144
|
+
* surfaces alike. While the run is alive the platform's own poll raises the
|
|
145
|
+
* numbers (~30s cadence over a gateway that batches its logs late), so a
|
|
146
|
+
* polling reader sees them tick; once settled, the settled figures replace
|
|
147
|
+
* the live ones under the same keys. The whole object is null when the meter
|
|
148
|
+
* has never answered — never a fabricated zero.
|
|
149
|
+
*/
|
|
150
|
+
interface UsageReading {
|
|
151
|
+
/**
|
|
152
|
+
* True while every number is a LOWER BOUND that can still grow — the run is
|
|
153
|
+
* alive, or its settled lane is not yet confirmed. False = settled; the
|
|
154
|
+
* reading will not move again.
|
|
155
|
+
*/
|
|
156
|
+
provisional: boolean;
|
|
157
|
+
/**
|
|
158
|
+
* Metered model spend so far, USD. Null = the money was never measured
|
|
159
|
+
* (which a trial's `spend_source` lane `assumed_cap` states; the token
|
|
160
|
+
* fields beside it may still carry real readings).
|
|
161
|
+
*/
|
|
162
|
+
spent_usd: number | null;
|
|
163
|
+
/** Prompt tokens so far, INCLUDING the cached share and the cache-write share. */
|
|
164
|
+
input_tokens: number | null;
|
|
165
|
+
/** The cached share of `input_tokens` (read from the provider's prompt cache). */
|
|
166
|
+
cached_input_tokens: number | null;
|
|
167
|
+
/**
|
|
168
|
+
* The share of `input_tokens` WRITTEN to the provider's prompt cache.
|
|
169
|
+
* Anthropic bills it at a premium above the plain input price, so it is
|
|
170
|
+
* the fourth count `spent_usd` needs to be reproducible from the tokens;
|
|
171
|
+
* providers without a cache-write price report 0. Null when the meter
|
|
172
|
+
* never answered — and on a run settled before the platform recorded this
|
|
173
|
+
* share (an older server omits the key), where the three counts beside it
|
|
174
|
+
* stay real: null is never a fabricated 0.
|
|
175
|
+
*/
|
|
176
|
+
cache_write_tokens: number | null;
|
|
177
|
+
/** Completion tokens so far. */
|
|
178
|
+
output_tokens: number | null;
|
|
179
|
+
/** When this reading was taken — show its age, never the figure alone. */
|
|
180
|
+
as_of: string | null;
|
|
181
|
+
}
|
|
182
|
+
/** Where a trial's verifier executed: inside the agent's environment, or a separate one */
|
|
183
|
+
type VerifierEnvironmentMode = "shared" | "separate";
|
|
184
|
+
/**
|
|
185
|
+
* Which step a RUNNING trial is in, so a polling caller can tell a slow build
|
|
186
|
+
* from a slow agent — RUNNING alone cannot.
|
|
187
|
+
*/
|
|
188
|
+
type AttemptPhase = "prepare" | "build" | "boot" | "install" | "agent" | "verify" | "persist";
|
|
189
|
+
/**
|
|
190
|
+
* Trial count histogram by status. EVERY status is present, zeros included, so
|
|
191
|
+
* a status bar can be drawn straight off the response without hardcoding the
|
|
192
|
+
* enum and discovering a new status only when a bar goes missing.
|
|
193
|
+
*/
|
|
194
|
+
type TrialCounts = Record<TrialStatus, number>;
|
|
195
|
+
/**
|
|
196
|
+
* The one "how many" shape: a total plus the zeros-included histogram.
|
|
197
|
+
* `byStatus` is one of the four frozen camelCase wire keys.
|
|
198
|
+
*/
|
|
199
|
+
interface TrialStatusTally {
|
|
200
|
+
total: number;
|
|
201
|
+
byStatus: TrialCounts;
|
|
202
|
+
}
|
|
203
|
+
/** A resolved dataset reference as echoed on job bodies. */
|
|
204
|
+
interface DatasetRef {
|
|
205
|
+
name: string;
|
|
206
|
+
version: string;
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* One dataset a job runs, with per-dataset task filters. `task_names` and
|
|
210
|
+
* `exclude_task_names` are glob patterns; `n_tasks` caps the task count AFTER
|
|
211
|
+
* filtering. A bare `name` resolves to the active version (`no_active_version`
|
|
212
|
+
* when none).
|
|
213
|
+
*/
|
|
214
|
+
interface DatasetSelector {
|
|
215
|
+
/** Catalog dataset name. */
|
|
216
|
+
name: string;
|
|
217
|
+
/** Pin a version; omitted, the active version is used. */
|
|
218
|
+
version?: string;
|
|
219
|
+
/** Include filter — glob patterns over task names. */
|
|
220
|
+
task_names?: string[];
|
|
221
|
+
/** Exclude filter — glob patterns over task names. */
|
|
222
|
+
exclude_task_names?: string[];
|
|
223
|
+
/** Cap the task count after filters are applied. */
|
|
224
|
+
n_tasks?: number;
|
|
225
|
+
}
|
|
226
|
+
/**
|
|
227
|
+
* One agent arm of a job: an agent (built-in or registered) plus a model. A
|
|
228
|
+
* model is always required; the server applies no default.
|
|
229
|
+
*
|
|
230
|
+
* `version` pins an agent version; omitted, the platform resolves the latest
|
|
231
|
+
* supported (`agent_version_not_found` when a pin cannot resolve). The version
|
|
232
|
+
* that actually RAN is recorded on every trial as `agent_info.version`.
|
|
233
|
+
*
|
|
234
|
+
* `reasoning_effort` is the platform extension: declared effort, PART OF THE
|
|
235
|
+
* ARM'S IDENTITY like the agent, the model and the version pin — the same
|
|
236
|
+
* agent and model at "low" and at "high" are two systems, and they
|
|
237
|
+
* de-duplicate separately. Accepted values are published by /api/meta; an
|
|
238
|
+
* effort the agent cannot apply is refused at creation, never recorded and
|
|
239
|
+
* silently dropped.
|
|
240
|
+
*/
|
|
241
|
+
interface AgentArmInput {
|
|
242
|
+
/** Agent name — a built-in or one registered under /api/agents. */
|
|
243
|
+
name: string;
|
|
244
|
+
model_name: string;
|
|
245
|
+
version?: string | null;
|
|
246
|
+
reasoning_effort?: string | null;
|
|
247
|
+
/**
|
|
248
|
+
* Agent kwargs, Harbor's `--ak` channel and wire shape. The one key this
|
|
249
|
+
* platform delivers is `config`: an INLINE JSON object converted into the
|
|
250
|
+
* harness's native settings document inside the sandbox (the CLI resolves
|
|
251
|
+
* `--ak config=<path>` to the file's parsed content before sending — the
|
|
252
|
+
* server never reads a client path). The user document is the base;
|
|
253
|
+
* platform routing is stamped on top. Part of the arm's identity: the same
|
|
254
|
+
* agent+model with two configs are two arms.
|
|
255
|
+
*
|
|
256
|
+
* Server acceptance is typed, never silent: an unrecognized kwarg key is
|
|
257
|
+
* `agent_kwarg_unsupported`, `config` for an agent without native-config
|
|
258
|
+
* support (see /api/meta `supports_config`) is `agent_config_unsupported`,
|
|
259
|
+
* and a config key touching billing, base URLs, routing, or env is
|
|
260
|
+
* `agent_config_key_refused`.
|
|
261
|
+
*/
|
|
262
|
+
kwargs?: Record<string, unknown> | null;
|
|
263
|
+
/**
|
|
264
|
+
* Named agent-settings preset for this arm: `no-internet` (vendor
|
|
265
|
+
* server-side web tools off — Claude settings deny WebSearch/WebFetch,
|
|
266
|
+
* Codex `-c web_search=disabled`) or `pinned-context` (one fixed effective
|
|
267
|
+
* context window). A platform-authored bundle delivered through the same
|
|
268
|
+
* channel as `kwargs.config`, stamped ON TOP of the user document — normal
|
|
269
|
+
* users tick a box and never learn what `--ak` is. Part of the arm's
|
|
270
|
+
* identity: the same agent+model with and without a preset are two arms.
|
|
271
|
+
*
|
|
272
|
+
* Acceptance is typed, never silent: an unknown preset name is
|
|
273
|
+
* `invalid_input`, and a known preset on an agent whose /api/meta entry
|
|
274
|
+
* does not list it under `presets` is `agent_preset_unsupported` — a
|
|
275
|
+
* preset the platform cannot guarantee is refused, never half-applied.
|
|
276
|
+
* Enforcement is harness configuration, exactly as Harbor delivers it.
|
|
277
|
+
*/
|
|
278
|
+
preset?: string | null;
|
|
279
|
+
/**
|
|
280
|
+
* Skill references mounted into every run of this arm — Harbor's
|
|
281
|
+
* trial-config shape: a list of source strings (models/trial/config.py:81).
|
|
282
|
+
* Accepted forms: `skills.sh/<owner>/<repo>[/<skill>]`, `org/repo[@ref]`,
|
|
283
|
+
* an https git URL (optionally `/tree/<ref>/<subdir>`), `upload:<id>`
|
|
284
|
+
* naming a skill uploaded to the platform, or `name:<skill-name>` — the
|
|
285
|
+
* caller's moving name pointer, resolved SERVER-SIDE at creation to its
|
|
286
|
+
* current record and pinned as that record's `upload:<id>` (an unknown
|
|
287
|
+
* name is the typed `skill_name_not_found`). The SDK passes the string
|
|
288
|
+
* through — no client-side resolution. Local filesystem paths are a
|
|
289
|
+
* CLIENT-side convenience only — the CLI uploads the folder first and sends
|
|
290
|
+
* the `upload:<id>` handle; the server refuses raw paths.
|
|
291
|
+
*
|
|
292
|
+
* Git references are PINNED at job creation (the exact commit resolved
|
|
293
|
+
* once, recorded, and used by every trial), so a moving branch can never
|
|
294
|
+
* make two trials of one job run different skill content. Part of the
|
|
295
|
+
* arm's identity: the same agent and model with different skills are two
|
|
296
|
+
* arms.
|
|
297
|
+
*/
|
|
298
|
+
skills?: string[] | null;
|
|
299
|
+
}
|
|
300
|
+
/**
|
|
301
|
+
* Provenance of one skill an arm's runs actually mounted — Harbor's
|
|
302
|
+
* AgentSkillLock vocabulary (models/job/lock.py:141): name, source reference,
|
|
303
|
+
* content digest, and for git-backed skills the repo URL and exact commit.
|
|
304
|
+
*/
|
|
305
|
+
interface SkillLock {
|
|
306
|
+
name: string;
|
|
307
|
+
/** The pinned reference the content came from. */
|
|
308
|
+
source: string;
|
|
309
|
+
/** Content digest, Harbor's recipe: "sha256:<hex>". */
|
|
310
|
+
digest: string;
|
|
311
|
+
git_url: string | null;
|
|
312
|
+
git_commit_id: string | null;
|
|
313
|
+
}
|
|
314
|
+
/** One agent arm as echoed on job bodies (requested pin; null = took the latest). */
|
|
315
|
+
interface AgentArm {
|
|
316
|
+
name: string;
|
|
317
|
+
model_name: string;
|
|
318
|
+
version: string | null;
|
|
319
|
+
reasoning_effort: string | null;
|
|
320
|
+
/** The arm's agent kwargs as accepted; null when none were declared. */
|
|
321
|
+
kwargs: Record<string, unknown> | null;
|
|
322
|
+
/** The arm's named settings preset; null when none was declared. */
|
|
323
|
+
preset: string | null;
|
|
324
|
+
/** The arm's skill references, pinned spelling. Empty = no skills. */
|
|
325
|
+
skills: string[];
|
|
326
|
+
/**
|
|
327
|
+
* What actually mounted, one lock per skill — stamped when the arm's first
|
|
328
|
+
* trial resolves its skills; null until then (and stays null on a job whose
|
|
329
|
+
* trials never ran).
|
|
330
|
+
*/
|
|
331
|
+
skill_locks: SkillLock[] | null;
|
|
332
|
+
}
|
|
333
|
+
/**
|
|
334
|
+
* Provenance of a derived job. `action: "regrade"` = verifier-only re-run of
|
|
335
|
+
* the source; `action: "resume"` (platform extension) = new job over the
|
|
336
|
+
* source's failed and stopped trials; `action: "retry"` = manual retry — new
|
|
337
|
+
* job over caller-SELECTED source trials (explicit ids, failed-only, or the
|
|
338
|
+
* whole job). `type` is always "hub" on this hosted surface.
|
|
339
|
+
*/
|
|
340
|
+
interface SourceJob {
|
|
341
|
+
action: "regrade" | "resume" | "retry";
|
|
342
|
+
type: "hub";
|
|
343
|
+
job_id: string;
|
|
344
|
+
}
|
|
345
|
+
/**
|
|
346
|
+
* Auto-retry policy input — Harbor's RetryConfig vocabulary verbatim. Only
|
|
347
|
+
* trials that settle INFRASTRUCTURE_ERROR are ever considered; the
|
|
348
|
+
* include/exclude sets refine within that class by exception name, exclude
|
|
349
|
+
* taking precedence (Harbor's rule). Every omitted field takes Harbor's own
|
|
350
|
+
* default, with ONE named deviation: `max_retries` omitted takes the
|
|
351
|
+
* PLATFORM fleet default (published as `limits.job.default_max_retries` on
|
|
352
|
+
* GET /api/meta; 2 unless the operator tuned it) rather than Harbor's 0 —
|
|
353
|
+
* infrastructure errors on a hosted fleet retry automatically. Send
|
|
354
|
+
* `max_retries: 0` to turn retries off. Each attempt carries the job's FULL
|
|
355
|
+
* per-trial spend cap, and `worst_case_spend_usd` states the
|
|
356
|
+
* (max_retries + 1) product outright.
|
|
357
|
+
*/
|
|
358
|
+
interface RetryConfigInput {
|
|
359
|
+
/** Maximum automatic retries per trial (0-10). Omitted = fleet default; 0 = off. */
|
|
360
|
+
max_retries?: number;
|
|
361
|
+
/**
|
|
362
|
+
* Exception types to retry on. Null, omitted, or the empty array = no
|
|
363
|
+
* filter — Harbor's include check treats the empty set exactly like None,
|
|
364
|
+
* so `[]` never means "retry nothing".
|
|
365
|
+
*/
|
|
366
|
+
include_exceptions?: string[] | null;
|
|
367
|
+
/**
|
|
368
|
+
* Exception types to NOT retry on; wins over include_exceptions. Omitted =
|
|
369
|
+
* Harbor's default non-retryable set (AgentTimeoutError,
|
|
370
|
+
* VerifierTimeoutError, RewardFileNotFoundError, RewardFileEmptyError,
|
|
371
|
+
* VerifierOutputParseError, ApiUsageLimitError, AgentSafetyRefusalError,
|
|
372
|
+
* AgentAuthenticationError, ModelNotFoundError). An EXPLICIT null is
|
|
373
|
+
* DIFFERENT from omitting: null turns exclusions off entirely — everything
|
|
374
|
+
* the include set admits is retried, the default set included — exactly
|
|
375
|
+
* Harbor's None (their exclude check is guarded by
|
|
376
|
+
* `if exclude_exceptions and ...`, so None disables it).
|
|
377
|
+
*/
|
|
378
|
+
exclude_exceptions?: string[] | null;
|
|
379
|
+
/** Multiplier for exponential backoff wait time (default 1.0). */
|
|
380
|
+
wait_multiplier?: number;
|
|
381
|
+
/** Minimum wait in seconds between retries (default 1.0). */
|
|
382
|
+
min_wait_sec?: number;
|
|
383
|
+
/** Maximum wait in seconds between retries (default 60.0; Harbor's field, no ceiling). */
|
|
384
|
+
max_wait_sec?: number;
|
|
385
|
+
}
|
|
386
|
+
/**
|
|
387
|
+
* The RESOLVED auto-retry policy a job runs under, echoed on every job body —
|
|
388
|
+
* the caller's values or the defaults of the day, resolved at create and
|
|
389
|
+
* stored. Backoff between attempts is Harbor's formula:
|
|
390
|
+
* min(min_wait_sec x wait_multiplier^attempt, max_wait_sec).
|
|
391
|
+
*/
|
|
392
|
+
interface RetryConfig {
|
|
393
|
+
max_retries: number;
|
|
394
|
+
include_exceptions: string[] | null;
|
|
395
|
+
exclude_exceptions: string[];
|
|
396
|
+
wait_multiplier: number;
|
|
397
|
+
min_wait_sec: number;
|
|
398
|
+
max_wait_sec: number;
|
|
399
|
+
}
|
|
400
|
+
/**
|
|
401
|
+
* One analysis criterion — Harbor's RubricCriterion verbatim (their
|
|
402
|
+
* cli/quality_checker/models.py `{name, description, guidance}`). The name
|
|
403
|
+
* becomes the key of the matching entry in `checks`; the guidance is what the
|
|
404
|
+
* analyzer agent is instructed with.
|
|
405
|
+
*/
|
|
406
|
+
interface RubricCriterion {
|
|
407
|
+
/**
|
|
408
|
+
* Criterion identifier, snake_case (it keys the result's `checks` object).
|
|
409
|
+
* The platform's default rubrics name seven criteria for analyze and eleven for check.
|
|
410
|
+
*/
|
|
411
|
+
name: string;
|
|
412
|
+
/** What the criterion evaluates, one sentence. */
|
|
413
|
+
description: string;
|
|
414
|
+
/**
|
|
415
|
+
* Evaluation guidance handed to the analyzer agent — what evidence to read
|
|
416
|
+
* and what PASS / FAIL / NOT_APPLICABLE / UNKNOWN mean for this criterion.
|
|
417
|
+
*/
|
|
418
|
+
guidance: string;
|
|
419
|
+
}
|
|
420
|
+
/**
|
|
421
|
+
* An analysis rubric — Harbor's Rubric shape (`{criteria: [...]}`, their
|
|
422
|
+
* cli/quality_checker/models.py). The criteria set is FROZEN into each
|
|
423
|
+
* analysis at enqueue: the stored result is validated against exactly this
|
|
424
|
+
* set, a missing or extra criterion is a stored typed failure, never a
|
|
425
|
+
* partial pass.
|
|
426
|
+
*/
|
|
427
|
+
interface Rubric {
|
|
428
|
+
criteria: RubricCriterion[];
|
|
429
|
+
}
|
|
430
|
+
/**
|
|
431
|
+
* Trace-analysis configuration — Harbor's `harbor analyze` vocabulary (their
|
|
432
|
+
* cli/analyze.py: `--model`, `--rubric`, `--prompt`). PRESENCE of this object is the
|
|
433
|
+
* switch: on `JobCreate.analyze` it arms the embedded trigger (each trial is
|
|
434
|
+
* analyzed server-side right after it settles; CANCELLED trials are skipped);
|
|
435
|
+
* as the body of `POST /api/jobs/{jobId}/analyze` it configures that manual
|
|
436
|
+
* wave. `{}` is legal and means "all defaults":
|
|
437
|
+
* openrouter/deepseek/deepseek-v4.1-flash at its per-model effort (high)
|
|
438
|
+
* over the platform's default analyze rubric (seven criteria, score_is_earned
|
|
439
|
+
* first) and its default prompt body.
|
|
440
|
+
*
|
|
441
|
+
* The analyzer always runs the claude-code harness (Harbor's default analyze
|
|
442
|
+
* agent) in its own sealed sandbox — on the provider `sandbox_provider`
|
|
443
|
+
* names, or the platform's analysis default when it names none; its spend is
|
|
444
|
+
* capped per analysis and metered as its own line, never blended into the
|
|
445
|
+
* trial's own bill.
|
|
446
|
+
*
|
|
447
|
+
* Which trials, and how wide, are Harbor's own analyze options with their
|
|
448
|
+
* exact names — `n_concurrent` (`-n/--n-concurrent`), `passing` / `failing`,
|
|
449
|
+
* `n_trials` (`-l/--n-trials`; their cli/analyze.py:278-290). All omitted
|
|
450
|
+
* is every analyzable trial, as wide as the organization's
|
|
451
|
+
* `max_concurrent_analyses` allows. Harbor's `-a/--agent`, `--job-name`,
|
|
452
|
+
* `-o/--jobs-dir`, `-k/--n-attempts` and the local-runner kwargs are not
|
|
453
|
+
* on this surface; the contract (`AnalyzeConfigInput` in spec/openapi.yaml)
|
|
454
|
+
* records each with its reason.
|
|
455
|
+
*/
|
|
456
|
+
interface AnalyzeConfigInput {
|
|
457
|
+
/**
|
|
458
|
+
* Model the analyzer agent runs — Harbor's `--model`. The default is
|
|
459
|
+
* openrouter/deepseek/deepseek-v4.1-flash on this platform's claude
|
|
460
|
+
* roster (DeepSeek V4.1 Flash served through OpenRouter, at its default
|
|
461
|
+
* effort high — the owner's ruling 2026-09-10: far more parallel capacity
|
|
462
|
+
* through OpenRouter's provider pool than one pinned Fireworks host) — a
|
|
463
|
+
* recorded deviation from Harbor's default analyze model (their
|
|
464
|
+
* cli/analyze.py `claude-haiku-4-5`): analysis is input-dominated, and
|
|
465
|
+
* this is the roster's intelligence-per-input-dollar pick; `glm-5.3-flash`
|
|
466
|
+
* (at max, the effort its published scores use) and `haiku` stay on the
|
|
467
|
+
* roster as alternatives, `glm-5.3` to escalate, and the same model on
|
|
468
|
+
* its Fireworks route, `fireworks/deepseek-v4.1-flash`, is a further
|
|
469
|
+
* option (the OpenRouter id stays the default). The value speaks
|
|
470
|
+
* the same vocabulary as `agents[].model_name`: either advertised
|
|
471
|
+
* spelling is accepted and stored AS GIVEN (the default is the roster
|
|
472
|
+
* alias), the wire id is resolved only when the analyzer runs, and every
|
|
473
|
+
* stored analysis serves the spelling it was created under. Must be on
|
|
474
|
+
* the claude roster (`GET /api/meta`, `agents[].models`); anything else
|
|
475
|
+
* is refused at accept (`invalid_input`, roster in the message).
|
|
476
|
+
*/
|
|
477
|
+
model_name?: string;
|
|
478
|
+
rubric?: Rubric;
|
|
479
|
+
/**
|
|
480
|
+
* The analyzer's prompt template — the TEXT of Harbor's `-p/--prompt <file>`
|
|
481
|
+
* ("Prompt file for the evaluator agent. Uses built-in default if not
|
|
482
|
+
* specified.", their cli/analyze.py:252-255). It REPLACES the platform's
|
|
483
|
+
* default body as the body of the analyzer's
|
|
484
|
+
* instruction and is rendered with the same three tokens (`{trial_path}`,
|
|
485
|
+
* `{task_section}`, `{criteria_guidance}`): the three tokens are
|
|
486
|
+
* substituted, an unknown `{token}` renders empty, `{{` and `}}` write a
|
|
487
|
+
* literal brace, and any other brace text is left as written (Harbor's
|
|
488
|
+
* Python renderer would render it empty, convert it or raise there —
|
|
489
|
+
* `{ x }` empty, `{x!r}` quoted, `{"a": 1}` an error); the output contract (write
|
|
490
|
+
* `analysis.json` matching the rubric's schema) is appended after it exactly
|
|
491
|
+
* as Harbor appends it, so a custom prompt can never opt out of the
|
|
492
|
+
* deliverable. Stored AS GIVEN and FROZEN into each analysis the config
|
|
493
|
+
* enqueues, like the rubric. Omitted = the built-in prompt (`null` on the
|
|
494
|
+
* resolved echo and on the analysis). Present, it must be non-empty plain
|
|
495
|
+
* text (no NUL character) of ANY length — stored whole, never truncated
|
|
496
|
+
* (no invented number, owner 2026-09-13). The CLI reads the file for
|
|
497
|
+
* you: `evolve analyze -p prompt.txt`.
|
|
498
|
+
*/
|
|
499
|
+
prompt?: string;
|
|
500
|
+
/**
|
|
501
|
+
* Reasoning effort the analyzer runs at — the platform's `agents[].
|
|
502
|
+
* reasoning_effort` vocabulary applied to the analyzer, which IS the
|
|
503
|
+
* claude harness: the accepted values are `GET /api/meta`'s
|
|
504
|
+
* `analyze.reasoning_efforts`, an unknown value is refused
|
|
505
|
+
* `invalid_input` exactly as an arm's is. Omitted, the PER-MODEL default
|
|
506
|
+
* applies (`analyze.models[].default_reasoning_effort`: high on
|
|
507
|
+
* openrouter/deepseek/deepseek-v4.1-flash, the default model — DeepSeek's
|
|
508
|
+
* own documented default, the owner's ruling 2026-09-10; max on
|
|
509
|
+
* glm-5.3-flash — the platform's ruling 2026-09-08, the effort its
|
|
510
|
+
* published scores use; the claude harness default elsewhere). The
|
|
511
|
+
* effort is always passed to the analyzer explicitly and
|
|
512
|
+
* recorded on the analysis (`TrialAnalysis.reasoning_effort`). A hosted
|
|
513
|
+
* extension: Harbor's analyze has no effort option; this is the run
|
|
514
|
+
* door's existing platform vocabulary applied to analyze.
|
|
515
|
+
*/
|
|
516
|
+
reasoning_effort?: string;
|
|
517
|
+
/**
|
|
518
|
+
* The provider whose sandbox the analyzer boots — the job lineup, the same
|
|
519
|
+
* vocabulary as `JobCreate.sandbox_provider` and held to the same rule: an
|
|
520
|
+
* unknown value is refused `invalid_input` naming the lineup. Stored as
|
|
521
|
+
* given and honored wherever this config enqueues an analysis — every
|
|
522
|
+
* embedded analysis of the job, or the manual wave this body configures.
|
|
523
|
+
* Omitted, the platform's analysis default applies at each enqueue
|
|
524
|
+
* (daytona unless the operator retuned the fleet) — the value the resolved
|
|
525
|
+
* `AnalyzeConfig.sandbox_provider` echo reports.
|
|
526
|
+
*/
|
|
527
|
+
sandbox_provider?: EvalSandboxProvider;
|
|
528
|
+
/**
|
|
529
|
+
* How many of this wave's analyses run at once — Harbor's
|
|
530
|
+
* `-n/--n-concurrent` ("Max concurrent trial analyses", their
|
|
531
|
+
* cli/analyze.py:278-280). Bounded by the organization's
|
|
532
|
+
* `max_concurrent_analyses` at every claim: the job never holds more
|
|
533
|
+
* than the smaller of the two RUNNING fleet-wide. Omitted, the
|
|
534
|
+
* organization's ceiling alone bounds the wave (its fleet default is 60,
|
|
535
|
+
* fifteen times Harbor's own default of 4) and the resolved echo reads `null`. An integer in
|
|
536
|
+
* `[1, 150]`; anything else is refused `invalid_input` naming
|
|
537
|
+
* `analyze.n_concurrent`.
|
|
538
|
+
*/
|
|
539
|
+
n_concurrent?: number;
|
|
540
|
+
/**
|
|
541
|
+
* Analyze only the passing trials — Harbor's `--passing` ("Only analyze
|
|
542
|
+
* passing trials (reward=1.0)", their cli/analyze.py:282-284): a trial
|
|
543
|
+
* passes when it is SCORED with a primary reward of exactly 1. Mutually
|
|
544
|
+
* exclusive with `failing`: both true is refused `invalid_input` —
|
|
545
|
+
* Harbor's own "Cannot use both --passing and --failing".
|
|
546
|
+
*/
|
|
547
|
+
passing?: boolean;
|
|
548
|
+
/**
|
|
549
|
+
* Analyze only the failing trials — Harbor's `--failing` ("Only analyze
|
|
550
|
+
* failing trials (reward<1.0 or exception)", their cli/analyze.py:285-287):
|
|
551
|
+
* every analyzable trial that is not passing — a reward below 1 or none,
|
|
552
|
+
* and every error status. CANCELLED trials are never analyzed under
|
|
553
|
+
* either filter.
|
|
554
|
+
*/
|
|
555
|
+
failing?: boolean;
|
|
556
|
+
/**
|
|
557
|
+
* At most this many trials get an analysis — Harbor's `-l/--n-trials`
|
|
558
|
+
* ("Max trials to analyze", their cli/analyze.py:288-290), applied AFTER
|
|
559
|
+
* the reward filter: on the manual wave the first `n_trials` matching
|
|
560
|
+
* trials in the job's trial order; on the embedded trigger the first
|
|
561
|
+
* `n_trials` matching trials to settle. An integer of at least 1;
|
|
562
|
+
* anything else is refused `invalid_input` naming `analyze.n_trials`.
|
|
563
|
+
*/
|
|
564
|
+
n_trials?: number;
|
|
565
|
+
}
|
|
566
|
+
/**
|
|
567
|
+
* The RESOLVED trace-analysis policy — the caller's values or the defaults of
|
|
568
|
+
* the day, resolved at accept and stored, so the record always states the
|
|
569
|
+
* policy it executes (same law as RetryConfig). Echoed on the job body when
|
|
570
|
+
* the job was created with `analyze`; each analysis additionally carries the
|
|
571
|
+
* exact policy IT ran under (`Trial.analysis.model_name` / `.rubric` /
|
|
572
|
+
* `.prompt`), which a later manual re-analysis may have changed.
|
|
573
|
+
*/
|
|
574
|
+
interface AnalyzeConfig {
|
|
575
|
+
model_name: string;
|
|
576
|
+
rubric: Rubric;
|
|
577
|
+
/** The caller's prompt template as stored; null = the platform's default analyze body. */
|
|
578
|
+
prompt: string | null;
|
|
579
|
+
/**
|
|
580
|
+
* The effort this policy's analyses run at. Named at create it is served
|
|
581
|
+
* as stored; when the create named none, this echoes the per-model
|
|
582
|
+
* default of the day for `model_name` — the value the next enqueue under
|
|
583
|
+
* this policy stamps (the same nuance as `sandbox_provider` below).
|
|
584
|
+
*/
|
|
585
|
+
reasoning_effort: string;
|
|
586
|
+
/**
|
|
587
|
+
* The provider this policy's analyses run on. Named at create it is served
|
|
588
|
+
* as stored, forever. When the create named none, this echoes the
|
|
589
|
+
* platform's analysis default OF THE DAY — the value the next enqueue
|
|
590
|
+
* under this policy would stamp — because that default is an operator
|
|
591
|
+
* fleet knob, resolved where an analysis is actually enqueued rather than
|
|
592
|
+
* baked into the stored policy (the one deliberate nuance to the
|
|
593
|
+
* resolved-at-accept law above, stated so the echo is never read as
|
|
594
|
+
* history).
|
|
595
|
+
*/
|
|
596
|
+
sandbox_provider: EvalSandboxProvider;
|
|
597
|
+
/**
|
|
598
|
+
* The per-job width this policy's analyses are claimed under
|
|
599
|
+
* (`AnalyzeConfigInput.n_concurrent`, as stored); null = none named, the
|
|
600
|
+
* organization's `max_concurrent_analyses` alone bounds the wave.
|
|
601
|
+
*/
|
|
602
|
+
n_concurrent: number | null;
|
|
603
|
+
/** The reward filter as stored; both `passing` and `failing` false = every analyzable trial. */
|
|
604
|
+
passing: boolean;
|
|
605
|
+
failing: boolean;
|
|
606
|
+
/** The trial cap as stored (`AnalyzeConfigInput.n_trials`); null = no cap. */
|
|
607
|
+
n_trials: number | null;
|
|
608
|
+
}
|
|
609
|
+
/** The job-creation body — POST /api/jobs. */
|
|
610
|
+
interface JobCreate {
|
|
611
|
+
/** User-facing label; server-generated when omitted. */
|
|
612
|
+
job_name?: string;
|
|
613
|
+
datasets: DatasetSelector[];
|
|
614
|
+
agents: AgentArmInput[];
|
|
615
|
+
/** Attempts per task per agent arm (default 1, max 100). */
|
|
616
|
+
n_attempts?: number;
|
|
617
|
+
/** Parallel trials across the job (default 4, max 16). */
|
|
618
|
+
n_concurrent_trials?: number;
|
|
619
|
+
/**
|
|
620
|
+
* Per-trial spend cap in USD, minted onto each trial's gateway key — the
|
|
621
|
+
* platform's ONLY spend enforcement (there is no job-wide budget). Omitted,
|
|
622
|
+
* the server applies its published default ($200 unless the operator tuned
|
|
623
|
+
* it); the response echoes the RESOLVED cap either way, and states the
|
|
624
|
+
* resulting worst case for the job as a whole.
|
|
625
|
+
*/
|
|
626
|
+
max_trial_spend_usd?: number;
|
|
627
|
+
/** Sandbox provider to run on (optional; server default: `daytona`). */
|
|
628
|
+
sandbox_provider?: EvalSandboxProvider;
|
|
629
|
+
/** Auto-retry policy (Harbor RetryConfig grammar); omitted = the fleet defaults. */
|
|
630
|
+
retry?: RetryConfigInput;
|
|
631
|
+
/**
|
|
632
|
+
* Trace-analysis policy (Harbor's `harbor analyze` vocabulary). PRESENCE
|
|
633
|
+
* arms the embedded trigger: each trial is analyzed server-side right after
|
|
634
|
+
* it settles (CANCELLED trials are skipped — withdrawn work is not billed
|
|
635
|
+
* an uninvited analysis). `{}` means "analyze with all defaults"; omitted
|
|
636
|
+
* means no embedded analysis — `jobs().analyze()` remains the manual door.
|
|
637
|
+
*/
|
|
638
|
+
analyze?: AnalyzeConfigInput;
|
|
639
|
+
/**
|
|
640
|
+
* Multiplier for task timeouts — Harbor's `--timeout-multiplier`, all five
|
|
641
|
+
* fields flat on this body exactly as Harbor's JobConfig carries them. The
|
|
642
|
+
* worker multiplies each TASK-DECLARED timeout at the point that phase's
|
|
643
|
+
* timeout is armed; the task itself is never rewritten, so the same task
|
|
644
|
+
* runs unstretched in every other job. Values below 1 shrink, as in
|
|
645
|
+
* Harbor. Every multiplier must be a finite number greater than 0 —
|
|
646
|
+
* Harbor's own rule and nothing more; no ceiling of the platform's — and
|
|
647
|
+
* a zero, negative or non-finite value is refused with a typed
|
|
648
|
+
* `invalid_input` naming the rule, never silently clamped. The one real
|
|
649
|
+
* bound is the runtime's timer ceiling (2,147,483,647 ms, about 24.86
|
|
650
|
+
* days — Node sets a longer timer to 1 ms): every selected task's
|
|
651
|
+
* declared timeout x its phase's effective multiplier is checked at
|
|
652
|
+
* create and a product past it is refused `invalid_input` on the field
|
|
653
|
+
* that set the multiplier, naming the task, the phase, the product and
|
|
654
|
+
* the source. Default 1.0.
|
|
655
|
+
*/
|
|
656
|
+
timeout_multiplier?: number;
|
|
657
|
+
/** Multiplier for the agent execution timeout (overrides timeout_multiplier). */
|
|
658
|
+
agent_timeout_multiplier?: number;
|
|
659
|
+
/** Multiplier for the verifier timeout (overrides timeout_multiplier). */
|
|
660
|
+
verifier_timeout_multiplier?: number;
|
|
661
|
+
/** Multiplier for the agent setup timeout (overrides timeout_multiplier). */
|
|
662
|
+
agent_setup_timeout_multiplier?: number;
|
|
663
|
+
/** Multiplier for the environment build timeout (overrides timeout_multiplier). */
|
|
664
|
+
environment_build_timeout_multiplier?: number;
|
|
665
|
+
/**
|
|
666
|
+
* Env injected into every agent run — a pass-through slot: the client sends
|
|
667
|
+
* it verbatim and the server owns acceptance (refused where unsupported,
|
|
668
|
+
* never silently dropped).
|
|
669
|
+
*/
|
|
670
|
+
agent_env?: Record<string, string>;
|
|
671
|
+
/**
|
|
672
|
+
* Env injected into every verifier run — same pass-through contract. The
|
|
673
|
+
* hosted platform honors exactly two keys, Harbor rewardkit's per-run judge
|
|
674
|
+
* override (their `--ve` mechanism): `REWARDKIT_JUDGE` overwrites the
|
|
675
|
+
* rubric's `[judge].judge` field and `REWARDKIT_MODEL` overwrites its
|
|
676
|
+
* `[judge].model` field when the judge is an agent. Both are delivered into
|
|
677
|
+
* the verifier environment in both verifier modes, over any task-declared
|
|
678
|
+
* value of the same name; any other key is refused at create.
|
|
679
|
+
*/
|
|
680
|
+
verifier_env?: Record<string, string>;
|
|
681
|
+
/**
|
|
682
|
+
* Env secrets to deliver into every agent run — REFERENCES to the
|
|
683
|
+
* caller's own stored env secrets, plus INLINE entries ({name, value,
|
|
684
|
+
* delivery, label?, as?}) whose values are saved into the vault as
|
|
685
|
+
* normal env secrets first and then pinned like any other reference
|
|
686
|
+
* (WIRE LAW: the stored job never contains a value; a (name, label)
|
|
687
|
+
* collision splits on proof — a byte-equal restatement of the stored
|
|
688
|
+
* row, same value and delivery, attaches it so retries of the same
|
|
689
|
+
* request converge, while a different value or delivery is the typed
|
|
690
|
+
* `secret_exists` refusal — attach by reference or pick a label, never
|
|
691
|
+
* a silent overwrite). References are resolved at
|
|
692
|
+
* create and pinned: an omitted `label` takes the 'default'-labeled row
|
|
693
|
+
* when one exists (the single row when exactly one exists), and a bare
|
|
694
|
+
* name matching several labels with no 'default' is the typed
|
|
695
|
+
* `secret_ambiguous` refusal naming the labels — a job never guesses
|
|
696
|
+
* which secret it runs with. `as` renames the env var inside the
|
|
697
|
+
* sandbox; names the trial contract owns (the EVOLVE_ prefix,
|
|
698
|
+
* gateway/vendor key slots, the judge-override pair) are refused.
|
|
699
|
+
* DELIVERY MODES: every stored env secret carries `delivery` —
|
|
700
|
+
* 'brokered' (the value never enters any sandbox; the managed-agents
|
|
701
|
+
* egress-proxy machinery) or 'direct' (the raw value is placed in the
|
|
702
|
+
* sandbox environment). Eval trials deliver exactly the DIRECT mode:
|
|
703
|
+
* the value enters the trial env and is scrubbed at the credential seal,
|
|
704
|
+
* before hidden tests enter. Attaching a brokered secret is the typed
|
|
705
|
+
* `secret_brokered_unsupported` refusal at create — never a silent
|
|
706
|
+
* downgrade.
|
|
707
|
+
*/
|
|
708
|
+
secrets?: Array<JobSecretRef | JobSecretInline>;
|
|
709
|
+
}
|
|
710
|
+
/**
|
|
711
|
+
* One attached env secret: a reference to a stored secret of the caller's,
|
|
712
|
+
* by name and optional label, with an optional in-sandbox rename. The same
|
|
713
|
+
* {name, label?, as?} shape as the managed-agents lane's ManagedSecretRef,
|
|
714
|
+
* resolved by the same server-side law.
|
|
715
|
+
*/
|
|
716
|
+
interface JobSecretRef {
|
|
717
|
+
/** The stored secret's name (env-var-shaped; the EVOLVE_ prefix is reserved). */
|
|
718
|
+
name: string;
|
|
719
|
+
/** Which labeled row of that name; omitted = 'default' resolution law. */
|
|
720
|
+
label?: string;
|
|
721
|
+
/** Env var the value lands under in the sandbox (default: the name). */
|
|
722
|
+
as?: string;
|
|
723
|
+
}
|
|
724
|
+
/**
|
|
725
|
+
* One INLINE env secret on a job create — the convenience door into the
|
|
726
|
+
* same vault, not a second wire shape for values: the value is saved as a
|
|
727
|
+
* normal env secret first (delivery as stated, `label` defaulting to
|
|
728
|
+
* 'default') and the job then stores only the reference. A (name, label)
|
|
729
|
+
* identity that is already a stored row splits on proof: a byte-equal
|
|
730
|
+
* restatement (same value, same delivery) attaches that row — retries of
|
|
731
|
+
* the same request converge — while a different value or delivery is the
|
|
732
|
+
* typed `secret_exists` refusal (409); `delivery: 'brokered'` refuses as
|
|
733
|
+
* `secret_brokered_unsupported` until eval trials can broker.
|
|
734
|
+
*/
|
|
735
|
+
interface JobSecretInline {
|
|
736
|
+
/** Same grammar and reserved-name law as JobSecretRef.name. */
|
|
737
|
+
name: string;
|
|
738
|
+
/** The secret value to vault (at most 190 bytes). Never stored on the job. */
|
|
739
|
+
value: string;
|
|
740
|
+
/** The saved secret's delivery mode — REQUIRED, no silent default. */
|
|
741
|
+
delivery: "brokered" | "direct";
|
|
742
|
+
/** The labeled row to claim in the vault (default 'default'). */
|
|
743
|
+
label?: string;
|
|
744
|
+
/** Same in-sandbox rename law as JobSecretRef.as. */
|
|
745
|
+
as?: string;
|
|
746
|
+
}
|
|
747
|
+
/** Body of POST /api/jobs/{jobId}/resume. */
|
|
748
|
+
interface ResumeRequest {
|
|
749
|
+
/**
|
|
750
|
+
* Which failures to resume, matched against
|
|
751
|
+
* `exception_info.exception_type`. Omitted, the default set is
|
|
752
|
+
* ["ScoringError", "InfrastructureError", "ApiUsageLimitError",
|
|
753
|
+
* "IncompleteTrialError"] plus
|
|
754
|
+
* stopped trials (settled CANCELLED, exception type "CancelledError")
|
|
755
|
+
* and still-QUEUED trials of a cancelled source.
|
|
756
|
+
*/
|
|
757
|
+
filter_error_types?: string[];
|
|
758
|
+
}
|
|
759
|
+
/**
|
|
760
|
+
* Body of POST /api/jobs/{jobId}/retry — the selection, `trial_ids` XOR
|
|
761
|
+
* `failed_only`. Omitted (or `{}`) selects every trial of the (terminal)
|
|
762
|
+
* job; passing both fields is a contradiction the server refuses (400).
|
|
763
|
+
*/
|
|
764
|
+
interface RetryRequest {
|
|
765
|
+
/**
|
|
766
|
+
* Exactly these trials of the source job, all-or-nothing: an unknown id
|
|
767
|
+
* refuses the whole request (`trial_not_found`). Each named trial must be
|
|
768
|
+
* settled — SCORED, SCORING_ERROR, INFRASTRUCTURE_ERROR, BUDGET,
|
|
769
|
+
* INDETERMINATE, or CANCELLED (`trial_not_settled` otherwise) — but the JOB may still be
|
|
770
|
+
* running: a settled trial's facts are final. Duplicates are deduplicated.
|
|
771
|
+
*/
|
|
772
|
+
trial_ids?: string[];
|
|
773
|
+
/**
|
|
774
|
+
* Select the source's failed trials only (SCORING_ERROR,
|
|
775
|
+
* INFRASTRUCTURE_ERROR, BUDGET, INDETERMINATE). Stopped (CANCELLED) and scored
|
|
776
|
+
* trials are not failures — name them in `trial_ids`, or use resume for
|
|
777
|
+
* stopped work.
|
|
778
|
+
*/
|
|
779
|
+
failed_only?: boolean;
|
|
780
|
+
}
|
|
781
|
+
/**
|
|
782
|
+
* Optional filter narrowing which trials a job-level regrade re-runs.
|
|
783
|
+
* Omitted, every regradable trial is regraded.
|
|
784
|
+
*/
|
|
785
|
+
interface RegradeRequest {
|
|
786
|
+
statuses?: TrialStatus[];
|
|
787
|
+
/** Restrict to one task's trials. */
|
|
788
|
+
task_name?: string;
|
|
789
|
+
}
|
|
790
|
+
/**
|
|
791
|
+
* Per-(agent, model, dataset) statistics. The evals key format is
|
|
792
|
+
* `{agent}__{model}__{dataset}` — the dataset ref is always the LAST `__`
|
|
793
|
+
* segment, which is where Harbor-compatible readers recover it — with the
|
|
794
|
+
* platform extension of an `__{effort}` segment inserted BEFORE the dataset
|
|
795
|
+
* when a declared reasoning effort is part of the arm identity:
|
|
796
|
+
* `{agent}__{model}__{effort}__{dataset}`.
|
|
797
|
+
*/
|
|
798
|
+
interface AgentDatasetStats {
|
|
799
|
+
/** Trials that produced a rewards map — rewarded, not merely settled. */
|
|
800
|
+
n_trials?: number;
|
|
801
|
+
/** Trials carrying `exception_info` — indeterminate and cancelled included. */
|
|
802
|
+
n_errors?: number;
|
|
803
|
+
/**
|
|
804
|
+
* Metric results (a mean entry per arm today: the primary reward averaged
|
|
805
|
+
* over EVERY trial of the group, unrewarded trials counting 0); open objects.
|
|
806
|
+
*/
|
|
807
|
+
metrics?: Record<string, unknown>[];
|
|
808
|
+
/**
|
|
809
|
+
* pass@k for this group — the standard unbiased estimator
|
|
810
|
+
* `1 - C(n-c, k)/C(n, k)` per task, averaged over the group's tasks. Keys
|
|
811
|
+
* are k as strings (JSON object keys always are), values in [0,1]. The k set
|
|
812
|
+
* is the powers of two and the multiples of five up to the group's sparsest
|
|
813
|
+
* task's attempt count, so k=1 is never present and a single-attempt job
|
|
814
|
+
* answers `{}`. An attempt that produced no reward counts as a FAILED
|
|
815
|
+
* attempt, never an excluded one.
|
|
816
|
+
*
|
|
817
|
+
* `{}` means the group cannot answer: its rewards are not binary, no
|
|
818
|
+
* eligible k exists, or attempts are still in flight (the statistic appears
|
|
819
|
+
* once every attempt of the group has settled). `passAtK(job)` reads this
|
|
820
|
+
* field into sorted numeric points.
|
|
821
|
+
*/
|
|
822
|
+
pass_at_k?: Record<string, number>;
|
|
823
|
+
/** reward key -> reward value -> trial identifiers. */
|
|
824
|
+
reward_stats?: Record<string, Record<string, string[]>>;
|
|
825
|
+
/** exception type -> trial identifiers. */
|
|
826
|
+
exception_stats?: Record<string, string[]>;
|
|
827
|
+
}
|
|
828
|
+
/**
|
|
829
|
+
* Aggregate statistics of a job. Progress counters, token totals, and measured
|
|
830
|
+
* cost. The `n_*` counters are CUMULATIVE, Harbor-style: errored trials are a
|
|
831
|
+
* subset of completed, cancelled a subset of errored — a cancelled trial
|
|
832
|
+
* counts in all three. The disjoint per-status breakdown rides
|
|
833
|
+
* `Job.trials.byStatus`. `cost_usd` is what the trials actually spent so far —
|
|
834
|
+
* reporting, never a gate (enforcement is the per-trial cap).
|
|
835
|
+
*/
|
|
836
|
+
interface JobStats {
|
|
837
|
+
/** Cumulative: every trial that produced a result — errored and cancelled included. */
|
|
838
|
+
n_completed_trials?: number;
|
|
839
|
+
/** Cumulative: every completed trial carrying `exception_info`, cancelled included. */
|
|
840
|
+
n_errored_trials?: number;
|
|
841
|
+
n_running_trials?: number;
|
|
842
|
+
n_pending_trials?: number;
|
|
843
|
+
/** A subset of `n_errored_trials`. */
|
|
844
|
+
n_cancelled_trials?: number;
|
|
845
|
+
n_retries?: number;
|
|
846
|
+
/** Keyed `{agent}__{model}__{dataset}` — dataset ref last, optional effort segment before it. */
|
|
847
|
+
evals?: Record<string, AgentDatasetStats>;
|
|
848
|
+
/** Total input tokens (cache included); null until recorded. */
|
|
849
|
+
n_input_tokens?: number | null;
|
|
850
|
+
n_cache_tokens?: number | null;
|
|
851
|
+
n_output_tokens?: number | null;
|
|
852
|
+
/**
|
|
853
|
+
* Measured spend across settled trials — the WHOLE model bill, agent and
|
|
854
|
+
* judge together; null before any settled.
|
|
855
|
+
*/
|
|
856
|
+
cost_usd?: number | null;
|
|
857
|
+
/**
|
|
858
|
+
* Sum of the job's per-trial GPU compute ESTIMATES (each trial's
|
|
859
|
+
* `gpu_cost.estimate_usd`) — a SEPARATE labeled figure, never merged into
|
|
860
|
+
* `cost_usd` (metered model spend). Null when no trial of the job carries
|
|
861
|
+
* an estimate; a real $0 (a GPU trial that provably never booted a
|
|
862
|
+
* sandbox) keeps the sum non-null. Absent on servers predating the field.
|
|
863
|
+
*/
|
|
864
|
+
gpu_cost_usd?: number | null;
|
|
865
|
+
/**
|
|
866
|
+
* The judge share of `cost_usd`, itemized: what the trials' verifier-phase
|
|
867
|
+
* judge keys spent. 0 for a job with no judge-enabled tasks; null before
|
|
868
|
+
* anything settled, like `cost_usd`.
|
|
869
|
+
*/
|
|
870
|
+
judge_cost_usd?: number | null;
|
|
871
|
+
/**
|
|
872
|
+
* HOW MANY SETTLED TRIALS `cost_usd` CANNOT ACCOUNT FOR — trials whose
|
|
873
|
+
* `spend_source` lane is `assumed_cap`, meaning nobody ever measured their
|
|
874
|
+
* gateway spend. Such a trial stores 0, and the job total is the sum of its
|
|
875
|
+
* trials, so each one folds a zero in and `cost_usd` comes out LOWER than
|
|
876
|
+
* what was really spent. A plain count, never null: before anything settles
|
|
877
|
+
* it is honestly 0, where `cost_usd` is null. It is a FLOOR — a
|
|
878
|
+
* retried-away attempt's lineage snapshot keeps no spend-source column, so
|
|
879
|
+
* an earlier attempt nobody measured cannot be counted here. Absent on
|
|
880
|
+
* servers predating the field.
|
|
881
|
+
*/
|
|
882
|
+
n_unmeasured_trials?: number;
|
|
883
|
+
/**
|
|
884
|
+
* The judge half of the same fact, itemized the way `judge_cost_usd`
|
|
885
|
+
* itemizes `cost_usd`: trials that ran a judge whose spend was never
|
|
886
|
+
* measured (`judge_spend_source` `assumed_cap`). 0 on jobs with no
|
|
887
|
+
* judge-enabled tasks.
|
|
888
|
+
*/
|
|
889
|
+
n_unmeasured_judge_trials?: number;
|
|
890
|
+
/**
|
|
891
|
+
* Aggregate of the job's trace analyses; null when no trial of this job has
|
|
892
|
+
* ever been analyzed. Never a fabricated empty object — absence of analysis
|
|
893
|
+
* is stated as null, here and on each trial.
|
|
894
|
+
*/
|
|
895
|
+
analysis?: JobAnalysisStats | null;
|
|
896
|
+
}
|
|
897
|
+
/**
|
|
898
|
+
* The job-level analysis aggregate — Harbor's job `analysis.json` is a flat
|
|
899
|
+
* list of per-trial results (their analyze/models.py AnalyzeReport); each
|
|
900
|
+
* trial's own result rides `Trial.analysis`, and this object aggregates them.
|
|
901
|
+
* LATEST-per-trial: a re-analyzed trial contributes only its newest analysis,
|
|
902
|
+
* matching Harbor, where a re-run overwrites the trial directory's
|
|
903
|
+
* `analysis.json`.
|
|
904
|
+
*/
|
|
905
|
+
interface JobAnalysisStats {
|
|
906
|
+
/** Trials whose latest analysis produced a valid result. */
|
|
907
|
+
n_completed: number;
|
|
908
|
+
/**
|
|
909
|
+
* Trials whose latest analysis is a stored typed failure (invalid result,
|
|
910
|
+
* or an infrastructure failure of the analyzer run).
|
|
911
|
+
*/
|
|
912
|
+
n_failed: number;
|
|
913
|
+
/** Trials whose latest analysis is still queued or running. */
|
|
914
|
+
n_pending: number;
|
|
915
|
+
/**
|
|
916
|
+
* Measured spend of the LATEST analyses summed — the analyzer's own metered
|
|
917
|
+
* line, never part of `stats.cost_usd`. Null when no analysis recorded
|
|
918
|
+
* measured spend (mirrors Harbor's estimated_total_cost_usd, which is None
|
|
919
|
+
* when nothing was recorded).
|
|
920
|
+
*/
|
|
921
|
+
cost_usd: number | null;
|
|
922
|
+
/**
|
|
923
|
+
* Per-criterion outcome tally over the completed latest analyses, keyed by
|
|
924
|
+
* criterion name. Criteria are whatever the contributing rubrics named —
|
|
925
|
+
* after a re-analysis under a different rubric, keys from both may appear,
|
|
926
|
+
* each counting only analyses that carried it.
|
|
927
|
+
*/
|
|
928
|
+
checks: Record<string, {
|
|
929
|
+
n_pass: number;
|
|
930
|
+
n_fail: number;
|
|
931
|
+
n_not_applicable: number;
|
|
932
|
+
n_unknown: number;
|
|
933
|
+
}>;
|
|
934
|
+
}
|
|
935
|
+
/**
|
|
936
|
+
* GPU COMPUTE ESTIMATE of one trial — present on settled GPU trials only,
|
|
937
|
+
* null on every other trial. The estimate is the trial's MEASURED
|
|
938
|
+
* agent-sandbox lifetime multiplied by a versioned, source-dated rate-card
|
|
939
|
+
* row (the provider's public list price per GPU-second, times `gpu_count`).
|
|
940
|
+
* A SEPARATE labeled figure by law: never merged into
|
|
941
|
+
* `agent_result.cost_usd`, which is metered model spend. Exactly one of
|
|
942
|
+
* `estimate_usd` / `unpriced_reason` is set — an unmeasurable lifetime (a
|
|
943
|
+
* reaped run) or a request that let the provider choose the device (`any`,
|
|
944
|
+
* or several candidates) states its reason instead of a guessed number, and
|
|
945
|
+
* a GPU trial that provably never booted a sandbox carries a real
|
|
946
|
+
* `estimate_usd: 0`.
|
|
947
|
+
*
|
|
948
|
+
* WHICH DEVICE IS PRICED: the type the provider reported pinned to the box
|
|
949
|
+
* (`attached_gpu_type`) when it reported one, else the ONE type the create
|
|
950
|
+
* request carried (`resolved_gpu_types`: modal reserves exactly the task's
|
|
951
|
+
* first spelling; daytona receives the task's types in its own names, in
|
|
952
|
+
* order, and pins the first with capacity). The three provenance fields
|
|
953
|
+
* make the choice auditable; records priced under `rate_card.version` 1
|
|
954
|
+
* predate them and serve `declared_gpu_types` as the single spelling they
|
|
955
|
+
* kept, the other two null.
|
|
956
|
+
*/
|
|
957
|
+
interface TrialGpuCost {
|
|
958
|
+
/** The estimate, USD, micro-dollar resolution. Null exactly when `unpriced_reason` is set. */
|
|
959
|
+
estimate_usd: number | null;
|
|
960
|
+
/** Why no estimate exists. Null exactly when `estimate_usd` is set. */
|
|
961
|
+
unpriced_reason: string | null;
|
|
962
|
+
provider: EvalSandboxProvider;
|
|
963
|
+
/**
|
|
964
|
+
* The rate card's billing name the rate was looked up under (e.g. `H100`):
|
|
965
|
+
* the attached type when reported, else the one type the request carried.
|
|
966
|
+
* Null when no single device type is known.
|
|
967
|
+
*/
|
|
968
|
+
gpu_type: string | null;
|
|
969
|
+
/** The task's own `gpu_types` list, verbatim; null when it named none (any type). */
|
|
970
|
+
declared_gpu_types: string[] | null;
|
|
971
|
+
/**
|
|
972
|
+
* What the create request carried for the provider, in the provider's
|
|
973
|
+
* spelling: modal's one reservation, daytona's ordered candidates. Null
|
|
974
|
+
* when no constraint travelled, and on rate-card v1 records.
|
|
975
|
+
*/
|
|
976
|
+
resolved_gpu_types: string[] | null;
|
|
977
|
+
/** The device type the provider reported pinned to the box; null when none was reported. */
|
|
978
|
+
attached_gpu_type: string | null;
|
|
979
|
+
gpu_count: number;
|
|
980
|
+
/** Measured sandbox lifetime, fractional seconds; null when unmeasured. */
|
|
981
|
+
duration_sec: number | null;
|
|
982
|
+
/** The applied list price per GPU per second; null when no rate applied. */
|
|
983
|
+
rate_usd_per_gpu_sec: number | null;
|
|
984
|
+
/** Which card priced this trial: version, provider pricing page, and the date it was read. */
|
|
985
|
+
rate_card: {
|
|
986
|
+
version: number;
|
|
987
|
+
source: string | null;
|
|
988
|
+
source_date: string | null;
|
|
989
|
+
};
|
|
990
|
+
/** Observed sandbox birth (ISO); null when unmeasured. */
|
|
991
|
+
measured_from: string | null;
|
|
992
|
+
/** Observed sandbox end (ISO); null when unmeasured. */
|
|
993
|
+
measured_to: string | null;
|
|
994
|
+
}
|
|
995
|
+
/** One pass@k number: the estimate over `k` attempts. */
|
|
996
|
+
interface PassAtKPoint {
|
|
997
|
+
/** How many attempts the estimate is over — always 2 or more. */
|
|
998
|
+
k: number;
|
|
999
|
+
/** Probability that k attempts contain at least one success, in [0,1]. */
|
|
1000
|
+
value: number;
|
|
1001
|
+
}
|
|
1002
|
+
/** One evals group's pass@k curve, ready to plot or print. */
|
|
1003
|
+
interface PassAtKGroup {
|
|
1004
|
+
/** The `stats.evals` key these numbers belong to. */
|
|
1005
|
+
evals_key: string;
|
|
1006
|
+
/** Ascending by k; never empty (a group with no numbers is not returned). */
|
|
1007
|
+
points: PassAtKPoint[];
|
|
1008
|
+
}
|
|
1009
|
+
/**
|
|
1010
|
+
* Read a job's pass@k out of `stats.evals`, as numbers instead of the wire's
|
|
1011
|
+
* string keys. Groups that cannot answer (empty `pass_at_k` — non-binary
|
|
1012
|
+
* rewards, no eligible k, or attempts still in flight) are left out entirely,
|
|
1013
|
+
* so an empty array means "this job has no pass@k to show", and the shape is
|
|
1014
|
+
* the same whether the job is running or finished.
|
|
1015
|
+
*
|
|
1016
|
+
* Pure reading: no request is made, nothing is recomputed. The numbers are the
|
|
1017
|
+
* platform's, and the same ones the job's download archive carries.
|
|
1018
|
+
*/
|
|
1019
|
+
declare function passAtK(job: Job): PassAtKGroup[];
|
|
1020
|
+
/**
|
|
1021
|
+
* Provenance of an UPLOADED job (`jobs().upload()`) — what the archive's own
|
|
1022
|
+
* record files said about themselves: the job id its result.json carried and
|
|
1023
|
+
* the job_name its config.json carried (each null when the file did not state
|
|
1024
|
+
* one — never fabricated), plus when the platform ingested it. The
|
|
1025
|
+
* platform-minted row ids replace the archive's ids everywhere else on the
|
|
1026
|
+
* surface; these fields are where the originals remain readable. Null on
|
|
1027
|
+
* every job this platform executed; non-null marks a terminal RECORD — resume,
|
|
1028
|
+
* retry and regrade refuse it (`job_uploaded`), analyze works on it unchanged.
|
|
1029
|
+
*/
|
|
1030
|
+
interface UploadProvenance {
|
|
1031
|
+
original_job_id: string | null;
|
|
1032
|
+
original_job_name: string | null;
|
|
1033
|
+
uploaded_at: string;
|
|
1034
|
+
/**
|
|
1035
|
+
* The job-level sum of the trials' `upload.reported_agent_result` figures
|
|
1036
|
+
* — the uploader's own claims aggregated once at ingest, REPORTED like
|
|
1037
|
+
* their per-trial parts and never entering the platform-metered fields
|
|
1038
|
+
* (`stats.cost_usd` and the token stats stay null for an uploaded job).
|
|
1039
|
+
* Each total sums the trials that reported that field and is null when
|
|
1040
|
+
* none did (a zero would be a claim); `n_trials_reporting` counts the
|
|
1041
|
+
* trials that carried any reported figure, against the job's
|
|
1042
|
+
* `n_total_trials` — the honesty note for a partially reporting archive.
|
|
1043
|
+
* Null only on jobs ingested before this field existed.
|
|
1044
|
+
*/
|
|
1045
|
+
reported_totals: {
|
|
1046
|
+
cost_usd: number | null;
|
|
1047
|
+
n_input_tokens: number | null;
|
|
1048
|
+
n_cache_tokens: number | null;
|
|
1049
|
+
n_output_tokens: number | null;
|
|
1050
|
+
n_trials_reporting: number;
|
|
1051
|
+
} | null;
|
|
1052
|
+
/**
|
|
1053
|
+
* One row per task of the uploaded job, in archive order (spec
|
|
1054
|
+
* JobTaskLink): how many of its trials linked to a stored task — and so
|
|
1055
|
+
* analyze with the task folder in `/app/task` — to which dataset version,
|
|
1056
|
+
* and why the rest did not. Null only on jobs ingested before the link
|
|
1057
|
+
* law existed (not backfillable, never guessed).
|
|
1058
|
+
*/
|
|
1059
|
+
task_links: JobTaskLink[] | null;
|
|
1060
|
+
}
|
|
1061
|
+
/**
|
|
1062
|
+
* How an uploaded trial was linked to a stored task — the ONE rule the
|
|
1063
|
+
* ingest ran for its job (spec TaskLinkedBy): `dataset_flag` (the `-d`
|
|
1064
|
+
* dataset, by task name — the override), `job_dataset_record` (the job's
|
|
1065
|
+
* own config.json named a dataset the caller can see; linked by Harbor's
|
|
1066
|
+
* task hash inside it), `task_hash` (nothing named; exactly one visible
|
|
1067
|
+
* dataset carries the hash), or `none` (not linked — `TaskLinkReason` says
|
|
1068
|
+
* why).
|
|
1069
|
+
*/
|
|
1070
|
+
declare const TASK_LINKED_BY: readonly ["dataset_flag", "job_dataset_record", "task_hash", "none"];
|
|
1071
|
+
type TaskLinkedBy = (typeof TASK_LINKED_BY)[number];
|
|
1072
|
+
/**
|
|
1073
|
+
* Why an uploaded trial was NOT linked to a stored task (spec
|
|
1074
|
+
* TaskLinkReason) — its analyses run without the task folder:
|
|
1075
|
+
* `hash_mismatch` (the named dataset holds the task name, but with different
|
|
1076
|
+
* bytes), `task_not_in_dataset` (the named version holds neither the name
|
|
1077
|
+
* nor the hash), `no_dataset_named` (nothing named and no lock digest —
|
|
1078
|
+
* nothing to match on), `dataset_ambiguous` (the same bytes are published in
|
|
1079
|
+
* two or more datasets; `candidates` names them, `-d` decides),
|
|
1080
|
+
* `no_hash_match` (no visible task carries the trial's digest),
|
|
1081
|
+
* `no_task_digest` (a dataset was named but the trial's archive carries no
|
|
1082
|
+
* lock digest; a name alone never links without `-d`).
|
|
1083
|
+
*/
|
|
1084
|
+
declare const TASK_LINK_REASONS: readonly ["hash_mismatch", "task_not_in_dataset", "no_dataset_named", "dataset_ambiguous", "no_hash_match", "no_task_digest"];
|
|
1085
|
+
type TaskLinkReason = (typeof TASK_LINK_REASONS)[number];
|
|
1086
|
+
/**
|
|
1087
|
+
* How THIS uploaded trial linked to a stored task, or why not (spec
|
|
1088
|
+
* TrialTaskLink). A linked trial names its `dataset` and `version`; an
|
|
1089
|
+
* unlinked one carries `link_reason`. `task_digest` is the trial's own
|
|
1090
|
+
* lock.json `task.digest` when its archive carried one (Harbor's
|
|
1091
|
+
* `sha256:<hex>`); `candidates` the `name@version` refs an ambiguous hash
|
|
1092
|
+
* matched ([] otherwise).
|
|
1093
|
+
*/
|
|
1094
|
+
interface TrialTaskLink {
|
|
1095
|
+
linked_by: TaskLinkedBy;
|
|
1096
|
+
/** Set exactly when `linked_by` is `none`. */
|
|
1097
|
+
link_reason: TaskLinkReason | null;
|
|
1098
|
+
dataset: string | null;
|
|
1099
|
+
version: string | null;
|
|
1100
|
+
task_digest: string | null;
|
|
1101
|
+
candidates: string[];
|
|
1102
|
+
}
|
|
1103
|
+
/**
|
|
1104
|
+
* One task of an uploaded job, rolled up (spec JobTaskLink): its trial
|
|
1105
|
+
* count, how many linked (`n_linked` analyze with the task folder,
|
|
1106
|
+
* `n_unlinked` without), the rule that linked them, the `name@version` refs
|
|
1107
|
+
* they linked to (sorted; normally one), the unlinked trials per reason, and
|
|
1108
|
+
* the refs an ambiguous hash matched.
|
|
1109
|
+
*/
|
|
1110
|
+
interface JobTaskLink {
|
|
1111
|
+
/** The platform task key (the leaf of a registry-qualified name). */
|
|
1112
|
+
task_name: string;
|
|
1113
|
+
n_trials: number;
|
|
1114
|
+
n_linked: number;
|
|
1115
|
+
n_unlinked: number;
|
|
1116
|
+
/** The job's link rule when any trial of the task linked, else `none`. */
|
|
1117
|
+
linked_by: TaskLinkedBy;
|
|
1118
|
+
datasets: string[];
|
|
1119
|
+
/** Unlinked trials per `TaskLinkReason`. */
|
|
1120
|
+
link_reasons: Partial<Record<TaskLinkReason, number>>;
|
|
1121
|
+
candidates: string[];
|
|
1122
|
+
}
|
|
1123
|
+
/**
|
|
1124
|
+
* The deletion receipt of `DELETE /api/jobs/{jobId}`: what was destroyed.
|
|
1125
|
+
* The contract's own minimal shape — Harbor's hub delete answers no wire
|
|
1126
|
+
* body, so there was no shape to mirror. `trials_deleted` counts the trial
|
|
1127
|
+
* rows destroyed with the job (their trace events, attempts and stored
|
|
1128
|
+
* trace objects went with them); `analyses_deleted` the trial-analysis
|
|
1129
|
+
* rows, their stored analyzer streams included.
|
|
1130
|
+
*/
|
|
1131
|
+
interface JobDeleteResult {
|
|
1132
|
+
/** The deleted job. */
|
|
1133
|
+
job_id: string;
|
|
1134
|
+
trials_deleted: number;
|
|
1135
|
+
analyses_deleted: number;
|
|
1136
|
+
}
|
|
1137
|
+
/**
|
|
1138
|
+
* Why a job FAILED — deliberately NOT under the key `error`, which on this
|
|
1139
|
+
* surface always means "this request failed". `if (body.error) throw` stays
|
|
1140
|
+
* correct on a healthy 200 read of a failed job.
|
|
1141
|
+
*/
|
|
1142
|
+
interface JobFailure {
|
|
1143
|
+
/** `job_execution_failed` when the runner recorded no code. */
|
|
1144
|
+
code: string;
|
|
1145
|
+
message: string;
|
|
1146
|
+
}
|
|
1147
|
+
/**
|
|
1148
|
+
* One dataset of a job whose selection excluded tasks that FAILED to build
|
|
1149
|
+
* (the partial-publish model). `note` is the sentence to show, and the
|
|
1150
|
+
* structured fields beside it are the same fact for a UI: `n_tasks_selected`
|
|
1151
|
+
* is how many READY tasks the caller's filters matched BEFORE any `n_tasks`
|
|
1152
|
+
* cap, `n_tasks_ran` how many the job actually ran from this dataset (fewer
|
|
1153
|
+
* than `n_tasks_selected` only under an `n_tasks` cap), and
|
|
1154
|
+
* `n_tasks_failed_to_build` what the filters would have taken but the build
|
|
1155
|
+
* lost. Uncapped, the note reads "ran N of M tasks — K failed to build" with
|
|
1156
|
+
* M = n_tasks_selected + K; capped it reads "selection matched M tasks:
|
|
1157
|
+
* K failed to build: …; ran R (n_tasks cap)" — the run was short for two
|
|
1158
|
+
* separate reasons and the sentence keeps them apart. `failed_task_names`
|
|
1159
|
+
* names every one, sorted; the reasons live on the dataset's `failed_tasks`
|
|
1160
|
+
* and the per-task build route (datasets().getTaskBuild()). (Jobs recorded
|
|
1161
|
+
* before `n_tasks_selected` existed answer it as `n_tasks_ran` — read as
|
|
1162
|
+
* uncapped.)
|
|
1163
|
+
*/
|
|
1164
|
+
interface JobBuildExclusion {
|
|
1165
|
+
dataset: DatasetRef;
|
|
1166
|
+
n_tasks_ran: number;
|
|
1167
|
+
n_tasks_selected: number;
|
|
1168
|
+
n_tasks_failed_to_build: number;
|
|
1169
|
+
failed_task_names: string[];
|
|
1170
|
+
note: string;
|
|
1171
|
+
}
|
|
1172
|
+
/**
|
|
1173
|
+
* THE job body — the same shape from create, get, list items, cancel, resume,
|
|
1174
|
+
* and regrade responses; no field appears on some responses and not others.
|
|
1175
|
+
*/
|
|
1176
|
+
interface Job {
|
|
1177
|
+
id: string;
|
|
1178
|
+
/** User-facing label. */
|
|
1179
|
+
job_name: string;
|
|
1180
|
+
status: JobStatus;
|
|
1181
|
+
/** The resolved dataset references this job ran. */
|
|
1182
|
+
datasets: DatasetRef[];
|
|
1183
|
+
agents: AgentArm[];
|
|
1184
|
+
n_attempts: number;
|
|
1185
|
+
n_concurrent_trials: number;
|
|
1186
|
+
/** The resolved per-trial cap every trial key was minted with. */
|
|
1187
|
+
max_trial_spend_usd: number;
|
|
1188
|
+
/**
|
|
1189
|
+
* The most this job can cost: every trial spending its whole cap on every
|
|
1190
|
+
* attempt the retry policy allows — cap x trials x (retry.max_retries + 1),
|
|
1191
|
+
* since each attempt is minted its own full cap. Stated outright — the
|
|
1192
|
+
* per-trial cap is the only enforcement, so the product is the number
|
|
1193
|
+
* someone approving a 500-trial run actually needs to see.
|
|
1194
|
+
*/
|
|
1195
|
+
worst_case_spend_usd: number;
|
|
1196
|
+
/** The RESOLVED auto-retry policy this job runs under. */
|
|
1197
|
+
retry: RetryConfig;
|
|
1198
|
+
/**
|
|
1199
|
+
* The resolved embedded-analysis policy the job was created with; null when
|
|
1200
|
+
* the create named none (a later manual `analyze()` does not rewrite it —
|
|
1201
|
+
* the job row states what the CREATE asked for, each analysis states what
|
|
1202
|
+
* IT ran under). Always null on a regrade job.
|
|
1203
|
+
*/
|
|
1204
|
+
analyze: AnalyzeConfig | null;
|
|
1205
|
+
/**
|
|
1206
|
+
* The RESOLVED timeout multipliers this job's phases arm under — Harbor's
|
|
1207
|
+
* five flat JobConfig fields, echoed on every job body. The global one is
|
|
1208
|
+
* always a number (1.0 when the create request named none); each phase
|
|
1209
|
+
* field is null when not overridden, meaning the global applies.
|
|
1210
|
+
*/
|
|
1211
|
+
timeout_multiplier: number;
|
|
1212
|
+
agent_timeout_multiplier: number | null;
|
|
1213
|
+
verifier_timeout_multiplier: number | null;
|
|
1214
|
+
agent_setup_timeout_multiplier: number | null;
|
|
1215
|
+
environment_build_timeout_multiplier: number | null;
|
|
1216
|
+
/**
|
|
1217
|
+
* Where this job's trials execute. Null exactly on an UPLOADED job
|
|
1218
|
+
* (`upload` non-null): an ingested record never executed on any platform
|
|
1219
|
+
* sandbox, and naming a provider would be an execution claim. Never null
|
|
1220
|
+
* on a job this platform ran.
|
|
1221
|
+
*/
|
|
1222
|
+
sandbox_provider: EvalSandboxProvider | null;
|
|
1223
|
+
/** Entity cardinality only — things with no status of their own. */
|
|
1224
|
+
counts: {
|
|
1225
|
+
agents: number;
|
|
1226
|
+
tasks: number;
|
|
1227
|
+
};
|
|
1228
|
+
/**
|
|
1229
|
+
* THE RESULTS-HONESTY LABEL of the partial-publish model: one entry per
|
|
1230
|
+
* dataset of this job whose selection excluded tasks that FAILED to build
|
|
1231
|
+
* — a whole-dataset (or glob) run over a partially built version runs the
|
|
1232
|
+
* READY tasks, and this field is where the job says so plainly instead of
|
|
1233
|
+
* silently truncating. Always present; empty when nothing was excluded
|
|
1234
|
+
* (every selected dataset fully built, or every failed task was already
|
|
1235
|
+
* outside the caller's own filters). Recorded at create and immutable;
|
|
1236
|
+
* derived jobs (resume/retry) answer empty — their honesty lives on the
|
|
1237
|
+
* source job.
|
|
1238
|
+
*/
|
|
1239
|
+
build_exclusions: JobBuildExclusion[];
|
|
1240
|
+
n_total_trials: number;
|
|
1241
|
+
/** The zeros-included 8-status histogram, beside the coarser counters in `stats`. */
|
|
1242
|
+
trials: TrialStatusTally;
|
|
1243
|
+
stats: JobStats;
|
|
1244
|
+
/** Why the job FAILED, or null. Never the key `error` — see JobFailure. */
|
|
1245
|
+
failure: JobFailure | null;
|
|
1246
|
+
/** Empty for an original job. */
|
|
1247
|
+
source_jobs: SourceJob[];
|
|
1248
|
+
/** Derived: any source_jobs entry with action "regrade". */
|
|
1249
|
+
is_regrade: boolean;
|
|
1250
|
+
/**
|
|
1251
|
+
* The upload provenance echo — null for every job this platform executed,
|
|
1252
|
+
* non-null only on a job ingested by jobs().upload(). See UploadProvenance.
|
|
1253
|
+
*/
|
|
1254
|
+
upload: UploadProvenance | null;
|
|
1255
|
+
/** True only on a response that replayed an existing job for an Idempotency-Key. */
|
|
1256
|
+
idempotent_replay: boolean;
|
|
1257
|
+
started_at: string;
|
|
1258
|
+
updated_at: string;
|
|
1259
|
+
/** Null while the job is live. */
|
|
1260
|
+
finished_at: string | null;
|
|
1261
|
+
}
|
|
1262
|
+
/**
|
|
1263
|
+
* A phase's wall-clock as a start/stop pair (never a duration). Either bound
|
|
1264
|
+
* is null while the phase has not reached it.
|
|
1265
|
+
*/
|
|
1266
|
+
interface TimingInfo {
|
|
1267
|
+
started_at: string | null;
|
|
1268
|
+
finished_at: string | null;
|
|
1269
|
+
}
|
|
1270
|
+
interface ModelInfo {
|
|
1271
|
+
name: string;
|
|
1272
|
+
/** Null means "not specified", never "unknown provider". */
|
|
1273
|
+
provider?: string | null;
|
|
1274
|
+
}
|
|
1275
|
+
/**
|
|
1276
|
+
* The agent that ran a trial. `version` is the version actually RESOLVED and
|
|
1277
|
+
* used (null until resolved) — the requested pin lives on the job's
|
|
1278
|
+
* `agents[].version`. `reasoning_effort` is the platform's arm-identity
|
|
1279
|
+
* extension.
|
|
1280
|
+
*/
|
|
1281
|
+
interface AgentInfo {
|
|
1282
|
+
name: string;
|
|
1283
|
+
version: string | null;
|
|
1284
|
+
model_info: ModelInfo;
|
|
1285
|
+
reasoning_effort?: string | null;
|
|
1286
|
+
}
|
|
1287
|
+
/**
|
|
1288
|
+
* What the agent phase produced and consumed. `n_input_tokens` includes cache
|
|
1289
|
+
* tokens. `cost_usd` is the settled spend (see `spend_source` on the trial for
|
|
1290
|
+
* which lane it came from, and whether it is final). `metadata` carries open
|
|
1291
|
+
* per-run detail:
|
|
1292
|
+
* the harness bundle digest and runtime, the network mode the trial ran under
|
|
1293
|
+
* and where that decision came from, and any harness-reported usage detail.
|
|
1294
|
+
*/
|
|
1295
|
+
interface AgentResult {
|
|
1296
|
+
n_input_tokens?: number | null;
|
|
1297
|
+
n_cache_tokens?: number | null;
|
|
1298
|
+
n_output_tokens?: number | null;
|
|
1299
|
+
/** Null until the trial has executed; null never means $0. */
|
|
1300
|
+
cost_usd?: number | null;
|
|
1301
|
+
/** Reserved for token-level rollout detail; null today. */
|
|
1302
|
+
rollout_details?: Record<string, unknown>[] | null;
|
|
1303
|
+
metadata?: Record<string, unknown> | null;
|
|
1304
|
+
}
|
|
1305
|
+
/**
|
|
1306
|
+
* What the verifier phase's LLM judge consumed — the judge half of a trial's
|
|
1307
|
+
* model bill. A judge-enabled task's verifier holds a DISTINCT short-lived
|
|
1308
|
+
* gateway key (scoped to the requested credential's model family only, minted
|
|
1309
|
+
* at verify start, revoked after scoring; judge model selection itself is
|
|
1310
|
+
* Harbor-exact — the rubric names the model, or rewardkit's own library
|
|
1311
|
+
* default applies), and these figures are that key's spend and tokens as the platform
|
|
1312
|
+
* measured them at the gateway — never anything the verifier reported about
|
|
1313
|
+
* itself. `cost_usd` is the judge share alone; `agent_result.cost_usd` stays
|
|
1314
|
+
* the agent's, and the trial's whole bill is the sum. See `judge_spend_source`
|
|
1315
|
+
* on the trial for which lane the figure is in.
|
|
1316
|
+
*/
|
|
1317
|
+
interface JudgeResult {
|
|
1318
|
+
n_input_tokens?: number | null;
|
|
1319
|
+
n_cache_tokens?: number | null;
|
|
1320
|
+
n_output_tokens?: number | null;
|
|
1321
|
+
/** Null until measured; null never means $0. */
|
|
1322
|
+
cost_usd?: number | null;
|
|
1323
|
+
}
|
|
1324
|
+
/**
|
|
1325
|
+
* One place a verdict rests on. `where` names a step_id in the trajectory, a
|
|
1326
|
+
* file and line, or a command; `quote` is the exact text found there. The
|
|
1327
|
+
* page links each entry to the record; a reader verifies by eye — nothing
|
|
1328
|
+
* verifies the quotes mechanically.
|
|
1329
|
+
*/
|
|
1330
|
+
interface AnalysisEvidence {
|
|
1331
|
+
where: string;
|
|
1332
|
+
quote: string;
|
|
1333
|
+
}
|
|
1334
|
+
/**
|
|
1335
|
+
* One criterion's verdict — Harbor's QualityCheckModel (their
|
|
1336
|
+
* cli/quality_checker/models.py `{explanation, outcome}`) extended by the
|
|
1337
|
+
* platform's result schema: a fourth outcome, `unknown`, for a criterion the
|
|
1338
|
+
* record cannot decide (`not_applicable` keeps Harbor's meaning, no subject),
|
|
1339
|
+
* and the `evidence` list — at least one entry behind a `pass` or a `fail`;
|
|
1340
|
+
* `not_applicable` and `unknown` may carry none. Results stored before the
|
|
1341
|
+
* evidence field existed serve an empty list.
|
|
1342
|
+
*/
|
|
1343
|
+
interface AnalysisCheck {
|
|
1344
|
+
outcome: "pass" | "fail" | "not_applicable" | "unknown";
|
|
1345
|
+
/** The analyzer's reasoning, in plain words, opening with a few words that name what it found. */
|
|
1346
|
+
explanation: string;
|
|
1347
|
+
evidence: AnalysisEvidence[];
|
|
1348
|
+
}
|
|
1349
|
+
/**
|
|
1350
|
+
* The derived label of an analysis — computed by the platform from the
|
|
1351
|
+
* outcomes when the result is stored, never asked from the model. Null until
|
|
1352
|
+
* completed, and null on a completed analysis whose rubric is not the
|
|
1353
|
+
* default one (a custom rubric carries its per-criterion outcomes and no
|
|
1354
|
+
* label). Precedence: `flagged` on a fail of score_is_earned,
|
|
1355
|
+
* score_is_correct, task_was_fair or report_is_truthful; else `env_fault` on
|
|
1356
|
+
* a fail of environment_worked; else `unclear` on an unknown of any of those
|
|
1357
|
+
* five, or a not_applicable of score_is_earned or score_is_correct; else
|
|
1358
|
+
* `clean`.
|
|
1359
|
+
*/
|
|
1360
|
+
type AnalysisLabel = "flagged" | "env_fault" | "unclear" | "clean";
|
|
1361
|
+
/**
|
|
1362
|
+
* The derived label of a task check — computed the same way. `has_a_problem`
|
|
1363
|
+
* on a fail of any criterion; else `unclear` on an unknown of any of the six
|
|
1364
|
+
* file-based criteria; else `no_problem_found`. Null under a custom rubric.
|
|
1365
|
+
*/
|
|
1366
|
+
type CheckLabel = "has_a_problem" | "unclear" | "no_problem_found";
|
|
1367
|
+
/**
|
|
1368
|
+
* Why an analysis FAILED — a stored typed failure, never a silent absence and
|
|
1369
|
+
* never a fake pass. NOT under the key `error` for the same reason JobFailure
|
|
1370
|
+
* is not.
|
|
1371
|
+
*/
|
|
1372
|
+
interface AnalysisFailure {
|
|
1373
|
+
/**
|
|
1374
|
+
* Which part failed: `invalid_result` (the analyzer ran but its
|
|
1375
|
+
* analysis.json failed validation — the message preserves every validator
|
|
1376
|
+
* reason, one per line; a run cut by its budget is `timeout` instead),
|
|
1377
|
+
* `inputs` (the trial tree or task content could not be assembled),
|
|
1378
|
+
* `timeout` (the analyzer's run budget ran out with no valid analysis.json
|
|
1379
|
+
* — the file missing, or a partial one that failed validation, its reasons
|
|
1380
|
+
* in the message — never re-run: a timeout is deterministic; the message
|
|
1381
|
+
* names the budget, the seconds used and the exit code), or an
|
|
1382
|
+
* infrastructure stage of the analyzer run (`mint_key`, `boot`,
|
|
1383
|
+
* `harness_install`, `agent`, `artifact_read`, `lease_expired`, ...).
|
|
1384
|
+
*/
|
|
1385
|
+
phase: string;
|
|
1386
|
+
message: string;
|
|
1387
|
+
}
|
|
1388
|
+
/**
|
|
1389
|
+
* One trace analysis of a trial. The result half is Harbor's AnalyzeResult
|
|
1390
|
+
* shape (their analyze/models.py: `summary`, `checks` keyed by criterion,
|
|
1391
|
+
* `estimated_cost_usd`; the enclosing trial is Harbor's `trial_name`), its
|
|
1392
|
+
* checks extended by the result schema (four outcomes, an evidence list); the
|
|
1393
|
+
* rest is provenance — which model and rubric THIS analysis ran under, its
|
|
1394
|
+
* lifecycle status, and its typed failure when it failed.
|
|
1395
|
+
*
|
|
1396
|
+
* `estimated_cost_usd` is the analyzer agent's OWN metered spend (its sealed
|
|
1397
|
+
* sandbox runs on its own capped gateway key) — its own line, never part of
|
|
1398
|
+
* the trial's `agent_result.cost_usd` or the job's `stats.cost_usd`; the job
|
|
1399
|
+
* aggregate is `stats.analysis.cost_usd`. Null when nothing was measured,
|
|
1400
|
+
* never a fabricated 0.
|
|
1401
|
+
*/
|
|
1402
|
+
interface TrialAnalysis {
|
|
1403
|
+
id: string;
|
|
1404
|
+
/**
|
|
1405
|
+
* Provenance: the analyzed trial, its job, and its task. Redundant on
|
|
1406
|
+
* `Trial.analysis` (the trial is the enclosing object) and the whole point
|
|
1407
|
+
* of a `analyses().list()` row, where nothing else says which run the
|
|
1408
|
+
* verdict judged. Harbor's `trial_name` names the same thing by directory.
|
|
1409
|
+
*/
|
|
1410
|
+
trial_id: string;
|
|
1411
|
+
job_id: string;
|
|
1412
|
+
task_name: string;
|
|
1413
|
+
/**
|
|
1414
|
+
* Lifecycle of this analysis. Every non-terminal analysis reaches
|
|
1415
|
+
* `completed` or `failed`; a worker death mid-run is reaped to a typed
|
|
1416
|
+
* `failed`, never left `running` forever.
|
|
1417
|
+
*/
|
|
1418
|
+
status: AnalysisStatus;
|
|
1419
|
+
model_name: string;
|
|
1420
|
+
/**
|
|
1421
|
+
* The reasoning effort THIS analysis ran at — passed to the analyzer
|
|
1422
|
+
* explicitly, so it is what the model was asked for. Null only on
|
|
1423
|
+
* analyses recorded before the effort was stamped.
|
|
1424
|
+
*/
|
|
1425
|
+
reasoning_effort: string | null;
|
|
1426
|
+
rubric: Rubric;
|
|
1427
|
+
/**
|
|
1428
|
+
* The prompt template THIS analysis ran under, frozen at enqueue
|
|
1429
|
+
* (`AnalyzeConfigInput.prompt`); null = the platform's default analyze body.
|
|
1430
|
+
*/
|
|
1431
|
+
prompt: string | null;
|
|
1432
|
+
/**
|
|
1433
|
+
* 3–5 sentence overview of what happened during the trial (the default
|
|
1434
|
+
* output section's summary contract). Null until completed.
|
|
1435
|
+
*/
|
|
1436
|
+
summary: string | null;
|
|
1437
|
+
/**
|
|
1438
|
+
* One entry per rubric criterion, keys exactly the rubric's criterion names
|
|
1439
|
+
* (the frozen-criteria law). Null until completed.
|
|
1440
|
+
*/
|
|
1441
|
+
checks: Record<string, AnalysisCheck> | null;
|
|
1442
|
+
/** The derived label (AnalysisLabel states the rule); null until completed, and null under a custom rubric. */
|
|
1443
|
+
label: AnalysisLabel | null;
|
|
1444
|
+
estimated_cost_usd: number | null;
|
|
1445
|
+
/**
|
|
1446
|
+
* The analyzer's one-home usage reading — the SAME object, same keys, the
|
|
1447
|
+
* trial and session surfaces serve, built from the analyzer's OWN gateway
|
|
1448
|
+
* records. Present and ticking while the analysis runs — a mid-run reading
|
|
1449
|
+
* is a lagging LOWER BOUND, always `provisional: true` — and settled by
|
|
1450
|
+
* the same read that writes `estimated_cost_usd`, which stays Harbor's
|
|
1451
|
+
* word for the FINAL figure. Null = the meter never answered, never zero.
|
|
1452
|
+
* Absent on servers predating the field.
|
|
1453
|
+
*/
|
|
1454
|
+
usage?: UsageReading | null;
|
|
1455
|
+
/**
|
|
1456
|
+
* How many analyzer attempts this analysis has made: 1, or 2 when the
|
|
1457
|
+
* bounded automatic re-run fired. An analyzer run that completes without
|
|
1458
|
+
* producing a valid analysis.json (the missing file included) is re-run
|
|
1459
|
+
* at most once — same model, same frozen rubric, fresh sandbox — and a
|
|
1460
|
+
* second failure of that class settles `failed` with phase
|
|
1461
|
+
* `invalid_result`, both attempts recorded. A run cut by its budget is
|
|
1462
|
+
* not that class: it settles `failed` with phase `timeout` at once and is
|
|
1463
|
+
* never re-run. Infrastructure failures never auto re-run. When the
|
|
1464
|
+
* re-run fired, `estimated_cost_usd` and the token totals cover BOTH
|
|
1465
|
+
* attempts. Absent on servers predating the field.
|
|
1466
|
+
*/
|
|
1467
|
+
attempts?: number;
|
|
1468
|
+
/** Non-null exactly when status is `failed`. */
|
|
1469
|
+
failure: AnalysisFailure | null;
|
|
1470
|
+
/** When this analysis was enqueued. */
|
|
1471
|
+
created_at: string;
|
|
1472
|
+
/** When it settled; null while queued or running. */
|
|
1473
|
+
finished_at: string | null;
|
|
1474
|
+
}
|
|
1475
|
+
/**
|
|
1476
|
+
* The verifier's rewards map. The primary-reward convention: the value under
|
|
1477
|
+
* the key "reward"; else, when exactly one key exists, that value; else no
|
|
1478
|
+
* primary reward. Zero is a reward.
|
|
1479
|
+
*/
|
|
1480
|
+
interface VerifierResult {
|
|
1481
|
+
rewards?: Record<string, number> | null;
|
|
1482
|
+
}
|
|
1483
|
+
/**
|
|
1484
|
+
* Why a trial failed, when it did. `exception_type` is one of the platform's
|
|
1485
|
+
* stable failure names (ScoringError, InfrastructureError, ApiUsageLimitError,
|
|
1486
|
+
* CancelledError, IncompleteTrialError) — but filter with `Trial.status`,
|
|
1487
|
+
* which is the primary key for failure classes; this is the detail.
|
|
1488
|
+
*/
|
|
1489
|
+
interface ExceptionInfo {
|
|
1490
|
+
exception_type: string;
|
|
1491
|
+
/** Truncated to 2000 chars on list rows; full on the detail route. */
|
|
1492
|
+
exception_message: string;
|
|
1493
|
+
/** Empty when the platform recorded no traceback. */
|
|
1494
|
+
exception_traceback?: string;
|
|
1495
|
+
occurred_at: string;
|
|
1496
|
+
}
|
|
1497
|
+
/**
|
|
1498
|
+
* What ONE step of a multi-step task produced. Present on
|
|
1499
|
+
* `Trial.step_results`, one entry per step the trial actually RAN, in execution
|
|
1500
|
+
* order — a trial that stopped early (a step raised, or its `min_reward` was
|
|
1501
|
+
* not met) carries only the steps that ran, because the ones after an abort did
|
|
1502
|
+
* not happen.
|
|
1503
|
+
*
|
|
1504
|
+
* `verifier_result` null means this step produced no verifier result at all (it
|
|
1505
|
+
* crashed, the step aborted before reaching it, or verification was disabled);
|
|
1506
|
+
* a result whose `rewards` is null means the verifier ran and produced no
|
|
1507
|
+
* reward map. The distinction decides the trial's reward: the mean strategy
|
|
1508
|
+
* excludes a step with no result from its denominator, but counts a result with
|
|
1509
|
+
* no rewards as zero.
|
|
1510
|
+
*
|
|
1511
|
+
* `agent_result` is null when no per-step spend was measured — never a
|
|
1512
|
+
* per-step $0.
|
|
1513
|
+
*/
|
|
1514
|
+
interface StepResult {
|
|
1515
|
+
step_name?: string;
|
|
1516
|
+
agent_result?: AgentResult | null;
|
|
1517
|
+
verifier_result?: VerifierResult | null;
|
|
1518
|
+
exception_info?: ExceptionInfo | null;
|
|
1519
|
+
agent_execution?: TimingInfo | null;
|
|
1520
|
+
verifier?: TimingInfo | null;
|
|
1521
|
+
}
|
|
1522
|
+
/**
|
|
1523
|
+
* The ONE public trial shape, shared verbatim by list rows and the detail
|
|
1524
|
+
* route (detail returns `exception_info.exception_message` untruncated — the
|
|
1525
|
+
* only documented difference). A trial id is globally addressable; `job_id` is
|
|
1526
|
+
* the reverse pointer.
|
|
1527
|
+
*
|
|
1528
|
+
* Execution facts (`sandbox_provider`, `verifier_environment_mode`,
|
|
1529
|
+
* `agent_result.cost_usd`, `spend_source`) are null until the trial has
|
|
1530
|
+
* actually executed: a QUEUED or CANCELLED trial never ran, so null means
|
|
1531
|
+
* "did not run" and never zero.
|
|
1532
|
+
*/
|
|
1533
|
+
interface Trial {
|
|
1534
|
+
id: string;
|
|
1535
|
+
job_id: string;
|
|
1536
|
+
task_name: string;
|
|
1537
|
+
/** The dataset this trial's task came from. */
|
|
1538
|
+
source: string;
|
|
1539
|
+
agent_info: AgentInfo;
|
|
1540
|
+
/** Attempt index within the arm (1..n_attempts). */
|
|
1541
|
+
attempt: number;
|
|
1542
|
+
status: TrialStatus;
|
|
1543
|
+
/**
|
|
1544
|
+
* Convenience primary reward derived from `verifier_result.rewards` by the
|
|
1545
|
+
* primary-reward convention. Zero is a reward; null means the trial did not
|
|
1546
|
+
* score.
|
|
1547
|
+
*/
|
|
1548
|
+
reward: number | null;
|
|
1549
|
+
verifier_result: VerifierResult | null;
|
|
1550
|
+
exception_info: ExceptionInfo | null;
|
|
1551
|
+
agent_result: AgentResult | null;
|
|
1552
|
+
/**
|
|
1553
|
+
* The judge share of the trial's bill, itemized. Null when no judge ever
|
|
1554
|
+
* ran — the task requested no judge credential, or the trial never reached
|
|
1555
|
+
* its verify phase. Null never means "$0 of judging".
|
|
1556
|
+
*/
|
|
1557
|
+
judge_result?: JudgeResult | null;
|
|
1558
|
+
/**
|
|
1559
|
+
* The trial's LATEST trace analysis; null when the trial has never been
|
|
1560
|
+
* analyzed — never a fabricated empty object. A re-analysis (same job,
|
|
1561
|
+
* different rubric or model) replaces what this field serves, matching
|
|
1562
|
+
* Harbor, where a re-run overwrites the trial directory's analysis.json;
|
|
1563
|
+
* earlier analyses stay stored as the audit record.
|
|
1564
|
+
*/
|
|
1565
|
+
analysis?: TrialAnalysis | null;
|
|
1566
|
+
environment_setup: TimingInfo | null;
|
|
1567
|
+
agent_setup: TimingInfo | null;
|
|
1568
|
+
agent_execution: TimingInfo | null;
|
|
1569
|
+
/**
|
|
1570
|
+
* The verifier COMMAND window — the graded command alone, and not the work
|
|
1571
|
+
* that prepared it. On a SHARED-mode trial the preparation that runs first
|
|
1572
|
+
* is reported beside this pair as `shared_verify_setup`. Read this pair
|
|
1573
|
+
* against `verifier_timeout_sec` and nothing else.
|
|
1574
|
+
*/
|
|
1575
|
+
verifier: TimingInfo | null;
|
|
1576
|
+
/**
|
|
1577
|
+
* How long the trial sat claimable before a worker began it. It ends at the
|
|
1578
|
+
* run's beginning, which is APPROXIMATELY — not exactly — where
|
|
1579
|
+
* `environment_setup` starts, so never treat the two pairs as adjacent. The
|
|
1580
|
+
* open bound is when the row became claimable, which for a retried trial is
|
|
1581
|
+
* its backoff deadline rather than its creation: this never bills the
|
|
1582
|
+
* attempt that failed before it.
|
|
1583
|
+
*/
|
|
1584
|
+
queue_wait: TimingInfo | null;
|
|
1585
|
+
/**
|
|
1586
|
+
* The harness bundle resolve, NESTED inside `environment_setup`. A miss that
|
|
1587
|
+
* actually BUILT also carries the publish upload back to the shared bundle
|
|
1588
|
+
* store, which the building caller waits out inside this window — a trial
|
|
1589
|
+
* that joined someone else's build, or hydrated the bytes from that store,
|
|
1590
|
+
* pays neither. Read with `harness_bundle_cache_hit`.
|
|
1591
|
+
*/
|
|
1592
|
+
harness_bundle: TimingInfo | null;
|
|
1593
|
+
/**
|
|
1594
|
+
* The provider image/snapshot/template ensure, also NESTED inside
|
|
1595
|
+
* `environment_setup` and excluding the boot that follows it. Near-zero on
|
|
1596
|
+
* modal BY DESIGN — modal pre-builds nothing, so the real pull-and-cache
|
|
1597
|
+
* happens provider-side when the box is created — so never compare it across
|
|
1598
|
+
* providers raw.
|
|
1599
|
+
*/
|
|
1600
|
+
image_prepare: TimingInfo | null;
|
|
1601
|
+
/**
|
|
1602
|
+
* What a SHARED-mode verify did BEFORE its command — the judge key mint, the
|
|
1603
|
+
* rewardkit bundle resolve and upload, the test-file uploads, the env write.
|
|
1604
|
+
* It ends exactly where `verifier` begins, and the two never overlap.
|
|
1605
|
+
* Null on separate-mode trials, on multi-step trials (which verify per step
|
|
1606
|
+
* and report no trial-level verifier window), and on anything that settled
|
|
1607
|
+
* before the pair was recorded — never a zero-length pair standing in for
|
|
1608
|
+
* "did not happen".
|
|
1609
|
+
*/
|
|
1610
|
+
shared_verify_setup: TimingInfo | null;
|
|
1611
|
+
/**
|
|
1612
|
+
* True when the bundle resolve served bytes already on the worker. False
|
|
1613
|
+
* covers every path that had to produce them — a shared-store hydrate, a
|
|
1614
|
+
* builder run, or joining another trial's in-flight build. Null is
|
|
1615
|
+
* unrecorded (a trial older than these timers, or one whose resolve failed),
|
|
1616
|
+
* never "miss".
|
|
1617
|
+
*/
|
|
1618
|
+
harness_bundle_cache_hit: boolean | null;
|
|
1619
|
+
/**
|
|
1620
|
+
* Per-step results for a multi-step task, in execution order. Null on every
|
|
1621
|
+
* single-step trial — "this trial has no steps", never "it ran zero of
|
|
1622
|
+
* them". A trial that stopped early carries only the steps that ran.
|
|
1623
|
+
*/
|
|
1624
|
+
step_results: StepResult[] | null;
|
|
1625
|
+
/** Which lane `agent_result.cost_usd` came from — see SpendSource; only "measured" is final. */
|
|
1626
|
+
spend_source: SpendSource | null;
|
|
1627
|
+
/**
|
|
1628
|
+
* Which lane `judge_result.cost_usd` is in — the same three-lane vocabulary
|
|
1629
|
+
* as `spend_source`, same rules. Null exactly when `judge_result` is null:
|
|
1630
|
+
* no judge ever ran.
|
|
1631
|
+
*/
|
|
1632
|
+
judge_spend_source?: SpendSource | null;
|
|
1633
|
+
/**
|
|
1634
|
+
* A mid-run LOWER BOUND on spend, never the trial's cost. Only ever climbs
|
|
1635
|
+
* while the trial runs, and is CLEARED when the trial settles — on a
|
|
1636
|
+
* terminal trial read `agent_result.cost_usd` and `spend_source`; those are
|
|
1637
|
+
* the settled truth. Null is "no reading yet", never $0.
|
|
1638
|
+
*/
|
|
1639
|
+
live_spent_usd: number | null;
|
|
1640
|
+
/** When that reading was taken — show its age, never the figure alone. */
|
|
1641
|
+
live_spend_at: string | null;
|
|
1642
|
+
/**
|
|
1643
|
+
* THE ONE-HOME USAGE READING: spend so far plus the token breakdown from
|
|
1644
|
+
* the same gateway records, `provisional` saying whether the numbers can
|
|
1645
|
+
* still grow — present and ticking while the trial runs, settled once the
|
|
1646
|
+
* lane is `measured`. The overlapping fields (`agent_result` tokens,
|
|
1647
|
+
* `live_spent_usd`, `spend_source`) remain for their existing readers; this
|
|
1648
|
+
* is where a client reads the whole answer at once, with the same keys the
|
|
1649
|
+
* managed-agents session surfaces serve. Null = the meter never answered,
|
|
1650
|
+
* never zero. Absent on servers predating the field.
|
|
1651
|
+
*/
|
|
1652
|
+
usage?: UsageReading | null;
|
|
1653
|
+
/**
|
|
1654
|
+
* The cap THIS trial's gateway key carried — history, which can differ from
|
|
1655
|
+
* the job's current cap for rows settled before a change.
|
|
1656
|
+
*/
|
|
1657
|
+
max_trial_spend_usd: number | null;
|
|
1658
|
+
sandbox_provider: EvalSandboxProvider | null;
|
|
1659
|
+
/**
|
|
1660
|
+
* Present when this trial's task declared GPUs the job's stamped provider
|
|
1661
|
+
* could not allocate, so the trial ran on modal instead: `from` is the
|
|
1662
|
+
* job's request, `to` where the boxes actually ran, `reason` the refusing
|
|
1663
|
+
* provider's own sentence. Null on every other trial; absent on servers
|
|
1664
|
+
* predating the field.
|
|
1665
|
+
*/
|
|
1666
|
+
sandbox_provider_degrade?: {
|
|
1667
|
+
from: EvalSandboxProvider;
|
|
1668
|
+
to: EvalSandboxProvider;
|
|
1669
|
+
reason: string;
|
|
1670
|
+
} | null;
|
|
1671
|
+
/**
|
|
1672
|
+
* GPU compute ESTIMATE — measured sandbox lifetime x the platform's
|
|
1673
|
+
* versioned, source-dated rate card (see TrialGpuCost). Present on settled
|
|
1674
|
+
* GPU trials only; null on every other trial, and never merged into
|
|
1675
|
+
* `agent_result.cost_usd`. Absent on servers predating the field.
|
|
1676
|
+
*/
|
|
1677
|
+
gpu_cost?: TrialGpuCost | null;
|
|
1678
|
+
/** Provider id of the box the agent executed in; null when none booted. */
|
|
1679
|
+
sandbox_id: string | null;
|
|
1680
|
+
/** The separate verifier box; null in shared mode or when never reached. */
|
|
1681
|
+
verifier_sandbox_id: string | null;
|
|
1682
|
+
verifier_environment_mode: VerifierEnvironmentMode | null;
|
|
1683
|
+
/**
|
|
1684
|
+
* Which step a RUNNING trial is in, so a polling caller can tell a slow
|
|
1685
|
+
* build from a slow agent. Null when the trial is not mid-phase.
|
|
1686
|
+
*/
|
|
1687
|
+
attempt_phase: AttemptPhase | null;
|
|
1688
|
+
/**
|
|
1689
|
+
* Automatic retries this trial consumed (0 = never retried). The trial
|
|
1690
|
+
* body always shows the LATEST attempt; `retries` holds the lineage.
|
|
1691
|
+
*/
|
|
1692
|
+
n_retries: number;
|
|
1693
|
+
/**
|
|
1694
|
+
* Attempt lineage: earlier attempts whose failure was retried away, oldest
|
|
1695
|
+
* first. The final attempt's outcome is the trial body itself and is not
|
|
1696
|
+
* repeated here; a never-retried trial answers [].
|
|
1697
|
+
*/
|
|
1698
|
+
retries: TrialRetry[];
|
|
1699
|
+
/** Reference to the agent session/trace, when recorded. */
|
|
1700
|
+
session_ref: string | null;
|
|
1701
|
+
/**
|
|
1702
|
+
* Provenance of an UPLOADED trial (its job's `upload` is non-null): the
|
|
1703
|
+
* identity and reported figures the archive's own record files carried.
|
|
1704
|
+
* Null for every trial this platform executed. An uploaded trial keeps
|
|
1705
|
+
* its execution facts (`sandbox_provider`, `agent_result`, `usage`,
|
|
1706
|
+
* `spend_source`) null forever — this platform's meter never saw the run;
|
|
1707
|
+
* the archive's own figures live under `upload.reported_agent_result` and
|
|
1708
|
+
* nowhere else.
|
|
1709
|
+
*/
|
|
1710
|
+
upload: TrialUploadProvenance | null;
|
|
1711
|
+
started_at: string | null;
|
|
1712
|
+
finished_at: string | null;
|
|
1713
|
+
}
|
|
1714
|
+
/**
|
|
1715
|
+
* What the uploaded archive said about THIS trial: its own ids, the full
|
|
1716
|
+
* task name verbatim, and the uploader's own usage figures. REPORTED means
|
|
1717
|
+
* exactly that — `reported_agent_result` is the archive's claim, served for
|
|
1718
|
+
* the reader; it never populates the platform-metered fields
|
|
1719
|
+
* (`agent_result`, `usage`, `spend_source`), which stay null because this
|
|
1720
|
+
* platform's meter never saw the run.
|
|
1721
|
+
*/
|
|
1722
|
+
interface TrialUploadProvenance {
|
|
1723
|
+
/** The archive trial result.json's own `id`; null when it stated none. */
|
|
1724
|
+
original_trial_id: string | null;
|
|
1725
|
+
/** The archive's own `trial_name` (the trial directory). */
|
|
1726
|
+
original_trial_name: string;
|
|
1727
|
+
/**
|
|
1728
|
+
* The archive's task name VERBATIM — possibly registry-qualified
|
|
1729
|
+
* (`org/name`); the trial's `task_name` serves the parsed leaf.
|
|
1730
|
+
*/
|
|
1731
|
+
original_task_name: string;
|
|
1732
|
+
/**
|
|
1733
|
+
* The uploaded `agent_result`'s own token and cost figures, or null when
|
|
1734
|
+
* the archive carried none. Uploader-reported, never platform-measured.
|
|
1735
|
+
*/
|
|
1736
|
+
reported_agent_result: {
|
|
1737
|
+
n_input_tokens: number | null;
|
|
1738
|
+
n_cache_tokens: number | null;
|
|
1739
|
+
n_output_tokens: number | null;
|
|
1740
|
+
cost_usd: number | null;
|
|
1741
|
+
} | null;
|
|
1742
|
+
/**
|
|
1743
|
+
* How this trial linked to a stored task, or why not. Null only on trials
|
|
1744
|
+
* ingested before the link law existed (not backfillable, never guessed).
|
|
1745
|
+
*/
|
|
1746
|
+
link: TrialTaskLink | null;
|
|
1747
|
+
}
|
|
1748
|
+
/**
|
|
1749
|
+
* One retired attempt of a trial — the terminal facts preserved when the
|
|
1750
|
+
* auto-retry scheduler put the trial back on the queue. Its `cost_usd` is
|
|
1751
|
+
* REAL money the job total includes; the trial's own `agent_result.cost_usd`
|
|
1752
|
+
* carries only the final attempt's.
|
|
1753
|
+
*/
|
|
1754
|
+
interface TrialRetry {
|
|
1755
|
+
/** 1-based dispatch number within this trial. */
|
|
1756
|
+
attempt_number: number;
|
|
1757
|
+
exception_info: ExceptionInfo;
|
|
1758
|
+
/** What THIS attempt spent; null when nothing was recorded. */
|
|
1759
|
+
cost_usd: number | null;
|
|
1760
|
+
/** When the attempt was claimed. */
|
|
1761
|
+
started_at: string | null;
|
|
1762
|
+
/** When its failure settled (and the retry was scheduled). */
|
|
1763
|
+
settled_at: string | null;
|
|
1764
|
+
}
|
|
1765
|
+
/** Per-id outcome of POST /api/trials/stop; every requested id appears in exactly one list. */
|
|
1766
|
+
interface StopResponse {
|
|
1767
|
+
/** Trials killed and settled by this request, with their settled rows. */
|
|
1768
|
+
stopped: Trial[];
|
|
1769
|
+
/**
|
|
1770
|
+
* Trace analyses killed and settled by this request, with their settled
|
|
1771
|
+
* rows (`failed`, failure phase `stopped`). Always served; a separate list
|
|
1772
|
+
* because `stopped` is an array of Trial.
|
|
1773
|
+
*/
|
|
1774
|
+
stopped_analyses: TrialAnalysis[];
|
|
1775
|
+
/** Ids that were already terminal; untouched. */
|
|
1776
|
+
already_terminal: string[];
|
|
1777
|
+
/** Ids that do not exist or are not the caller's. */
|
|
1778
|
+
not_found: string[];
|
|
1779
|
+
}
|
|
1780
|
+
/**
|
|
1781
|
+
* One parsed trace event of a trial's transcript. `seq` orders the stream and
|
|
1782
|
+
* is the paging cursor. `data` is the harness-native payload, deliberately
|
|
1783
|
+
* open — with ONE typed member: a `usage` event whose `data.update.source`
|
|
1784
|
+
* is `"gateway"` is the platform's gateway meter speaking (GatewayUsageEvent,
|
|
1785
|
+
* `gatewayUsageOf` reads it), and it is the ONLY usage line that carries
|
|
1786
|
+
* tokens and money a client may show. A harness's own `usage` line (no
|
|
1787
|
+
* `source`) stays in the stream as the raw record and is never rendered as
|
|
1788
|
+
* tokens or cost. Once a trial is terminal its gateway lines follow the last
|
|
1789
|
+
* harness row with `seq` at or past GATEWAY_TRACE_SEQ_BASE.
|
|
1790
|
+
*/
|
|
1791
|
+
interface TraceEvent {
|
|
1792
|
+
/** Monotonic sequence number (the resume position). */
|
|
1793
|
+
seq: number;
|
|
1794
|
+
type: string;
|
|
1795
|
+
data: Record<string, unknown>;
|
|
1796
|
+
}
|
|
1797
|
+
/**
|
|
1798
|
+
* THE GATEWAY METER'S PER-CALL LINE (spec GatewayUsageEvent): one model call
|
|
1799
|
+
* as the LiteLLM gateway priced and recorded it. `promptTokens` INCLUDES the
|
|
1800
|
+
* cached and cache-written shares (the same law as UsageReading.input_tokens);
|
|
1801
|
+
* `cachedTokens` is the cached share; the cache-write and reasoning counts
|
|
1802
|
+
* ride `extra` under the gateway's own names. `costUsd` is the gateway's
|
|
1803
|
+
* `response_cost` — no client prices anything.
|
|
1804
|
+
*/
|
|
1805
|
+
interface GatewayUsage {
|
|
1806
|
+
sessionUpdate: "usage";
|
|
1807
|
+
scope: "call";
|
|
1808
|
+
source: "gateway";
|
|
1809
|
+
/** LiteLLM's own id for the call. */
|
|
1810
|
+
callId: string;
|
|
1811
|
+
/** The gateway's status word for the call (`success` / `failure`). */
|
|
1812
|
+
status: string | null;
|
|
1813
|
+
startedAt: string | null;
|
|
1814
|
+
endedAt: string | null;
|
|
1815
|
+
/** When the platform stored the call. */
|
|
1816
|
+
receivedAt: string;
|
|
1817
|
+
usage: {
|
|
1818
|
+
promptTokens: number;
|
|
1819
|
+
completionTokens: number;
|
|
1820
|
+
cachedTokens: number;
|
|
1821
|
+
costUsd: number;
|
|
1822
|
+
extra: {
|
|
1823
|
+
cache_write_tokens: number;
|
|
1824
|
+
reasoning_tokens?: number;
|
|
1825
|
+
};
|
|
1826
|
+
};
|
|
1827
|
+
}
|
|
1828
|
+
/** The `data` of a gateway usage TraceEvent: the call's start instant and model beside the update. */
|
|
1829
|
+
interface GatewayUsageEvent {
|
|
1830
|
+
timestamp: string | null;
|
|
1831
|
+
model: string | null;
|
|
1832
|
+
update: GatewayUsage;
|
|
1833
|
+
}
|
|
1834
|
+
/**
|
|
1835
|
+
* The seq band a terminal trial's gateway lines ride on the trace-parsed
|
|
1836
|
+
* stream — a MIRROR of the server's constant (swarm_dashboard
|
|
1837
|
+
* lib/gateway-calls.ts GATEWAY_TRACE_SEQ_BASE); the transcript readers below
|
|
1838
|
+
* synthesize the same band for an analysis's or a task check's calls.
|
|
1839
|
+
*/
|
|
1840
|
+
declare const GATEWAY_TRACE_SEQ_BASE = 1000000000;
|
|
1841
|
+
/**
|
|
1842
|
+
* The gateway meter's usage on a trace event, or null: null for a harness's
|
|
1843
|
+
* own usage line (no `source`) and for every other event. THE ONE test a
|
|
1844
|
+
* renderer applies before it shows tokens or money from a trace.
|
|
1845
|
+
*/
|
|
1846
|
+
declare function gatewayUsageOf(event: Pick<TraceEvent, "data">): GatewayUsage | null;
|
|
1847
|
+
/**
|
|
1848
|
+
* One page of a trial's trace — trials().trace().
|
|
1849
|
+
*
|
|
1850
|
+
* Same envelope as every other collection, and nextCursor means the same
|
|
1851
|
+
* thing: pass it back as { cursor } for the next page, and NULL MEANS CAUGHT
|
|
1852
|
+
* UP. To resume a poll later, keep the last event's `seq` and pass it as
|
|
1853
|
+
* { cursor } — the trace's cursor IS its position in the seq timeline.
|
|
1854
|
+
*/
|
|
1855
|
+
type TraceEventPage = Page<TraceEvent>;
|
|
1856
|
+
/** How many of the trials behind an aggregate were SCORED (means cover SCORED only). */
|
|
1857
|
+
interface CompareCoverage {
|
|
1858
|
+
scored: number;
|
|
1859
|
+
total: number;
|
|
1860
|
+
}
|
|
1861
|
+
/**
|
|
1862
|
+
* One (task, job) cell. `status` is a TrialStatus when every trial in the cell
|
|
1863
|
+
* shares it, "MIXED" when they differ, "MISSING" when the job has no trials
|
|
1864
|
+
* for the task.
|
|
1865
|
+
*/
|
|
1866
|
+
interface CompareCell {
|
|
1867
|
+
job_id: string;
|
|
1868
|
+
status: TrialStatus | "MIXED" | "MISSING";
|
|
1869
|
+
/** Mean reward over the cell's SCORED trials; null when none. Zero is a reward. */
|
|
1870
|
+
mean_reward: number | null;
|
|
1871
|
+
coverage: CompareCoverage;
|
|
1872
|
+
}
|
|
1873
|
+
/** One matrix row of jobs().compare(): a task across the compared jobs */
|
|
1874
|
+
interface CompareTaskRow {
|
|
1875
|
+
task_name: string;
|
|
1876
|
+
/** True when the jobs' cells differ in status or reward for this task */
|
|
1877
|
+
disagreement: boolean;
|
|
1878
|
+
/** Cells in the caller's job-id order */
|
|
1879
|
+
cells: CompareCell[];
|
|
1880
|
+
}
|
|
1881
|
+
/** Per-job aggregate of jobs().compare() */
|
|
1882
|
+
interface CompareJobAggregate {
|
|
1883
|
+
id: string;
|
|
1884
|
+
datasets: DatasetRef[];
|
|
1885
|
+
status: JobStatus;
|
|
1886
|
+
/** Mean reward over SCORED trials only; null when none. Zero is a reward. */
|
|
1887
|
+
mean_reward: number | null;
|
|
1888
|
+
coverage: CompareCoverage;
|
|
1889
|
+
cost_usd: number;
|
|
1890
|
+
agents: AgentArm[];
|
|
1891
|
+
started_at: string;
|
|
1892
|
+
}
|
|
1893
|
+
/**
|
|
1894
|
+
* Result of jobs().compare([ids]): per-job aggregates plus a per-task matrix
|
|
1895
|
+
* (disagreement rows first). `taskMatrix` is a frozen camelCase wire key.
|
|
1896
|
+
*/
|
|
1897
|
+
interface CompareResponse {
|
|
1898
|
+
/** Aggregates in the caller's id order */
|
|
1899
|
+
jobs: CompareJobAggregate[];
|
|
1900
|
+
taskMatrix: CompareTaskRow[];
|
|
1901
|
+
}
|
|
1902
|
+
/** Fields every job event carries, whatever its type. */
|
|
1903
|
+
interface JobEventBase {
|
|
1904
|
+
/** Monotonic sequence number (SSE id; the Last-Event-ID resume position) */
|
|
1905
|
+
seq: number;
|
|
1906
|
+
}
|
|
1907
|
+
/** The job's resolved creation inputs, echoed so a watcher that joined late knows what it is watching. */
|
|
1908
|
+
interface JobCreatedData {
|
|
1909
|
+
datasets: DatasetRef[];
|
|
1910
|
+
task_count: number;
|
|
1911
|
+
agents: AgentArm[];
|
|
1912
|
+
n_attempts: number;
|
|
1913
|
+
n_concurrent_trials: number;
|
|
1914
|
+
max_trial_spend_usd: number;
|
|
1915
|
+
sandbox_provider: EvalSandboxProvider;
|
|
1916
|
+
trial_count: number;
|
|
1917
|
+
/** The resolved auto-retry policy the job runs under. */
|
|
1918
|
+
retry: RetryConfig;
|
|
1919
|
+
/**
|
|
1920
|
+
* The resolved timeout multipliers (the same five flat fields the job body
|
|
1921
|
+
* echoes); events older than the feature replay as every phase at 1.0.
|
|
1922
|
+
*/
|
|
1923
|
+
timeout_multiplier: number;
|
|
1924
|
+
agent_timeout_multiplier: number | null;
|
|
1925
|
+
verifier_timeout_multiplier: number | null;
|
|
1926
|
+
agent_setup_timeout_multiplier: number | null;
|
|
1927
|
+
environment_build_timeout_multiplier: number | null;
|
|
1928
|
+
}
|
|
1929
|
+
interface JobCancellingData {
|
|
1930
|
+
job_id: string;
|
|
1931
|
+
/** Queued trials cancelled outright by the request */
|
|
1932
|
+
cancelled_trials: number;
|
|
1933
|
+
/** Trials still in flight, winding down before the job settles */
|
|
1934
|
+
active_trials: number;
|
|
1935
|
+
}
|
|
1936
|
+
interface JobCancelledData {
|
|
1937
|
+
job_id: string;
|
|
1938
|
+
/** Total queued trials cancelled across the request and the settle */
|
|
1939
|
+
cancelled_trials: number;
|
|
1940
|
+
}
|
|
1941
|
+
interface TrialRunningData {
|
|
1942
|
+
trial_id: string;
|
|
1943
|
+
task_name: string;
|
|
1944
|
+
}
|
|
1945
|
+
interface TrialScoringData {
|
|
1946
|
+
trial_id: string;
|
|
1947
|
+
/** Bytes of agent stdout retained for the failure detail */
|
|
1948
|
+
captured_bytes?: number;
|
|
1949
|
+
}
|
|
1950
|
+
/**
|
|
1951
|
+
* A mid-run spend sample landed on a still-live trial. Emitted only when the
|
|
1952
|
+
* reading actually updated a RUNNING/SCORING row, so a poll that raced the
|
|
1953
|
+
* settle never fires one. The token sums come from the same ledger aggregation
|
|
1954
|
+
* that produced the money figure, present only when the sample carried them —
|
|
1955
|
+
* an older event replays without them.
|
|
1956
|
+
*/
|
|
1957
|
+
interface TrialSpendData {
|
|
1958
|
+
trial_id: string;
|
|
1959
|
+
task_name: string;
|
|
1960
|
+
/** The same lagging lower bound as Trial.live_spent_usd — not the trial's cost */
|
|
1961
|
+
live_spent_usd: number;
|
|
1962
|
+
/** Input tokens so far; includes cache tokens */
|
|
1963
|
+
n_input_tokens?: number;
|
|
1964
|
+
/** Cached input tokens so far (a subset of n_input_tokens) */
|
|
1965
|
+
n_cache_tokens?: number;
|
|
1966
|
+
/** Output tokens so far */
|
|
1967
|
+
n_output_tokens?: number;
|
|
1968
|
+
}
|
|
1969
|
+
/**
|
|
1970
|
+
* A trial reached a terminal status. `reward` is present only on the scored
|
|
1971
|
+
* path; `exception_type` and `exception_message` only on a failure (a cancel
|
|
1972
|
+
* carries `exception_type` alone — see `exception_message`); `attempt_phase`
|
|
1973
|
+
* appears when the settle happened mid-phase (worker death), which is exactly
|
|
1974
|
+
* when knowing the phase is worth having.
|
|
1975
|
+
*/
|
|
1976
|
+
interface TrialSettledData {
|
|
1977
|
+
trial_id: string;
|
|
1978
|
+
task_name: string;
|
|
1979
|
+
status: TrialStatus;
|
|
1980
|
+
/** Zero is a reward; absent means the trial did not score. */
|
|
1981
|
+
reward?: number | null;
|
|
1982
|
+
exception_type?: string;
|
|
1983
|
+
/**
|
|
1984
|
+
* The failure in its own words — the same text the trial's
|
|
1985
|
+
* `exception_info.exception_message` holds at settle time (a later
|
|
1986
|
+
* auto-retry verdict is appended to the trial's copy, never to this
|
|
1987
|
+
* frame's). Present only on a failure, beside `exception_type`. Two settled
|
|
1988
|
+
* frames carry `exception_type` alone: a cancel (`CancelledError` — stopped,
|
|
1989
|
+
* not failed; no words are stored) and an event recorded before this field
|
|
1990
|
+
* existed.
|
|
1991
|
+
*/
|
|
1992
|
+
exception_message?: string;
|
|
1993
|
+
attempt_phase?: AttemptPhase | null;
|
|
1994
|
+
}
|
|
1995
|
+
/**
|
|
1996
|
+
* The auto-retry policy put a failed trial back on the queue. Follows the
|
|
1997
|
+
* `trial.settled` of the failure it retries — a watcher that treats
|
|
1998
|
+
* `trial.settled` as final must check for a following `trial.retrying` on
|
|
1999
|
+
* the same trial. The trial runs again no earlier than `delay_sec` from
|
|
2000
|
+
* this event.
|
|
2001
|
+
*/
|
|
2002
|
+
interface TrialRetryingData {
|
|
2003
|
+
trial_id: string;
|
|
2004
|
+
task_name: string;
|
|
2005
|
+
/** Which retry this is (1-based). */
|
|
2006
|
+
retry: number;
|
|
2007
|
+
/** The policy's budget, for "retry 2/3" displays. */
|
|
2008
|
+
max_retries: number;
|
|
2009
|
+
/** The backoff applied before the trial is claimable again. */
|
|
2010
|
+
delay_sec: number;
|
|
2011
|
+
/** The exception name the adjudication ran under. */
|
|
2012
|
+
exception_type: string;
|
|
2013
|
+
}
|
|
2014
|
+
/**
|
|
2015
|
+
* The CLASS of infrastructure fault the auto-retry circuit breaker compares
|
|
2016
|
+
* on — resolved from the trial's typed failure phase, never from message
|
|
2017
|
+
* text (spec `InfraFailureSignature`). `sandbox_death`: the box ceased to
|
|
2018
|
+
* exist while a run still owed it. `provider_create_failure`: the box never
|
|
2019
|
+
* came up. `stream_disconnect`: the run's event stream ended without the
|
|
2020
|
+
* harness ever speaking. `exec_chdir_failure`: the container exec never
|
|
2021
|
+
* started the harness at all (the OCI runtime refused its working directory).
|
|
2022
|
+
*/
|
|
2023
|
+
type InfraFailureSignature = "sandbox_death" | "provider_create_failure" | "stream_disconnect" | "exec_chdir_failure";
|
|
2024
|
+
/**
|
|
2025
|
+
* The auto-retry circuit breaker refused a retry the policy would otherwise
|
|
2026
|
+
* have run: `consecutive` infrastructure failures of the same `signature` in
|
|
2027
|
+
* a row. Follows the `trial.settled` of the failure that tripped it, in the
|
|
2028
|
+
* place a `trial.retrying` would have taken — the trial stays terminal
|
|
2029
|
+
* (INFRASTRUCTURE_ERROR), its own `exception_info.exception_message` gains
|
|
2030
|
+
* the verdict after the failure's words, and `retries_unused` of
|
|
2031
|
+
* `max_retries` are never spent. Every key is present on every frame.
|
|
2032
|
+
*/
|
|
2033
|
+
interface TrialRetryCircuitBrokenData {
|
|
2034
|
+
trial_id: string;
|
|
2035
|
+
task_name: string;
|
|
2036
|
+
signature: InfraFailureSignature;
|
|
2037
|
+
/** Same-signature failures in a row; the breaker trips at two. */
|
|
2038
|
+
consecutive: number;
|
|
2039
|
+
/**
|
|
2040
|
+
* The typed failure phase the signature was resolved from (`sandbox_died`,
|
|
2041
|
+
* `sandbox_boot`, `harness_crash`, …) — the archive's `x_evolve.failurePhase`,
|
|
2042
|
+
* not the `attempt_phase` vocabulary.
|
|
2043
|
+
*/
|
|
2044
|
+
failure_phase: string;
|
|
2045
|
+
/** The policy's budget. */
|
|
2046
|
+
max_retries: number;
|
|
2047
|
+
/** How much of `max_retries` the break left unspent (at least one). */
|
|
2048
|
+
retries_unused: number;
|
|
2049
|
+
/**
|
|
2050
|
+
* The last failure in its own words, as it was settled — the text its
|
|
2051
|
+
* `trial.settled` frame carried, before the trial's own copy gained the
|
|
2052
|
+
* breaker's verdict. `null` only on a frame recorded before the words rode it.
|
|
2053
|
+
*/
|
|
2054
|
+
exception_message: string | null;
|
|
2055
|
+
}
|
|
2056
|
+
/**
|
|
2057
|
+
* One server-sent event from jobs().watch(), as a DISCRIMINATED UNION on
|
|
2058
|
+
* `type` and ONLY on `type`: several event types carry identically shaped
|
|
2059
|
+
* payloads (`job.running` and `job.completed` are both `{job_id}`), so payload
|
|
2060
|
+
* shape can never route a reader — the `type` constant does. Switching on
|
|
2061
|
+
* `type` narrows `data`.
|
|
2062
|
+
*
|
|
2063
|
+
* job.failed is declared terminal by the event stream and by this SDK, but NO
|
|
2064
|
+
* SERVER PATH EMITS IT today. It stays in the union because both consumers
|
|
2065
|
+
* treat it as terminal — the payload is fixed now so a client written today
|
|
2066
|
+
* parses it when it first appears. Treat it as RESERVED rather than expected.
|
|
2067
|
+
*/
|
|
2068
|
+
type JobEvent = (JobEventBase & {
|
|
2069
|
+
type: "job.created";
|
|
2070
|
+
data: JobCreatedData;
|
|
2071
|
+
}) | (JobEventBase & {
|
|
2072
|
+
type: "job.running";
|
|
2073
|
+
data: {
|
|
2074
|
+
job_id: string;
|
|
2075
|
+
};
|
|
2076
|
+
}) | (JobEventBase & {
|
|
2077
|
+
type: "job.cancelling";
|
|
2078
|
+
data: JobCancellingData;
|
|
2079
|
+
}) | (JobEventBase & {
|
|
2080
|
+
type: "job.cancelled";
|
|
2081
|
+
data: JobCancelledData;
|
|
2082
|
+
}) | (JobEventBase & {
|
|
2083
|
+
type: "job.completed";
|
|
2084
|
+
data: {
|
|
2085
|
+
job_id: string;
|
|
2086
|
+
};
|
|
2087
|
+
}) | (JobEventBase & {
|
|
2088
|
+
type: "job.failed";
|
|
2089
|
+
data: {
|
|
2090
|
+
job_id: string;
|
|
2091
|
+
};
|
|
2092
|
+
}) | (JobEventBase & {
|
|
2093
|
+
type: "trial.running";
|
|
2094
|
+
data: TrialRunningData;
|
|
2095
|
+
}) | (JobEventBase & {
|
|
2096
|
+
type: "trial.scoring";
|
|
2097
|
+
data: TrialScoringData;
|
|
2098
|
+
}) | (JobEventBase & {
|
|
2099
|
+
type: "trial.spend";
|
|
2100
|
+
data: TrialSpendData;
|
|
2101
|
+
}) | (JobEventBase & {
|
|
2102
|
+
type: "trial.settled";
|
|
2103
|
+
data: TrialSettledData;
|
|
2104
|
+
}) | (JobEventBase & {
|
|
2105
|
+
type: "trial.retrying";
|
|
2106
|
+
data: TrialRetryingData;
|
|
2107
|
+
}) | (JobEventBase & {
|
|
2108
|
+
type: "trial.retry_circuit_broken";
|
|
2109
|
+
data: TrialRetryCircuitBrokenData;
|
|
2110
|
+
});
|
|
2111
|
+
/**
|
|
2112
|
+
* The handle returned by jobs().watch(). It is both:
|
|
2113
|
+
* - a promise for the final Job — `await client.watch(id)` resolves once
|
|
2114
|
+
* the job reaches a terminal status (the original form); and
|
|
2115
|
+
* - an async iterable of events — `for await (const event of client.watch(id))`
|
|
2116
|
+
* yields each JobEvent and completes on the terminal event.
|
|
2117
|
+
*
|
|
2118
|
+
* Pick one form per call: both drive the same underlying SSE stream, so a
|
|
2119
|
+
* single handle should not be awaited and iterated at once.
|
|
2120
|
+
*/
|
|
2121
|
+
interface JobWatch extends Awaitable<Job>, AsyncIterable<JobEvent> {
|
|
2122
|
+
}
|
|
2123
|
+
/** Cursor page of jobs (newest first) */
|
|
2124
|
+
type JobPage = Page<Job>;
|
|
2125
|
+
/**
|
|
2126
|
+
* The handle returned by jobs().list(). Both:
|
|
2127
|
+
* - a promise for a single JobPage — `await client.list({ limit })`
|
|
2128
|
+
* returns one page (the original form); and
|
|
2129
|
+
* - an async iterable — `for await (const item of client.list())` walks every
|
|
2130
|
+
* job across cursor pages, fetching the next page for you.
|
|
2131
|
+
*/
|
|
2132
|
+
interface JobList extends Awaitable<JobPage>, AsyncIterable<Job> {
|
|
2133
|
+
}
|
|
2134
|
+
/** Cursor page of trace analyses (newest first) */
|
|
2135
|
+
type AnalysisPage = Page<TrialAnalysis>;
|
|
2136
|
+
/**
|
|
2137
|
+
* The handle returned by analyses().list(). Both a promise for one page and
|
|
2138
|
+
* an async iterable across cursor pages, like every other list handle.
|
|
2139
|
+
*/
|
|
2140
|
+
interface AnalysisList extends Awaitable<AnalysisPage>, AsyncIterable<TrialAnalysis> {
|
|
2141
|
+
}
|
|
2142
|
+
/** Cursor page of task quality checks (newest first) */
|
|
2143
|
+
type CheckPage = Page<Check>;
|
|
2144
|
+
/** The handle returned by checks().list() — one page on await, every page on for-await. */
|
|
2145
|
+
interface CheckList extends Awaitable<CheckPage>, AsyncIterable<Check> {
|
|
2146
|
+
}
|
|
2147
|
+
/**
|
|
2148
|
+
* One task's rollup within a job: its trial tally, mean reward over SCORED
|
|
2149
|
+
* trials, and measured cost. Sits between the job body and the trial list so
|
|
2150
|
+
* a caller need not fetch every trial to see which tasks are dragging.
|
|
2151
|
+
*/
|
|
2152
|
+
interface JobTaskRollup {
|
|
2153
|
+
task_name: string;
|
|
2154
|
+
/** The dataset the task came from. */
|
|
2155
|
+
source: string;
|
|
2156
|
+
trials: TrialStatusTally;
|
|
2157
|
+
/** Mean over SCORED trials only; null when none. Zero is a reward. */
|
|
2158
|
+
mean_reward: number | null;
|
|
2159
|
+
/** Measured spend across the task's settled trials. */
|
|
2160
|
+
cost_usd: number | null;
|
|
2161
|
+
/**
|
|
2162
|
+
* The task's LATEST quality check among the checks you may read that ran
|
|
2163
|
+
* on a dataset version this job spans (`checks().create({ source: {
|
|
2164
|
+
* dataset } })`); null = never checked. The job page's CHECK tab reads it.
|
|
2165
|
+
*/
|
|
2166
|
+
check: TaskCheck | null;
|
|
2167
|
+
}
|
|
2168
|
+
/** Cursor page of per-task rollups */
|
|
2169
|
+
type JobTaskRollupPage = Page<JobTaskRollup>;
|
|
2170
|
+
/**
|
|
2171
|
+
* The handle returned by jobs().tasks(). Both a promise for one page and an
|
|
2172
|
+
* async iterable across cursor pages, like every other list handle.
|
|
2173
|
+
*/
|
|
2174
|
+
interface JobTaskRollupList extends Awaitable<JobTaskRollupPage>, AsyncIterable<JobTaskRollup> {
|
|
2175
|
+
}
|
|
2176
|
+
/** Cursor page of trials */
|
|
2177
|
+
type TrialPage = Page<Trial>;
|
|
2178
|
+
/**
|
|
2179
|
+
* The handle returned by jobs().trials(). Both:
|
|
2180
|
+
* - a promise for a single TrialPage — `await client.trials(id, { limit })`
|
|
2181
|
+
* returns one page (the original form); and
|
|
2182
|
+
* - an async iterable — `for await (const trial of client.trials(id))` walks
|
|
2183
|
+
* every trial across cursor pages, fetching the next page for you.
|
|
2184
|
+
*/
|
|
2185
|
+
interface TrialList extends Awaitable<TrialPage>, AsyncIterable<Trial> {
|
|
2186
|
+
}
|
|
2187
|
+
/**
|
|
2188
|
+
* Dataset version lifecycle state (wire values). Terminal: READY, FAILED,
|
|
2189
|
+
* ARCHIVED. RECEIVING sits before the walk (register-first): the corpus is
|
|
2190
|
+
* still uploading through its resumable session — the row exists so the
|
|
2191
|
+
* publish is visible from the first byte, and moves to IMPORTING when the
|
|
2192
|
+
* upload completes and the publish is accepted.
|
|
2193
|
+
*/
|
|
2194
|
+
type DatasetVersionState = "DRAFT" | "RECEIVING" | "IMPORTING" | "BUILDING" | "READY" | "FAILED" | "ARCHIVED";
|
|
2195
|
+
/** One dataset.toml author: a name, and an email when the manifest gives one. */
|
|
2196
|
+
interface DatasetManifestAuthor {
|
|
2197
|
+
name: string;
|
|
2198
|
+
email: string | null;
|
|
2199
|
+
}
|
|
2200
|
+
/**
|
|
2201
|
+
* The metadata half of the Harbor dataset.toml manifest a version imported
|
|
2202
|
+
* under. The full manifest additionally pins per-task/per-file content
|
|
2203
|
+
* digests — verified server-side at import (a mismatch FAILS the import,
|
|
2204
|
+
* `manifest_digest_mismatch`) and readable in the retained package; the wire
|
|
2205
|
+
* carries identity + metadata.
|
|
2206
|
+
*/
|
|
2207
|
+
interface DatasetManifestMetadata {
|
|
2208
|
+
/** The manifest's own dataset name, Harbor `org/name` format. */
|
|
2209
|
+
name: string;
|
|
2210
|
+
/** The manifest's `[dataset].version`; null when it declares none. */
|
|
2211
|
+
version: string | null;
|
|
2212
|
+
description: string;
|
|
2213
|
+
authors: DatasetManifestAuthor[];
|
|
2214
|
+
keywords: string[];
|
|
2215
|
+
/**
|
|
2216
|
+
* How many `[[tasks]]` refs the manifest listed (duplicates included,
|
|
2217
|
+
* Harbor's task_count). Null on rows stored before the count was recorded.
|
|
2218
|
+
*/
|
|
2219
|
+
task_count: number | null;
|
|
2220
|
+
}
|
|
2221
|
+
/**
|
|
2222
|
+
* A version published from a git repository (`git_url` + `git_ref`): the
|
|
2223
|
+
* repository, the ref exactly as requested, the RESOLVED commit the clone
|
|
2224
|
+
* landed on (for an annotated tag, the peeled commit — never the tag
|
|
2225
|
+
* object), and the repository subfolder the corpus was read from.
|
|
2226
|
+
*/
|
|
2227
|
+
interface DatasetVersionGitSource {
|
|
2228
|
+
kind: "git";
|
|
2229
|
+
/**
|
|
2230
|
+
* The repository this version was imported from, userinfo (an embedded
|
|
2231
|
+
* token) stripped; null only when the stored url cannot be parsed.
|
|
2232
|
+
*/
|
|
2233
|
+
git_url: string | null;
|
|
2234
|
+
/** The ref the import was asked for, exactly as requested: a sha, a tag, or (legacy rows) a branch. */
|
|
2235
|
+
ref: string;
|
|
2236
|
+
/** The commit the corpus was actually read from — the resolved sha, peeled for an annotated tag. */
|
|
2237
|
+
commit: string;
|
|
2238
|
+
/** The repository subfolder the corpus was read from; null = repository root. */
|
|
2239
|
+
path: string | null;
|
|
2240
|
+
}
|
|
2241
|
+
/**
|
|
2242
|
+
* A version published from an UPLOADED archive — the multipart `archive`
|
|
2243
|
+
* part, which is what `publish({ source: { directory } })` and the CLI's
|
|
2244
|
+
* `--dir` send. No locator: the bytes came from the client.
|
|
2245
|
+
*/
|
|
2246
|
+
interface DatasetVersionArchiveSource {
|
|
2247
|
+
kind: "archive";
|
|
2248
|
+
/** `sha256:<hex>` over the uploaded tarball's bytes. */
|
|
2249
|
+
digest: string;
|
|
2250
|
+
}
|
|
2251
|
+
/** A version published from a fetched tarball (`archive_url`). */
|
|
2252
|
+
interface DatasetVersionArchiveUrlSource {
|
|
2253
|
+
kind: "archive_url";
|
|
2254
|
+
/** The public https url the corpus tarball was fetched from, as given. */
|
|
2255
|
+
archive_url: string;
|
|
2256
|
+
/** `sha256:<hex>` over the fetched tarball's bytes. */
|
|
2257
|
+
digest: string;
|
|
2258
|
+
}
|
|
2259
|
+
/** A version published from a Harbor hub package (`hub_package`). */
|
|
2260
|
+
interface DatasetVersionHubSource {
|
|
2261
|
+
kind: "hub_package";
|
|
2262
|
+
/** The package reference as given — `org/name` or `org/name@ref` (Harbor's reference grammar). */
|
|
2263
|
+
hub_package: string;
|
|
2264
|
+
/**
|
|
2265
|
+
* `sha256:<hex>` — the hub's content hash over the task file set (Harbor's
|
|
2266
|
+
* recipe), the immutable hub version the import was pinned to, whatever
|
|
2267
|
+
* the reference's tag points at today.
|
|
2268
|
+
*/
|
|
2269
|
+
digest: string;
|
|
2270
|
+
}
|
|
2271
|
+
/**
|
|
2272
|
+
* What one version was imported from — the LOCATOR the publish named (what
|
|
2273
|
+
* you would pass to publish the same source again) and the IDENTITY the
|
|
2274
|
+
* import resolved it to — one shape per publish kind, discriminated on
|
|
2275
|
+
* `kind` in the publish request's own vocabulary. Served on EVERY version
|
|
2276
|
+
* whatever its state: a version whose build FAILED can never become the
|
|
2277
|
+
* active version, so this is where its imported bytes stay observable.
|
|
2278
|
+
* Every `digest` is spelled `sha256:<hex>`; a git `commit` is a bare sha.
|
|
2279
|
+
*/
|
|
2280
|
+
type DatasetVersionSource = DatasetVersionGitSource | DatasetVersionArchiveSource | DatasetVersionArchiveUrlSource | DatasetVersionHubSource;
|
|
2281
|
+
/**
|
|
2282
|
+
* One task's own terminal build state inside a published version — the
|
|
2283
|
+
* per-task member of the DatasetVersionState family (the partial-publish
|
|
2284
|
+
* model). Every task of a settled build is exactly one of these; there is no
|
|
2285
|
+
* per-task "building" state on the wire, because outcomes are recorded only
|
|
2286
|
+
* when the version settles.
|
|
2287
|
+
*/
|
|
2288
|
+
type TaskBuildState = "READY" | "FAILED";
|
|
2289
|
+
/**
|
|
2290
|
+
* Why one task FAILED its independent build — the ONE failure grammar for
|
|
2291
|
+
* every step: parse-level refusals (schema/capability) and build-level
|
|
2292
|
+
* failures (image build, mirror, compose resolution, image-config read,
|
|
2293
|
+
* skills verification) speak it identically.
|
|
2294
|
+
*/
|
|
2295
|
+
interface TaskBuildFailure {
|
|
2296
|
+
/**
|
|
2297
|
+
* Typed reason. Parse refusals record `task_parse_failed`; build steps
|
|
2298
|
+
* record the builder's own family (`image_build_failed`,
|
|
2299
|
+
* `builder_unavailable`, `image_push_failed`, ...). Open set — render the
|
|
2300
|
+
* string; branch on `step` for coarse grouping.
|
|
2301
|
+
*/
|
|
2302
|
+
code: string;
|
|
2303
|
+
/**
|
|
2304
|
+
* The build step that failed: `parse`, `image-build`, `image-config`,
|
|
2305
|
+
* `skills-verify`, `compose-resolve`, `image-mirror`, or `store`.
|
|
2306
|
+
*/
|
|
2307
|
+
step: string;
|
|
2308
|
+
/** The failure sentence, naming the task's own defect. */
|
|
2309
|
+
message: string;
|
|
2310
|
+
/**
|
|
2311
|
+
* Bounded tail of the failing step's build log (the failing line and its
|
|
2312
|
+
* neighbourhood). Served by the per-task build route; list surfaces omit
|
|
2313
|
+
* it. Null when the step produced no log to excerpt (a parse refusal's
|
|
2314
|
+
* message IS the whole story).
|
|
2315
|
+
*/
|
|
2316
|
+
excerpt?: string | null;
|
|
2317
|
+
}
|
|
2318
|
+
/**
|
|
2319
|
+
* One task's build outcome inside one published version — the failure-detail
|
|
2320
|
+
* read (datasets().getTaskBuild()). READY tasks answer too (failure and log
|
|
2321
|
+
* pointer null), so a poller needs no negative-space reasoning.
|
|
2322
|
+
*/
|
|
2323
|
+
interface TaskBuild {
|
|
2324
|
+
task_name: string;
|
|
2325
|
+
state: TaskBuildState;
|
|
2326
|
+
/** The typed reason WITH the failing-step excerpt; null on READY. */
|
|
2327
|
+
failure: TaskBuildFailure | null;
|
|
2328
|
+
/**
|
|
2329
|
+
* Pointer to the FULL build log of the failing step
|
|
2330
|
+
* (`cloudwatch://<group>/<stream>` for image builds), for operators and
|
|
2331
|
+
* support tooling. Null when the failing step kept no separate log (parse
|
|
2332
|
+
* refusals), and on READY tasks.
|
|
2333
|
+
*/
|
|
2334
|
+
build_log_ref: string | null;
|
|
2335
|
+
}
|
|
2336
|
+
/**
|
|
2337
|
+
* One failed task on the dataset detail's `failed_tasks` list: the compact
|
|
2338
|
+
* typed reason. The failing-step excerpt and build-log pointer live on the
|
|
2339
|
+
* per-task build route (datasets().getTaskBuild()).
|
|
2340
|
+
*/
|
|
2341
|
+
interface DatasetFailedTask {
|
|
2342
|
+
task_name: string;
|
|
2343
|
+
failure: TaskBuildFailure;
|
|
2344
|
+
}
|
|
2345
|
+
/** One immutable version of a dataset — one shape on every surface */
|
|
2346
|
+
interface DatasetVersion {
|
|
2347
|
+
version: string;
|
|
2348
|
+
state: DatasetVersionState;
|
|
2349
|
+
created_at: string;
|
|
2350
|
+
/**
|
|
2351
|
+
* The READY (runnable) tasks of this version. Under the partial-publish
|
|
2352
|
+
* model this is what a whole-dataset job runs.
|
|
2353
|
+
*/
|
|
2354
|
+
task_count: number;
|
|
2355
|
+
/**
|
|
2356
|
+
* Tasks of the published corpus that FAILED their independent build and
|
|
2357
|
+
* are therefore not runnable in this version (the partial-publish model).
|
|
2358
|
+
* 0 on a fully built version — and on servers that predate the field. The
|
|
2359
|
+
* names and reasons are on the dataset detail's `failed_tasks`; the full
|
|
2360
|
+
* per-task detail (excerpt + build-log pointer) answers at
|
|
2361
|
+
* datasets().getTaskBuild(). Fixing one is a re-publish — versions are
|
|
2362
|
+
* immutable.
|
|
2363
|
+
*/
|
|
2364
|
+
n_failed_tasks: number;
|
|
2365
|
+
/**
|
|
2366
|
+
* The dataset.toml identity/metadata this version imported under. Null when
|
|
2367
|
+
* the corpus carried no manifest, and on servers that predate the field —
|
|
2368
|
+
* absence is "nothing to report", never a crash.
|
|
2369
|
+
*/
|
|
2370
|
+
manifest: DatasetManifestMetadata | null;
|
|
2371
|
+
/**
|
|
2372
|
+
* What THIS version was imported from, per publish kind (`kind`: git /
|
|
2373
|
+
* archive / archive_url / hub_package). Null when the platform recorded
|
|
2374
|
+
* nothing readable (a seeded directory, a pre-provenance row, or a fetched
|
|
2375
|
+
* archive_url / hub_package row whose locator was never stored), and on
|
|
2376
|
+
* servers that predate the field — absence is "nothing to report", never a
|
|
2377
|
+
* fabricated value.
|
|
2378
|
+
*/
|
|
2379
|
+
source: DatasetVersionSource | null;
|
|
2380
|
+
}
|
|
2381
|
+
/**
|
|
2382
|
+
* One provider's verdict for a task: runnable there, refused with the
|
|
2383
|
+
* limitation named (e.g. a multi-container task on a provider that cannot
|
|
2384
|
+
* host its services, or declared resources above the provider's ceiling), or
|
|
2385
|
+
* — GPU tasks only — runnable via a recorded DEGRADE: `ok: true` with
|
|
2386
|
+
* `degrades_to: "modal"` means a job stamped on this provider still runs the
|
|
2387
|
+
* task, on modal, and the trial records the same fact as
|
|
2388
|
+
* `sandbox_provider_degrade`; `reason` then carries this provider's own
|
|
2389
|
+
* sentence for why it could not serve the GPUs itself.
|
|
2390
|
+
*/
|
|
2391
|
+
type TaskProviderVerdict = {
|
|
2392
|
+
ok: true;
|
|
2393
|
+
degrades_to?: "modal";
|
|
2394
|
+
reason?: string;
|
|
2395
|
+
} | {
|
|
2396
|
+
ok: false;
|
|
2397
|
+
reason: string;
|
|
2398
|
+
};
|
|
2399
|
+
/**
|
|
2400
|
+
* A typed, non-fatal fact recorded about an ACCEPTED task — not a refusal
|
|
2401
|
+
* (the task imports and runs) and not a failure (nothing failed): a recorded
|
|
2402
|
+
* degrade the platform states where the publisher reads it.
|
|
2403
|
+
*
|
|
2404
|
+
* `tests_dockerfile_not_built`: the task ships a tests/Dockerfile the verifier
|
|
2405
|
+
* does not build, because upstream never would on its shape — the separate
|
|
2406
|
+
* verifier's effective environment pins a docker_image (Harbor boots it
|
|
2407
|
+
* as-is), or the verifier is shared and runs inside the agent box; a
|
|
2408
|
+
* dependency the recipe would install must already be in the image the
|
|
2409
|
+
* verifier boots. `message` is the platform's own sentence, naming the shape.
|
|
2410
|
+
*/
|
|
2411
|
+
interface TaskNote {
|
|
2412
|
+
code: "tests_dockerfile_not_built";
|
|
2413
|
+
message: string;
|
|
2414
|
+
}
|
|
2415
|
+
/** Public task fields only — instructions, environments, and tests never leave the server */
|
|
2416
|
+
interface Task {
|
|
2417
|
+
task_name: string;
|
|
2418
|
+
agent_timeout_sec: number;
|
|
2419
|
+
verifier_timeout_sec: number;
|
|
2420
|
+
/**
|
|
2421
|
+
* GPUs the task declares (task.toml [environment] gpus — Harbor's field
|
|
2422
|
+
* honored verbatim). 0 = a CPU task. Absent on servers predating the field
|
|
2423
|
+
* — treat as 0.
|
|
2424
|
+
*/
|
|
2425
|
+
gpus?: number;
|
|
2426
|
+
/**
|
|
2427
|
+
* Acceptable GPU types (e.g. ["H100"]), Harbor semantics: null means ANY
|
|
2428
|
+
* type is acceptable. Always null when gpus is 0.
|
|
2429
|
+
*/
|
|
2430
|
+
gpu_types?: string[] | null;
|
|
2431
|
+
/**
|
|
2432
|
+
* Where the task can run, per sandbox provider. Advisory for planning a
|
|
2433
|
+
* job's provider choice — creating a job whose tasks include one refused on
|
|
2434
|
+
* the chosen provider is rejected with the same reason, so nothing is ever
|
|
2435
|
+
* spent on a trial that cannot execute.
|
|
2436
|
+
*/
|
|
2437
|
+
providers: Record<EvalSandboxProvider, TaskProviderVerdict>;
|
|
2438
|
+
/**
|
|
2439
|
+
* Typed, non-fatal facts recorded about the task at import (TaskNote).
|
|
2440
|
+
* [] when there is nothing to say — every task imported before the notes
|
|
2441
|
+
* existed, and every task on a server predating the field.
|
|
2442
|
+
*/
|
|
2443
|
+
notes: TaskNote[];
|
|
2444
|
+
}
|
|
2445
|
+
/**
|
|
2446
|
+
* The active version's git PROVENANCE — git_url + the requested ref + the
|
|
2447
|
+
* resolved commit + the repository subfolder — plus, for a ref that can move,
|
|
2448
|
+
* the WATCH: where the ref points now versus what the version was built from,
|
|
2449
|
+
* the data behind a "new version available" badge. Null when the active
|
|
2450
|
+
* version did not come from a git remote (an uploaded tarball, a seeded
|
|
2451
|
+
* corpus, a pre-provenance row); null is never "up to date". A version pinned
|
|
2452
|
+
* to a commit sha serves the provenance with the watch at rest (latest_commit
|
|
2453
|
+
* / checked_at / error null, moved false) — nothing checks a pin. Nothing
|
|
2454
|
+
* here imports anything — a new version is always a row you create (or
|
|
2455
|
+
* `auto_import` creates).
|
|
2456
|
+
*/
|
|
2457
|
+
interface UpstreamStatus {
|
|
2458
|
+
/**
|
|
2459
|
+
* The repository the active version was imported from, with any userinfo
|
|
2460
|
+
* (an embedded token) stripped; null only when the stored url cannot be
|
|
2461
|
+
* parsed. Absent on a pre-provenance server.
|
|
2462
|
+
*/
|
|
2463
|
+
git_url?: string | null;
|
|
2464
|
+
/** The ref the active version was imported from, exactly as requested. */
|
|
2465
|
+
ref: string;
|
|
2466
|
+
/** The commit the active version was built from (the resolved sha). */
|
|
2467
|
+
current_commit: string;
|
|
2468
|
+
/**
|
|
2469
|
+
* The repository subfolder the corpus was read from; null = repository
|
|
2470
|
+
* root. Absent on a pre-provenance server.
|
|
2471
|
+
*/
|
|
2472
|
+
path?: string | null;
|
|
2473
|
+
/** Where the ref points upstream now; null when the last check failed. */
|
|
2474
|
+
latest_commit: string | null;
|
|
2475
|
+
/**
|
|
2476
|
+
* The newest commit a local version already exists for, whether or not it
|
|
2477
|
+
* is the active one; null before any import recorded one. Absent on an
|
|
2478
|
+
* older server.
|
|
2479
|
+
*/
|
|
2480
|
+
acked_commit?: string | null;
|
|
2481
|
+
/** True when upstream has moved off the built-from commit. Branch on this. */
|
|
2482
|
+
moved: boolean;
|
|
2483
|
+
/** Reserved; always null today. */
|
|
2484
|
+
behind_by: number | null;
|
|
2485
|
+
/** When the cached answer was taken; null before the first check. */
|
|
2486
|
+
checked_at: string | null;
|
|
2487
|
+
/** Why the last check failed. Show "could not check", not "up to date". */
|
|
2488
|
+
error: string | null;
|
|
2489
|
+
/** Whether a moved upstream automatically imports a new version. */
|
|
2490
|
+
auto_import: boolean;
|
|
2491
|
+
}
|
|
2492
|
+
/**
|
|
2493
|
+
* A dataset in the catalog.
|
|
2494
|
+
*
|
|
2495
|
+
* list() returns the summary fields; get() additionally populates versions,
|
|
2496
|
+
* selected_version, tasks, created_at, and updated_at.
|
|
2497
|
+
*/
|
|
2498
|
+
interface Dataset {
|
|
2499
|
+
name: string;
|
|
2500
|
+
title: string | null;
|
|
2501
|
+
description: string | null;
|
|
2502
|
+
/** The active version, or null when none is active (bare-name job refs refuse). */
|
|
2503
|
+
active_version: DatasetVersion | null;
|
|
2504
|
+
/**
|
|
2505
|
+
* The dataset's NEWEST version row (newest created_at first, id as the
|
|
2506
|
+
* tiebreak) — active or not.
|
|
2507
|
+
*
|
|
2508
|
+
* This is what makes a publish observable BEFORE it lands: a first
|
|
2509
|
+
* publish walks IMPORTING -> BUILDING here while `active_version` is
|
|
2510
|
+
* still null — the importer itself flips the finished build to READY and,
|
|
2511
|
+
* on an owner-stamped dataset, promotes it to the active version in the
|
|
2512
|
+
* same transaction. It can also hold a version that never landed (a
|
|
2513
|
+
* FAILED build), so it is NOT a substitute for `active_version`: a
|
|
2514
|
+
* bare-name job ref still resolves the active version and refuses
|
|
2515
|
+
* without one.
|
|
2516
|
+
*
|
|
2517
|
+
* Null when the dataset has no version rows at all — and also null when
|
|
2518
|
+
* talking to a server older than this field.
|
|
2519
|
+
*/
|
|
2520
|
+
latest_version: DatasetVersion | null;
|
|
2521
|
+
/** All versions, newest first (get() only) */
|
|
2522
|
+
versions?: DatasetVersion[];
|
|
2523
|
+
/** The version whose tasks are listed below (get() only) */
|
|
2524
|
+
selected_version?: DatasetVersion | null;
|
|
2525
|
+
/**
|
|
2526
|
+
* One page of the selected version's tasks (get() only). Paged like every
|
|
2527
|
+
* other collection: a SWE-bench-scale dataset has thousands of tasks, so
|
|
2528
|
+
* pass { limit, cursor } to get() and follow nextCursor.
|
|
2529
|
+
*/
|
|
2530
|
+
tasks?: Page<Task>;
|
|
2531
|
+
/**
|
|
2532
|
+
* The selected version's tasks that FAILED their independent build (get()
|
|
2533
|
+
* only; the partial-publish model). Always present on the detail body —
|
|
2534
|
+
* empty on a fully built version. Ordered by task name and capped at the
|
|
2535
|
+
* task page limit; `n_failed_tasks` on the version object is always the
|
|
2536
|
+
* exact count. Each entry carries the typed reason; the failing-step
|
|
2537
|
+
* excerpt and build-log pointer live on datasets().getTaskBuild().
|
|
2538
|
+
*/
|
|
2539
|
+
failed_tasks?: DatasetFailedTask[];
|
|
2540
|
+
upstream: UpstreamStatus | null;
|
|
2541
|
+
/** get() only */
|
|
2542
|
+
created_at?: string;
|
|
2543
|
+
/** get() only */
|
|
2544
|
+
updated_at?: string;
|
|
2545
|
+
}
|
|
2546
|
+
/** Body of datasets().update() — the only settable dataset field. */
|
|
2547
|
+
interface DatasetPatch {
|
|
2548
|
+
/** Automatically import a new version when the upstream git ref moves. */
|
|
2549
|
+
upstream_auto_import: boolean;
|
|
2550
|
+
}
|
|
2551
|
+
/**
|
|
2552
|
+
* A dataset's active version resolved to a runnable shape.
|
|
2553
|
+
*
|
|
2554
|
+
* Unlike Dataset, `version` and `tasks` are non-optional: datasets()
|
|
2555
|
+
* .getActive() throws NoActiveVersionError when there is no active version,
|
|
2556
|
+
* so callers never branch on a missing active version.
|
|
2557
|
+
*/
|
|
2558
|
+
interface ActiveDataset {
|
|
2559
|
+
name: string;
|
|
2560
|
+
title: string | null;
|
|
2561
|
+
description: string | null;
|
|
2562
|
+
/** The active version (always present) */
|
|
2563
|
+
active_version: DatasetVersion;
|
|
2564
|
+
/** The active version string (identical to active_version.version) */
|
|
2565
|
+
version: string;
|
|
2566
|
+
/** One page of the active version's tasks */
|
|
2567
|
+
tasks: Page<Task>;
|
|
2568
|
+
/** All versions, newest first */
|
|
2569
|
+
versions: DatasetVersion[];
|
|
2570
|
+
created_at: string;
|
|
2571
|
+
updated_at: string;
|
|
2572
|
+
}
|
|
2573
|
+
/** Cursor page of datasets */
|
|
2574
|
+
type DatasetPage = Page<Dataset>;
|
|
2575
|
+
/** Dual-use handle from datasets().list(): await one page, or iterate them all */
|
|
2576
|
+
interface DatasetList extends Awaitable<DatasetPage>, AsyncIterable<Dataset> {
|
|
2577
|
+
}
|
|
2578
|
+
/**
|
|
2579
|
+
* Source for datasets().publish(): a git repository pinned to a ref, a local
|
|
2580
|
+
* corpus directory (tarred deterministically on the client and uploaded), a
|
|
2581
|
+
* PUBLIC https tarball URL the server fetches itself, or a PUBLIC Harbor hub
|
|
2582
|
+
* package the server resolves and fetches — the last two move zero client
|
|
2583
|
+
* bytes.
|
|
2584
|
+
*
|
|
2585
|
+
* A UNION, not five optional fields: `{}` and two-branches-at-once are
|
|
2586
|
+
* compile errors rather than a 400 the caller discovers at run time, and
|
|
2587
|
+
* `?: never` on the absent branches' keys is what rejects the excess property
|
|
2588
|
+
* through a variable. `git_ref` is REQUIRED on the git branch — an unpinned
|
|
2589
|
+
* import is not reproducible.
|
|
2590
|
+
*/
|
|
2591
|
+
type DatasetSource = {
|
|
2592
|
+
/**
|
|
2593
|
+
* A git repository URL. https:// only — the import runs on a worker with
|
|
2594
|
+
* no ssh client, so ssh:// and git@ remotes are refused at validation.
|
|
2595
|
+
* For a private repository, put a token in the https url.
|
|
2596
|
+
*/
|
|
2597
|
+
git_url: string;
|
|
2598
|
+
/**
|
|
2599
|
+
* A PINNED ref, required: a full 40-hex commit sha, or a tag (the
|
|
2600
|
+
* server resolves it to its commit at accept time, stores the sha, and
|
|
2601
|
+
* verifies the tag still points there at import). A branch name is
|
|
2602
|
+
* refused with `unpinned_git_ref` — an unpinned import is not
|
|
2603
|
+
* reproducible — and the refusal's details carry the commit the branch
|
|
2604
|
+
* points at right now, the pin to use instead.
|
|
2605
|
+
*/
|
|
2606
|
+
git_ref: string;
|
|
2607
|
+
/**
|
|
2608
|
+
* Optional repository SUBFOLDER holding the corpus (POSIX path relative
|
|
2609
|
+
* to the repository root, e.g. "datasets/my-swe"). The server imports
|
|
2610
|
+
* only that folder, fetched via git sparse checkout. Ambiguity is
|
|
2611
|
+
* refused rather than interpreted: no absolute paths, no "." / ".." or
|
|
2612
|
+
* empty segments, no backslashes, whitespace, pattern characters, or
|
|
2613
|
+
* ".git" segments — and a path that is not a directory at the pinned
|
|
2614
|
+
* ref fails the import loudly instead of landing a 0-task version.
|
|
2615
|
+
*/
|
|
2616
|
+
git_path?: string;
|
|
2617
|
+
directory?: never;
|
|
2618
|
+
archive_url?: never;
|
|
2619
|
+
hub_package?: never;
|
|
2620
|
+
} | {
|
|
2621
|
+
/** A local standard-layout corpus directory — tarred + gzipped and uploaded. */
|
|
2622
|
+
directory: string;
|
|
2623
|
+
git_url?: never;
|
|
2624
|
+
git_ref?: never;
|
|
2625
|
+
git_path?: never;
|
|
2626
|
+
archive_url?: never;
|
|
2627
|
+
hub_package?: never;
|
|
2628
|
+
} | {
|
|
2629
|
+
/**
|
|
2630
|
+
* A PUBLIC https URL to a gzipped corpus tarball — the SERVER fetches
|
|
2631
|
+
* it, so no bytes leave this machine. Public sources only: credentials
|
|
2632
|
+
* in the URL are refused (authenticated sources are a planned
|
|
2633
|
+
* follow-up) and the host must be publicly resolvable. The fetched
|
|
2634
|
+
* bytes pass the same validation and size caps as an uploaded archive.
|
|
2635
|
+
* `name` and `version` are required with this source.
|
|
2636
|
+
*/
|
|
2637
|
+
archive_url: string;
|
|
2638
|
+
git_url?: never;
|
|
2639
|
+
git_ref?: never;
|
|
2640
|
+
git_path?: never;
|
|
2641
|
+
directory?: never;
|
|
2642
|
+
hub_package?: never;
|
|
2643
|
+
} | {
|
|
2644
|
+
/**
|
|
2645
|
+
* A PUBLIC Harbor hub package reference, `org/name[@ref]` in Harbor's
|
|
2646
|
+
* own grammar — no ref (or `latest`) is the latest tag, a number is a
|
|
2647
|
+
* revision, `sha256:<64 hex>` is a digest, anything else is a tag. The
|
|
2648
|
+
* server resolves it when the publish is accepted and the worker
|
|
2649
|
+
* fetches BY the resolved digest, so a tag moved after the 202 can
|
|
2650
|
+
* never deliver different bytes. A task package imports as a one-task
|
|
2651
|
+
* dataset; a dataset package fetches every digest-pinned member; both
|
|
2652
|
+
* are digest-verified against the hub's own pins. `name` defaults to
|
|
2653
|
+
* the package's short name and `version` to its resolved revision. A
|
|
2654
|
+
* missing or private package is refused `hub_package_not_found`; an
|
|
2655
|
+
* unreachable hub is `hub_unreachable` (502) — retry the publish.
|
|
2656
|
+
*/
|
|
2657
|
+
hub_package: string;
|
|
2658
|
+
git_url?: never;
|
|
2659
|
+
git_ref?: never;
|
|
2660
|
+
git_path?: never;
|
|
2661
|
+
directory?: never;
|
|
2662
|
+
archive_url?: never;
|
|
2663
|
+
};
|
|
2664
|
+
/** Input for datasets().preflight() — the dry-run half of publish. */
|
|
2665
|
+
interface PreflightDatasetInput {
|
|
2666
|
+
/**
|
|
2667
|
+
* A local standard-layout corpus directory. Only its METADATA moves: the
|
|
2668
|
+
* client walks the corpus shape (a single task directory, a tasks/ subdir,
|
|
2669
|
+
* or a root of task directories — the import's own reading), collects each
|
|
2670
|
+
* task's task.toml plus the optional dataset.toml manifest, and posts
|
|
2671
|
+
* kilobytes of JSON. The corpus bytes themselves never leave the machine.
|
|
2672
|
+
*/
|
|
2673
|
+
source: {
|
|
2674
|
+
directory: string;
|
|
2675
|
+
};
|
|
2676
|
+
}
|
|
2677
|
+
/**
|
|
2678
|
+
* One task's dry-run verdict. `ok: true` means no task.toml-decidable import
|
|
2679
|
+
* guard would refuse the task — never "this task will import"; the checks a
|
|
2680
|
+
* toml alone cannot decide are named in DatasetPreflight.deferred and the
|
|
2681
|
+
* real publish stays the authority. `ok: false` carries the importer's own
|
|
2682
|
+
* refusal sentence, exactly what a real import of this task would say.
|
|
2683
|
+
*/
|
|
2684
|
+
interface PreflightTaskVerdict {
|
|
2685
|
+
/** The task directory's basename. */
|
|
2686
|
+
name: string;
|
|
2687
|
+
ok: boolean;
|
|
2688
|
+
/** metadata.task_id, or its directory-name fallback. */
|
|
2689
|
+
task_key: string;
|
|
2690
|
+
/** Present with ok true: the recorded Harbor schema_version. */
|
|
2691
|
+
schema_version?: string;
|
|
2692
|
+
/**
|
|
2693
|
+
* Present with ok true: verdict per sandbox provider over the
|
|
2694
|
+
* toml-declared requirements (GPU, sizing, network) — the same
|
|
2695
|
+
* adjudication the published dataset's task stamps use, with the
|
|
2696
|
+
* compose/image-command halves deferred to the import.
|
|
2697
|
+
*/
|
|
2698
|
+
providers?: Record<EvalSandboxProvider, TaskProviderVerdict>;
|
|
2699
|
+
/**
|
|
2700
|
+
* Present with ok true: the typed task notes a task.toml alone decides —
|
|
2701
|
+
* today `tests_dockerfile_not_built` for a separate verifier whose
|
|
2702
|
+
* effective environment pins a docker_image (the image is booted as-is and
|
|
2703
|
+
* tests/ is never built, upstream semantics), worded conditionally because
|
|
2704
|
+
* the door never sees whether the tests tree ships a Dockerfile. [] when
|
|
2705
|
+
* there is nothing to say; absent on servers predating the field.
|
|
2706
|
+
*/
|
|
2707
|
+
notes?: TaskNote[];
|
|
2708
|
+
/** Present with ok false: the importer's refusal sentence. */
|
|
2709
|
+
reason?: string;
|
|
2710
|
+
}
|
|
2711
|
+
/** An import guard the pre-flight cannot run, and what it reads instead. */
|
|
2712
|
+
interface PreflightDeferredCheck {
|
|
2713
|
+
name: string;
|
|
2714
|
+
reads: string;
|
|
2715
|
+
}
|
|
2716
|
+
/** The dataset.toml manifest's dry-run verdict (null when none was sent). */
|
|
2717
|
+
interface PreflightManifestVerdict {
|
|
2718
|
+
ok: boolean;
|
|
2719
|
+
/** Present with ok true: [dataset].name (Harbor org/name). */
|
|
2720
|
+
name?: string;
|
|
2721
|
+
/** Present with ok true: the catalog-facing half of the name. */
|
|
2722
|
+
short_name?: string;
|
|
2723
|
+
/** Present with ok true: [dataset].version, null when it declares none. */
|
|
2724
|
+
version?: string | null;
|
|
2725
|
+
/** Present with ok true: unique (name, digest) task entries. */
|
|
2726
|
+
task_count?: number;
|
|
2727
|
+
/** Present with ok false: the import's manifest refusal sentence. */
|
|
2728
|
+
reason?: string;
|
|
2729
|
+
}
|
|
2730
|
+
/**
|
|
2731
|
+
* The dry-run answer of POST /api/datasets/preflight. Nothing was written to
|
|
2732
|
+
* produce it: `checks` names the guards that ran, `deferred` the import
|
|
2733
|
+
* guards a task.toml alone cannot decide.
|
|
2734
|
+
*/
|
|
2735
|
+
interface DatasetPreflight {
|
|
2736
|
+
/** The pinned parser revision that judged (e.g. "harbor-import/14"). */
|
|
2737
|
+
importer_version: string;
|
|
2738
|
+
checks: string[];
|
|
2739
|
+
deferred: PreflightDeferredCheck[];
|
|
2740
|
+
manifest: PreflightManifestVerdict | null;
|
|
2741
|
+
tasks: PreflightTaskVerdict[];
|
|
2742
|
+
tasks_total: number;
|
|
2743
|
+
tasks_ok: number;
|
|
2744
|
+
tasks_refused: number;
|
|
2745
|
+
}
|
|
2746
|
+
/** Input for datasets().publish() */
|
|
2747
|
+
interface PublishDatasetInput {
|
|
2748
|
+
source: DatasetSource;
|
|
2749
|
+
/**
|
|
2750
|
+
* Catalog dataset name the version lands under (created or extended).
|
|
2751
|
+
* Optional when a `directory` source carries a dataset.toml manifest — the
|
|
2752
|
+
* server derives the name from the manifest (the short segment of its
|
|
2753
|
+
* `org/name`) — and for a `hub_package` source, which defaults to the
|
|
2754
|
+
* package's short name. Always required for a git or `archive_url` source,
|
|
2755
|
+
* whose corpus the server only fetches after the publish is accepted.
|
|
2756
|
+
*/
|
|
2757
|
+
name?: string;
|
|
2758
|
+
/**
|
|
2759
|
+
* Version label for the new immutable version. Optional when a `directory`
|
|
2760
|
+
* source's dataset.toml declares `[dataset].version` and for a
|
|
2761
|
+
* `hub_package` source (defaults to the resolved hub revision number);
|
|
2762
|
+
* required otherwise.
|
|
2763
|
+
*/
|
|
2764
|
+
version?: string;
|
|
2765
|
+
}
|
|
2766
|
+
/** Options for datasets().publish() */
|
|
2767
|
+
interface PublishDatasetOptions {
|
|
2768
|
+
/**
|
|
2769
|
+
* Called as the archive's bytes go onto the wire (a `directory` source
|
|
2770
|
+
* only — a git source uploads nothing). `sentBytes` counts archive bytes
|
|
2771
|
+
* whose write the transport confirmed flushed; `totalBytes` is the
|
|
2772
|
+
* archive's size. Client-side by construction: the stream itself is the
|
|
2773
|
+
* measurement, no server call is made. Fires per flushed chunk — throttle
|
|
2774
|
+
* in the renderer, not here.
|
|
2775
|
+
*/
|
|
2776
|
+
onUploadProgress?: (sentBytes: number, totalBytes: number) => void;
|
|
2777
|
+
/**
|
|
2778
|
+
* Register-first: called ONCE, the moment the resumable door's session
|
|
2779
|
+
* open pre-creates the import (before any corpus byte moves), with the
|
|
2780
|
+
* import id the eventual 202 will carry. A watcher may attach to it right
|
|
2781
|
+
* away — `datasets().watchImport(importId)`, or `evolve dataset watch`
|
|
2782
|
+
* from any machine. Never called on the single-request door (nothing is
|
|
2783
|
+
* registered: the upload IS the request), on a fetched source, or when
|
|
2784
|
+
* the server registered nothing (an existing name@version, or an older
|
|
2785
|
+
* server).
|
|
2786
|
+
*/
|
|
2787
|
+
onRegistered?: (importId: string) => void;
|
|
2788
|
+
}
|
|
2789
|
+
/**
|
|
2790
|
+
* Dataset import status.
|
|
2791
|
+
*
|
|
2792
|
+
* The SAME four words a job uses, deliberately: a status chip rendering both
|
|
2793
|
+
* never carries a translation table. Terminal: "COMPLETED" (the corpus landed
|
|
2794
|
+
* as a dataset version; runnable once activated) and "FAILED".
|
|
2795
|
+
*/
|
|
2796
|
+
type DatasetImportStatus = "QUEUED" | "RUNNING" | "COMPLETED" | "FAILED";
|
|
2797
|
+
/** Structured failure detail for a FAILED import. */
|
|
2798
|
+
interface DatasetImportFailure {
|
|
2799
|
+
/** Stable machine-readable cause; "import_failed" when none was recorded. */
|
|
2800
|
+
code: string;
|
|
2801
|
+
/** What went wrong, e.g. "2/113 task(s) failed to parse" */
|
|
2802
|
+
message: string;
|
|
2803
|
+
/** Per-task parse/validation failures, when the corpus was reachable */
|
|
2804
|
+
failures?: {
|
|
2805
|
+
task_name: string;
|
|
2806
|
+
error: string;
|
|
2807
|
+
}[];
|
|
2808
|
+
}
|
|
2809
|
+
/**
|
|
2810
|
+
* Non-fatal but consequential import outcome. A version whose warnings include
|
|
2811
|
+
* `no_solutions_archived` permanently lacks its reference-solution record —
|
|
2812
|
+
* the record operator verification tooling reads, never a gate. The version
|
|
2813
|
+
* still publishes, activates, and runs; the warning makes the permanent gap
|
|
2814
|
+
* visible instead of silent.
|
|
2815
|
+
*
|
|
2816
|
+
* `tasks_failed_to_build` is the partial-publish model's warning: the import
|
|
2817
|
+
* COMPLETED (the version is READY — at least one task built) but some tasks
|
|
2818
|
+
* FAILED their independent build and are not runnable in this version. The
|
|
2819
|
+
* names and typed reasons are on the dataset detail's `failed_tasks`; fixing
|
|
2820
|
+
* them is a re-publish (immutable versions).
|
|
2821
|
+
*
|
|
2822
|
+
* `tests_dockerfile_not_built` names the READY tasks that ship a
|
|
2823
|
+
* tests/Dockerfile the verifier never builds — their verifier image is pinned,
|
|
2824
|
+
* or shared (upstream semantics: Harbor boots the pinned image as-is and never
|
|
2825
|
+
* builds tests/ on that shape). Not an absence and not a failure: a recorded
|
|
2826
|
+
* degrade. Each such task carries the same fact as a `tests_dockerfile_not_built`
|
|
2827
|
+
* note on the dataset detail (Task.notes).
|
|
2828
|
+
*/
|
|
2829
|
+
interface ImportWarning {
|
|
2830
|
+
code: "solutions_archiving_disabled" | "no_solutions_archived" | "partial_solutions_archived" | "tasks_failed_to_build" | "tests_dockerfile_not_built";
|
|
2831
|
+
message?: string;
|
|
2832
|
+
}
|
|
2833
|
+
/**
|
|
2834
|
+
* An asynchronous publish. Self-describing: every response names the
|
|
2835
|
+
* dataset@version being imported — the 202 from publish(), getImport(), and
|
|
2836
|
+
* listImports() all return this same shape, so a caller can render the row it
|
|
2837
|
+
* just created without a follow-up read.
|
|
2838
|
+
*/
|
|
2839
|
+
/**
|
|
2840
|
+
* The five phases of a publish, in the order they run (spec ImportPhaseName):
|
|
2841
|
+
* extracting (archive fetched and unpacked, or the git source cloned),
|
|
2842
|
+
* parsing (every task directory parsed, manifest gate included), building
|
|
2843
|
+
* (the image pool — Dockerfile build contexts and compose-service
|
|
2844
|
+
* resolutions), copying (upstream images mirrored into the platform
|
|
2845
|
+
* registry), verifying (storability census, solutions archive, the task
|
|
2846
|
+
* transaction, and the registry read-back before READY).
|
|
2847
|
+
*/
|
|
2848
|
+
type ImportPhase = "extracting" | "parsing" | "building" | "copying" | "verifying";
|
|
2849
|
+
/** One phase of the import timeline (spec ImportPhaseProgress). */
|
|
2850
|
+
interface ImportPhaseProgress {
|
|
2851
|
+
name: ImportPhase;
|
|
2852
|
+
started_at: string;
|
|
2853
|
+
/**
|
|
2854
|
+
* Absent while the phase runs — and stays absent forever on the phase a
|
|
2855
|
+
* FAILED import died in, which is how a reader finds where it died.
|
|
2856
|
+
*/
|
|
2857
|
+
completed_at?: string;
|
|
2858
|
+
/**
|
|
2859
|
+
* Units settled so far — task dirs (parsing), image-pool units (building),
|
|
2860
|
+
* unique images (copying), surviving tasks (verifying). Failures count as
|
|
2861
|
+
* settled; extracting has no unit and stays 0/0.
|
|
2862
|
+
*/
|
|
2863
|
+
done: number;
|
|
2864
|
+
total: number;
|
|
2865
|
+
/**
|
|
2866
|
+
* Of `done`, the units whose bytes already lived in the platform registry
|
|
2867
|
+
* (content-addressed hit — nothing copied). Image phases only.
|
|
2868
|
+
*/
|
|
2869
|
+
banked?: number;
|
|
2870
|
+
}
|
|
2871
|
+
/**
|
|
2872
|
+
* Live progress of a publish (spec DatasetImportProgress) — written by the
|
|
2873
|
+
* build worker at phase boundaries and coarse intervals, never per-second.
|
|
2874
|
+
* On a terminal import it is the settled record: all five phases with
|
|
2875
|
+
* wall-clock timestamps, the final image counts, and the publish's CodeBuild
|
|
2876
|
+
* copy-build minutes.
|
|
2877
|
+
*/
|
|
2878
|
+
interface DatasetImportProgress {
|
|
2879
|
+
/** What the import is doing now (the last entry of `phases`). */
|
|
2880
|
+
phase: ImportPhase;
|
|
2881
|
+
/** When the claimed run began on the worker — explicit, never derived. */
|
|
2882
|
+
started_at: string;
|
|
2883
|
+
phases: ImportPhaseProgress[];
|
|
2884
|
+
/**
|
|
2885
|
+
* The publish's cumulative image economics: `built` and `mirrored` are
|
|
2886
|
+
* fresh pushes this publish paid for; `banked` counts images that already
|
|
2887
|
+
* existed in the registry.
|
|
2888
|
+
*/
|
|
2889
|
+
images: {
|
|
2890
|
+
built: number;
|
|
2891
|
+
mirrored: number;
|
|
2892
|
+
banked: number;
|
|
2893
|
+
};
|
|
2894
|
+
/**
|
|
2895
|
+
* The publish's promotion fan-out meter: CodeBuild copy builds started and
|
|
2896
|
+
* their billed minutes. 0/0 on a fully banked re-publish.
|
|
2897
|
+
*/
|
|
2898
|
+
codebuild: {
|
|
2899
|
+
copy_builds: number;
|
|
2900
|
+
billed_minutes: number;
|
|
2901
|
+
};
|
|
2902
|
+
}
|
|
2903
|
+
interface DatasetImport {
|
|
2904
|
+
/** Import job id */
|
|
2905
|
+
id: string;
|
|
2906
|
+
status: DatasetImportStatus;
|
|
2907
|
+
/**
|
|
2908
|
+
* The register-first marker: true exactly while the corpus is still
|
|
2909
|
+
* uploading through its resumable session — the import is QUEUED and
|
|
2910
|
+
* cannot proceed without the client — and false from the moment the
|
|
2911
|
+
* publish is accepted. Absent on servers predating register-first.
|
|
2912
|
+
*/
|
|
2913
|
+
receiving?: boolean;
|
|
2914
|
+
/** Catalog dataset name the import creates or extends */
|
|
2915
|
+
name: string;
|
|
2916
|
+
/** Version label of the imported version */
|
|
2917
|
+
version: string;
|
|
2918
|
+
/**
|
|
2919
|
+
* Why the import FAILED; null otherwise. Named `failure`, never `error` —
|
|
2920
|
+
* see JobFailure.
|
|
2921
|
+
*/
|
|
2922
|
+
failure: DatasetImportFailure | null;
|
|
2923
|
+
/** Non-fatal but consequential outcomes — see ImportWarning. */
|
|
2924
|
+
warnings: ImportWarning[];
|
|
2925
|
+
/**
|
|
2926
|
+
* Live progress of the build — null until the worker's first report (a
|
|
2927
|
+
* QUEUED import, an older server, and every import that predates
|
|
2928
|
+
* progress). On a terminal import it is the settled five-phase record.
|
|
2929
|
+
*/
|
|
2930
|
+
progress: DatasetImportProgress | null;
|
|
2931
|
+
/** Number of tasks parsed, once counted */
|
|
2932
|
+
task_count?: number;
|
|
2933
|
+
created_at?: string;
|
|
2934
|
+
updated_at?: string;
|
|
2935
|
+
}
|
|
2936
|
+
/** Cursor page of dataset imports */
|
|
2937
|
+
type DatasetImportPage = Page<DatasetImport>;
|
|
2938
|
+
/** Dual-use handle from datasets().listImports(): await one page, or iterate them all */
|
|
2939
|
+
interface DatasetImportList extends Awaitable<DatasetImportPage>, AsyncIterable<DatasetImport> {
|
|
2940
|
+
}
|
|
2941
|
+
/**
|
|
2942
|
+
* Where a registered agent's executables came from: an install script run in a
|
|
2943
|
+
* throwaway builder sandbox, or a tarball uploaded from a local directory.
|
|
2944
|
+
* Echoed on every response; the SDK never guesses it.
|
|
2945
|
+
*/
|
|
2946
|
+
type AgentSource = "install_script" | "tarball";
|
|
2947
|
+
/**
|
|
2948
|
+
* A private agent registered by the caller. Once registered, its `name` is
|
|
2949
|
+
* usable in job `agents[].name` exactly like a built-in ("claude", "codex",
|
|
2950
|
+
* ...). Private to its owner: another user's name reads as `agent_not_found`,
|
|
2951
|
+
* never as a permission error — existence is never leaked.
|
|
2952
|
+
*/
|
|
2953
|
+
interface Agent {
|
|
2954
|
+
/** The name to put in job agents[].name */
|
|
2955
|
+
name: string;
|
|
2956
|
+
/** How the executables were produced */
|
|
2957
|
+
source: AgentSource;
|
|
2958
|
+
/** The command run headless with `sh -c` at the task working directory */
|
|
2959
|
+
run_command: string;
|
|
2960
|
+
/**
|
|
2961
|
+
* Caller-declared env injected at RUN time only. It may not override the run
|
|
2962
|
+
* contract's own keys — the server rejects that at registration with
|
|
2963
|
+
* `agent_invalid_env`.
|
|
2964
|
+
*/
|
|
2965
|
+
env: Record<string, string>;
|
|
2966
|
+
created_at: string;
|
|
2967
|
+
updated_at: string;
|
|
2968
|
+
}
|
|
2969
|
+
/**
|
|
2970
|
+
* The two sources a registered agent's executables can come from. A union, not
|
|
2971
|
+
* two optional fields — see DatasetSource for why `?: never` is load-bearing
|
|
2972
|
+
* rather than decorative.
|
|
2973
|
+
*/
|
|
2974
|
+
type AgentSourceInput = {
|
|
2975
|
+
/**
|
|
2976
|
+
* The install script itself (not a path). It runs in a throwaway builder
|
|
2977
|
+
* sandbox that has internet and ZERO secrets, so everything it fetches
|
|
2978
|
+
* must be publicly fetchable, and it must leave executables in
|
|
2979
|
+
* `$PREFIX/bin`.
|
|
2980
|
+
*/
|
|
2981
|
+
install_script: string;
|
|
2982
|
+
directory?: never;
|
|
2983
|
+
} | {
|
|
2984
|
+
/**
|
|
2985
|
+
* A local directory holding the agent — tarred + gzipped and uploaded.
|
|
2986
|
+
* Same build rules as an install script.
|
|
2987
|
+
*/
|
|
2988
|
+
directory: string;
|
|
2989
|
+
install_script?: never;
|
|
2990
|
+
};
|
|
2991
|
+
/**
|
|
2992
|
+
* Input for agents().create(): a name, a run command, and EXACTLY ONE source.
|
|
2993
|
+
* The source half is a union, so omitting both or passing both is a compile
|
|
2994
|
+
* error rather than a 400 the caller discovers at run time.
|
|
2995
|
+
*/
|
|
2996
|
+
type AgentInput = AgentSourceInput & {
|
|
2997
|
+
/** Agent name; also the value used later in job agents[].name */
|
|
2998
|
+
name: string;
|
|
2999
|
+
/** Command run headless with `sh -c` at the task working directory */
|
|
3000
|
+
run_command: string;
|
|
3001
|
+
/** Env injected at RUN time only; may not override the run contract's keys */
|
|
3002
|
+
env?: Record<string, string>;
|
|
3003
|
+
};
|
|
3004
|
+
/**
|
|
3005
|
+
* An agent upsert body. Same shape as AgentInput minus `name`, which the
|
|
3006
|
+
* upsert takes as its first argument — the name is the resource identity, not
|
|
3007
|
+
* a field of it.
|
|
3008
|
+
*/
|
|
3009
|
+
type AgentUpsertInput = AgentSourceInput & {
|
|
3010
|
+
/** Command run headless with `sh -c` at the task working directory */
|
|
3011
|
+
run_command: string;
|
|
3012
|
+
/** Env injected at RUN time only; may not override the run contract's keys */
|
|
3013
|
+
env?: Record<string, string>;
|
|
3014
|
+
};
|
|
3015
|
+
/** Cursor page of registered agents */
|
|
3016
|
+
type AgentPage = Page<Agent>;
|
|
3017
|
+
/** Dual-use handle from agents().list(): await one page, or iterate them all */
|
|
3018
|
+
interface AgentList extends Awaitable<AgentPage>, AsyncIterable<Agent> {
|
|
3019
|
+
}
|
|
3020
|
+
/** Options for jobs().start() and resume() */
|
|
3021
|
+
interface StartJobOptions {
|
|
3022
|
+
/**
|
|
3023
|
+
* Idempotency-Key header value: retries with the same key return the
|
|
3024
|
+
* original job (idempotent_replay: true) instead of creating a new one.
|
|
3025
|
+
*/
|
|
3026
|
+
idempotencyKey?: string;
|
|
3027
|
+
}
|
|
3028
|
+
/** Options for jobs().list() (default page 50, max 200) */
|
|
3029
|
+
interface ListJobsOptions extends PageOptions {
|
|
3030
|
+
/** Server-side free-text filter over job name and dataset names. */
|
|
3031
|
+
search?: string;
|
|
3032
|
+
/**
|
|
3033
|
+
* Visibility scope (Harbor's `--scope`): `my` — jobs you created, the
|
|
3034
|
+
* server's default; `shared` — your organizations' jobs that teammates
|
|
3035
|
+
* created. See JOB_LIST_SCOPES.
|
|
3036
|
+
*/
|
|
3037
|
+
scope?: JobListScope;
|
|
3038
|
+
}
|
|
3039
|
+
/** Options for analyses().list() (default page 50, max 200) */
|
|
3040
|
+
interface ListAnalysesOptions extends PageOptions {
|
|
3041
|
+
/** Visibility scope, exactly as on jobs().list(). */
|
|
3042
|
+
scope?: JobListScope;
|
|
3043
|
+
/** Only analyses of this job's trials. */
|
|
3044
|
+
job?: string;
|
|
3045
|
+
/** Only analyses in these statuses (the object's own lowercase ladder). */
|
|
3046
|
+
status?: AnalysisStatus[];
|
|
3047
|
+
}
|
|
3048
|
+
/** Options for checks().list() (default page 50, max 200) */
|
|
3049
|
+
interface ListChecksOptions extends PageOptions {
|
|
3050
|
+
/** Visibility scope, exactly as on jobs().list(): `my` (the default) or `shared`. */
|
|
3051
|
+
scope?: JobListScope;
|
|
3052
|
+
/** Only checks in these statuses (the check's own ladder, CHECK_STATUSES). */
|
|
3053
|
+
status?: CheckStatus[];
|
|
3054
|
+
}
|
|
3055
|
+
/** Options for jobs().tasks() (default page 50, max 200) */
|
|
3056
|
+
interface ListJobTasksOptions extends PageOptions {
|
|
3057
|
+
}
|
|
3058
|
+
/** Options for jobs().trials() (default page 50, max 200) */
|
|
3059
|
+
interface ListTrialsOptions extends PageOptions {
|
|
3060
|
+
/** Only trials in these statuses (e.g. the failures behind a resume decision) */
|
|
3061
|
+
status?: TrialStatus[];
|
|
3062
|
+
/** Only one dataset's trials — exact match on the trial's `source`. */
|
|
3063
|
+
dataset?: string;
|
|
3064
|
+
}
|
|
3065
|
+
/** Options for datasets().list() (default page 50, max 200) */
|
|
3066
|
+
interface ListDatasetsOptions extends PageOptions {
|
|
3067
|
+
/** Server-side free-text filter over name and description. */
|
|
3068
|
+
search?: string;
|
|
3069
|
+
}
|
|
3070
|
+
/** Options for agents().list() (default page 50, max 200) */
|
|
3071
|
+
interface ListAgentsOptions extends PageOptions {
|
|
3072
|
+
}
|
|
3073
|
+
/** Options for datasets().get() / getActive(): pages the TASK list (default 200, max 500) */
|
|
3074
|
+
interface GetDatasetOptions extends PageOptions {
|
|
3075
|
+
}
|
|
3076
|
+
/** Options for datasets().listImports() */
|
|
3077
|
+
interface ListImportsOptions extends PageOptions {
|
|
3078
|
+
/** Only imports in this status */
|
|
3079
|
+
status?: DatasetImportStatus;
|
|
3080
|
+
/** Only imports of this dataset name */
|
|
3081
|
+
dataset?: string;
|
|
3082
|
+
}
|
|
3083
|
+
/** Options for trials().trace() and traceEvents() */
|
|
3084
|
+
interface TraceOptions extends PageOptions {
|
|
3085
|
+
/**
|
|
3086
|
+
* Resume position: events with seq strictly greater than this cursor (omit =
|
|
3087
|
+
* from the beginning). A trace cursor IS a seq, so to resume a poll later
|
|
3088
|
+
* pass the last event's `seq` here as a string.
|
|
3089
|
+
*/
|
|
3090
|
+
cursor?: string;
|
|
3091
|
+
/** Max events per page (server default: 200, max: 1000) */
|
|
3092
|
+
limit?: number;
|
|
3093
|
+
/** Only events of exactly this type. */
|
|
3094
|
+
type?: string;
|
|
3095
|
+
/**
|
|
3096
|
+
* Only events whose type or serialized content matches this
|
|
3097
|
+
* case-insensitive POSIX regex (a plain string is a plain substring —
|
|
3098
|
+
* grep's own grammar). An invalid pattern is the server's typed
|
|
3099
|
+
* `invalid_input` refusal.
|
|
3100
|
+
*/
|
|
3101
|
+
grep?: string;
|
|
3102
|
+
/**
|
|
3103
|
+
* Only the last N MATCHING events — a floor on the seq timeline, after
|
|
3104
|
+
* which cursor paging proceeds normally, oldest-first.
|
|
3105
|
+
*/
|
|
3106
|
+
tail?: number;
|
|
3107
|
+
}
|
|
3108
|
+
/** Options for jobs().grep() */
|
|
3109
|
+
interface GrepJobOptions extends PageOptions {
|
|
3110
|
+
/** Only search events of exactly this type. */
|
|
3111
|
+
type?: string;
|
|
3112
|
+
}
|
|
3113
|
+
/**
|
|
3114
|
+
* One trial's slice of a job-wide grep (jobs().grep()): the EXACT number of
|
|
3115
|
+
* matching events plus the first few of them — the platform caps the sample
|
|
3116
|
+
* (at 5), the count never truncates. The full match list of one trial is
|
|
3117
|
+
* trials().trace() with the same pattern as { grep }.
|
|
3118
|
+
*/
|
|
3119
|
+
interface JobGrepGroup {
|
|
3120
|
+
trial_id: string;
|
|
3121
|
+
/** The trial's task, for orientation; null only when the task row is gone. */
|
|
3122
|
+
task_name: string | null;
|
|
3123
|
+
match_count: number;
|
|
3124
|
+
events: TraceEvent[];
|
|
3125
|
+
}
|
|
3126
|
+
/** One page of a job-wide grep, ordered by trial id. */
|
|
3127
|
+
type JobGrepPage = Page<JobGrepGroup>;
|
|
3128
|
+
/**
|
|
3129
|
+
* One stored file of a trial's tree (trials().files()), named by its
|
|
3130
|
+
* prefix-relative path — the same path trials().file() reads.
|
|
3131
|
+
*/
|
|
3132
|
+
interface TrialFile {
|
|
3133
|
+
path: string;
|
|
3134
|
+
size_bytes: number;
|
|
3135
|
+
}
|
|
3136
|
+
/** One page of a trial's stored file tree, sorted by path. */
|
|
3137
|
+
type TrialFilePage = Page<TrialFile>;
|
|
3138
|
+
/** Options for trials().files() (default page 200, max 1000) */
|
|
3139
|
+
interface ListTrialFilesOptions extends PageOptions {
|
|
3140
|
+
}
|
|
3141
|
+
/**
|
|
3142
|
+
* Byte range for trials().file() — inclusive positions, the wire's
|
|
3143
|
+
* `Range: bytes=start-end` grammar: { start } alone reads to the end,
|
|
3144
|
+
* { suffix } alone reads the last N bytes.
|
|
3145
|
+
*/
|
|
3146
|
+
interface TrialFileRange {
|
|
3147
|
+
start?: number;
|
|
3148
|
+
end?: number;
|
|
3149
|
+
/** The last N bytes (mutually exclusive with start/end). */
|
|
3150
|
+
suffix?: number;
|
|
3151
|
+
}
|
|
3152
|
+
/** Options for datasets().watchImport() */
|
|
3153
|
+
interface WatchImportOptions {
|
|
3154
|
+
/** Called on every observed import status change (including the first status seen) */
|
|
3155
|
+
onStatus?: (datasetImport: DatasetImport) => void;
|
|
3156
|
+
/**
|
|
3157
|
+
* Called on every observed change of the import's live `progress` — a
|
|
3158
|
+
* phase boundary, or new counts inside a phase. The server writes progress
|
|
3159
|
+
* at phase boundaries and coarse intervals (never per-second), so this
|
|
3160
|
+
* fires at that cadence, under the same poll (and the same 429-tolerant
|
|
3161
|
+
* posture) as `onStatus`. Never called while `progress` is null.
|
|
3162
|
+
*/
|
|
3163
|
+
onProgress?: (progress: DatasetImportProgress, datasetImport: DatasetImport) => void;
|
|
3164
|
+
/**
|
|
3165
|
+
* Called on every observed change of the imported VERSION's state during
|
|
3166
|
+
* the watch's settle phase — normally the single confirming READY read
|
|
3167
|
+
* (COMPLETED means the version is READY under build-then-READY), or the
|
|
3168
|
+
* walk to READY/ARCHIVED/FAILED against a mid-deploy older server.
|
|
3169
|
+
* `dataset` is the detail read the observation came from: its
|
|
3170
|
+
* `active_version` says whether the settled version is now the one a bare
|
|
3171
|
+
* dataset name resolves to.
|
|
3172
|
+
*/
|
|
3173
|
+
onVersion?: (version: DatasetVersion, dataset: Dataset) => void;
|
|
3174
|
+
/** Abort the watch (rejects with the abort reason) */
|
|
3175
|
+
signal?: AbortSignal;
|
|
3176
|
+
/** Poll interval between polls (default: 2000ms) */
|
|
3177
|
+
pollIntervalMs?: number;
|
|
3178
|
+
/**
|
|
3179
|
+
* Backstop bound on the settle phase — how long past import COMPLETED the
|
|
3180
|
+
* watch may wait for the version to settle before refusing with
|
|
3181
|
+
* ImportSettleError("settle_timeout") (default: 30 minutes). Normally
|
|
3182
|
+
* unused (COMPLETED means READY); it bounds the wait against a mid-deploy
|
|
3183
|
+
* older server, and the error carries the last observed state.
|
|
3184
|
+
*/
|
|
3185
|
+
settleTimeoutMs?: number;
|
|
3186
|
+
}
|
|
3187
|
+
/** Options for jobs().watchAnalysis() */
|
|
3188
|
+
interface WatchAnalysisOptions {
|
|
3189
|
+
/**
|
|
3190
|
+
* Called on every observed change of the job's analysis tally (including
|
|
3191
|
+
* the first non-null one seen), with the job body the observation came
|
|
3192
|
+
* from.
|
|
3193
|
+
*/
|
|
3194
|
+
onStats?: (job: Job) => void;
|
|
3195
|
+
/** Abort the watch (rejects with the abort reason) */
|
|
3196
|
+
signal?: AbortSignal;
|
|
3197
|
+
/**
|
|
3198
|
+
* Initial poll interval (default: 2000ms). Doubles while the tally stands
|
|
3199
|
+
* still, up to the 30-s ceiling the job watch's reconnect uses, and
|
|
3200
|
+
* returns to this value on every tally change.
|
|
3201
|
+
*/
|
|
3202
|
+
pollIntervalMs?: number;
|
|
3203
|
+
}
|
|
3204
|
+
/** Options for jobs().watch() */
|
|
3205
|
+
interface WatchJobOptions {
|
|
3206
|
+
/** Called for every event (replayed + live) */
|
|
3207
|
+
onEvent?: (event: JobEvent) => void;
|
|
3208
|
+
/** Abort the watch (rejects with the abort reason) */
|
|
3209
|
+
signal?: AbortSignal;
|
|
3210
|
+
/** Initial reconnect backoff (default: 1000ms; doubles up to maxReconnectDelayMs) */
|
|
3211
|
+
reconnectDelayMs?: number;
|
|
3212
|
+
/** Backoff ceiling (default: 30000ms) */
|
|
3213
|
+
maxReconnectDelayMs?: number;
|
|
3214
|
+
}
|
|
3215
|
+
/** Delivery options for datasets().download() */
|
|
3216
|
+
interface DownloadDatasetOptions {
|
|
3217
|
+
/** Directory to save the package into (returns the file path) */
|
|
3218
|
+
to?: string;
|
|
3219
|
+
/** Return the raw response stream instead of a Buffer */
|
|
3220
|
+
stream?: boolean;
|
|
3221
|
+
}
|
|
3222
|
+
/** Delivery options for jobs().download() — and for checks().download() / analyses().download(), the same three shapes. */
|
|
3223
|
+
interface DownloadJobOptions {
|
|
3224
|
+
/** Directory to save the archive into (returns the file path) */
|
|
3225
|
+
to?: string;
|
|
3226
|
+
/** Return the raw response stream instead of a Buffer */
|
|
3227
|
+
stream?: boolean;
|
|
3228
|
+
}
|
|
3229
|
+
/** Options for jobs().upload() */
|
|
3230
|
+
interface UploadJobOptions {
|
|
3231
|
+
/**
|
|
3232
|
+
* "name" or "name@version" of a published dataset — links the uploaded
|
|
3233
|
+
* trials to that version's tasks by task name (a bare name resolves to the
|
|
3234
|
+
* active version). Matched trials analyze against the real task content;
|
|
3235
|
+
* unmatched or unhinted trials analyze through the task-not-available
|
|
3236
|
+
* branch, exactly Harbor's fallback for a trial without a local task
|
|
3237
|
+
* directory. Resolved at ingest: a hint the caller cannot use fails the
|
|
3238
|
+
* import typed (the job-create vocabulary).
|
|
3239
|
+
*/
|
|
3240
|
+
dataset?: string;
|
|
3241
|
+
/**
|
|
3242
|
+
* Client-side upload progress: `(sentBytes, totalBytes)` over the
|
|
3243
|
+
* archive's bytes as the stream flushes them — the same reading
|
|
3244
|
+
* datasets().publish() hands out (PublishDatasetOptions.onUploadProgress).
|
|
3245
|
+
* Not called for an `archive_url` source (no bytes ride the request).
|
|
3246
|
+
*/
|
|
3247
|
+
onUploadProgress?: (sentBytes: number, totalBytes: number) => void;
|
|
3248
|
+
/**
|
|
3249
|
+
* Register-first: called once with the import id BEFORE the first byte
|
|
3250
|
+
* moves when the archive rides the resumable session door (over the
|
|
3251
|
+
* 256 MiB switch), so a watcher may attach — `jobs().watchImport(id)`,
|
|
3252
|
+
* `evolve job import <id> --watch` — while the transfer runs. The SAME id
|
|
3253
|
+
* upload() resolves with. Not called on the single-POST path, where the
|
|
3254
|
+
* id exists only once the 202 lands.
|
|
3255
|
+
*/
|
|
3256
|
+
onRegistered?: (importId: string) => void;
|
|
3257
|
+
}
|
|
3258
|
+
/**
|
|
3259
|
+
* Where a job import's archive came from (spec JobImportSource): the
|
|
3260
|
+
* uploaded bytes by their sha256, or the public https URL the worker
|
|
3261
|
+
* downloads. `hub` — Harbor's own hub job source (`SourceJobConfig(type=
|
|
3262
|
+
* "hub", job_id=...)`) — is RESERVED: the union carries the word so the arm
|
|
3263
|
+
* is additive when it lands; no door accepts it yet.
|
|
3264
|
+
*/
|
|
3265
|
+
type JobImportSource = {
|
|
3266
|
+
type: "archive";
|
|
3267
|
+
sha256: string;
|
|
3268
|
+
} | {
|
|
3269
|
+
type: "archive_url";
|
|
3270
|
+
url: string;
|
|
3271
|
+
} | {
|
|
3272
|
+
type: "hub";
|
|
3273
|
+
job_id: string;
|
|
3274
|
+
};
|
|
3275
|
+
/**
|
|
3276
|
+
* The four phases of a job ingest, in the order the worker runs them:
|
|
3277
|
+
* fetching (the archive from the store, or the URL download), extracting,
|
|
3278
|
+
* validating (every gate before any row), ingesting (the rows and the
|
|
3279
|
+
* trace-artifact stores).
|
|
3280
|
+
*/
|
|
3281
|
+
type JobImportPhaseName = "fetching" | "extracting" | "validating" | "ingesting";
|
|
3282
|
+
/** One phase of a job import's timeline (spec JobImportProgress.phases[]). */
|
|
3283
|
+
interface JobImportPhaseProgress {
|
|
3284
|
+
name: JobImportPhaseName;
|
|
3285
|
+
started_at: string;
|
|
3286
|
+
/** Absent while the phase runs — and forever, on the phase a FAILED import died in. */
|
|
3287
|
+
completed_at?: string;
|
|
3288
|
+
}
|
|
3289
|
+
/**
|
|
3290
|
+
* The worker's own statement of where a job ingest stands (spec
|
|
3291
|
+
* JobImportProgress), written at phase boundaries only. On a terminal
|
|
3292
|
+
* import it is the settled record.
|
|
3293
|
+
*/
|
|
3294
|
+
interface JobImportProgress {
|
|
3295
|
+
phase: JobImportPhaseName;
|
|
3296
|
+
started_at: string;
|
|
3297
|
+
phases: JobImportPhaseProgress[];
|
|
3298
|
+
}
|
|
3299
|
+
/**
|
|
3300
|
+
* Why a job import FAILED (spec JobImportFailure). The codes are the ones
|
|
3301
|
+
* the upload door once answered synchronously — `invalid_archive`,
|
|
3302
|
+
* `upload_too_large`, `not_a_job_dir`, `job_already_uploaded` (details name
|
|
3303
|
+
* `existing_job_id`), the dataset-hint codes, `invalid_trial` (details
|
|
3304
|
+
* name the `trial`) — plus the platform's own `import_failed` and
|
|
3305
|
+
* `import_lease_expired`. A trial over a physical per-trial bound is not a
|
|
3306
|
+
* failure: it is skipped (JobImportSkippedTrial) and the import
|
|
3307
|
+
* completes.
|
|
3308
|
+
*/
|
|
3309
|
+
interface JobImportFailure {
|
|
3310
|
+
code: string;
|
|
3311
|
+
message: string;
|
|
3312
|
+
details?: Record<string, unknown>;
|
|
3313
|
+
}
|
|
3314
|
+
/**
|
|
3315
|
+
* One trial a job import LEFT OUT, typed (spec JobImportSkippedTrial): the
|
|
3316
|
+
* failure-envelope grammar plus the trial directory it names.
|
|
3317
|
+
* `trial_too_large` is the one cause, every bound physical and named in
|
|
3318
|
+
* the message: the named `file` is a verbatim artifact over what the
|
|
3319
|
+
* store's single PutObject can land (S3's 5 GiB), or
|
|
3320
|
+
* `agent/trajectory.json` — with the `agent/` home tree counted in — would
|
|
3321
|
+
* cost more heap to parse than the worker can still commit at that moment
|
|
3322
|
+
* (V8's live reading; its structure counted from the bytes, never parsed),
|
|
3323
|
+
* or the `agent/` tree alone (`file` spelled `agent/`) is over that
|
|
3324
|
+
* reading; `details` carry the `bytes` measured and the `max_bytes`
|
|
3325
|
+
* reading. The rest of the archive lands; a skipped trial contributes
|
|
3326
|
+
* nothing to the job.
|
|
3327
|
+
*/
|
|
3328
|
+
interface JobImportSkippedTrial {
|
|
3329
|
+
trial: string;
|
|
3330
|
+
code: "trial_too_large";
|
|
3331
|
+
message: string;
|
|
3332
|
+
details?: {
|
|
3333
|
+
file: string;
|
|
3334
|
+
bytes: number;
|
|
3335
|
+
max_bytes: number;
|
|
3336
|
+
};
|
|
3337
|
+
}
|
|
3338
|
+
/**
|
|
3339
|
+
* An accepted job upload, from the 202 to a COMPLETED Job or a typed FAILED
|
|
3340
|
+
* — what jobs().upload() resolves with and jobs().getImport()/watchImport()
|
|
3341
|
+
* read. Same four status words a dataset import speaks.
|
|
3342
|
+
*/
|
|
3343
|
+
interface JobImport {
|
|
3344
|
+
id: string;
|
|
3345
|
+
status: DatasetImportStatus;
|
|
3346
|
+
/**
|
|
3347
|
+
* Register-first: true exactly while the archive is still arriving
|
|
3348
|
+
* through its resumable session (QUEUED, not yet claimable), false from
|
|
3349
|
+
* the 202 on.
|
|
3350
|
+
*/
|
|
3351
|
+
receiving: boolean;
|
|
3352
|
+
/** Null while a session is still receiving (nothing is stored yet). */
|
|
3353
|
+
source: JobImportSource | null;
|
|
3354
|
+
/** The `dataset` hint as given (`name` or `name@version`), or null. */
|
|
3355
|
+
dataset: string | null;
|
|
3356
|
+
/**
|
|
3357
|
+
* The ingested Job, from COMPLETED on — read it with jobs().get(). Null
|
|
3358
|
+
* again if that job was deleted (delete-then-reupload): the import stays
|
|
3359
|
+
* COMPLETED and honest.
|
|
3360
|
+
*/
|
|
3361
|
+
job_id: string | null;
|
|
3362
|
+
/** Trials the ingested job carries, from COMPLETED on (Harbor's own spelling). */
|
|
3363
|
+
n_trials_uploaded: number | null;
|
|
3364
|
+
/**
|
|
3365
|
+
* Trials the ingest left out, typed, from COMPLETED on — 0 when none
|
|
3366
|
+
* (Harbor's own spelling). Null until COMPLETED.
|
|
3367
|
+
*/
|
|
3368
|
+
n_trials_skipped: number | null;
|
|
3369
|
+
/** One entry per skipped trial, in archive order, from COMPLETED on ([] when none). Null until COMPLETED. */
|
|
3370
|
+
skipped_trials: JobImportSkippedTrial[] | null;
|
|
3371
|
+
/**
|
|
3372
|
+
* The ingested job's per-task task-linkage roll-up — the same rows the
|
|
3373
|
+
* job serves as `upload.task_links`, read from the job, so a watcher
|
|
3374
|
+
* learns from the import alone which tasks will analyze with their task
|
|
3375
|
+
* folder and why the rest will not. Null until COMPLETED, null again when
|
|
3376
|
+
* the job is gone (like `job_id`), and null on a job ingested before the
|
|
3377
|
+
* link law existed.
|
|
3378
|
+
*/
|
|
3379
|
+
task_links: JobTaskLink[] | null;
|
|
3380
|
+
failure: JobImportFailure | null;
|
|
3381
|
+
/** Null until the worker's first report (a QUEUED import). */
|
|
3382
|
+
progress: JobImportProgress | null;
|
|
3383
|
+
created_at?: string;
|
|
3384
|
+
updated_at?: string;
|
|
3385
|
+
}
|
|
3386
|
+
/** Cursor page of job imports */
|
|
3387
|
+
type JobImportPage = Page<JobImport>;
|
|
3388
|
+
/** Dual-use handle from jobs().listImports(): await one page, or iterate them all */
|
|
3389
|
+
interface JobImportList extends Awaitable<JobImportPage>, AsyncIterable<JobImport> {
|
|
3390
|
+
}
|
|
3391
|
+
/** Options for jobs().listImports() */
|
|
3392
|
+
interface ListJobImportsOptions extends PageOptions {
|
|
3393
|
+
/** Only imports in this status */
|
|
3394
|
+
status?: DatasetImportStatus;
|
|
3395
|
+
}
|
|
3396
|
+
/** Options for jobs().watchImport() */
|
|
3397
|
+
interface WatchJobImportOptions {
|
|
3398
|
+
/** Called on every observed import status change (including the first status seen, and the receiving flip). */
|
|
3399
|
+
onStatus?: (jobImport: JobImport) => void;
|
|
3400
|
+
/**
|
|
3401
|
+
* Called on every observed change of the import's live `progress` — a
|
|
3402
|
+
* phase boundary; the server writes at that cadence. Never called while
|
|
3403
|
+
* `progress` is null.
|
|
3404
|
+
*/
|
|
3405
|
+
onProgress?: (progress: JobImportProgress, jobImport: JobImport) => void;
|
|
3406
|
+
/** Poll interval in ms (default 2000) */
|
|
3407
|
+
pollIntervalMs?: number;
|
|
3408
|
+
/** Abort the watch (throws AbortError-shaped Error) */
|
|
3409
|
+
signal?: AbortSignal;
|
|
3410
|
+
}
|
|
3411
|
+
/** Client for the shared dataset catalog */
|
|
3412
|
+
interface DatasetsClient {
|
|
3413
|
+
/**
|
|
3414
|
+
* List datasets with their active versions (cursor-paged). Await the
|
|
3415
|
+
* result for one page, or `for await` it to walk the whole catalog.
|
|
3416
|
+
*/
|
|
3417
|
+
list(options?: ListDatasetsOptions): DatasetList;
|
|
3418
|
+
/**
|
|
3419
|
+
* Get one dataset: all versions + one page of the selected version's tasks.
|
|
3420
|
+
* ref is "name" (active version's tasks) or "name@version"; { limit, cursor }
|
|
3421
|
+
* page the tasks.
|
|
3422
|
+
*/
|
|
3423
|
+
get(ref: string, options?: GetDatasetOptions): Promise<Dataset>;
|
|
3424
|
+
/**
|
|
3425
|
+
* Get a dataset's active version resolved to a runnable shape: unlike
|
|
3426
|
+
* get(), `version` and `tasks` are guaranteed present. Throws
|
|
3427
|
+
* NoActiveVersionError when the dataset has no active version. Use get()
|
|
3428
|
+
* for the full multi-version detail with optional fields.
|
|
3429
|
+
*/
|
|
3430
|
+
getActive(name: string, options?: GetDatasetOptions): Promise<ActiveDataset>;
|
|
3431
|
+
/**
|
|
3432
|
+
* The failure-detail read of the partial-publish model: one task's own
|
|
3433
|
+
* build outcome inside one published version — its state (READY or
|
|
3434
|
+
* FAILED), the typed failure WITH the failing-step excerpt, and the full
|
|
3435
|
+
* build-log pointer. The dataset detail's `failed_tasks` carries the
|
|
3436
|
+
* compact reasons for every failed task; this call is where the excerpt
|
|
3437
|
+
* and the log pointer live, one task at a time.
|
|
3438
|
+
*
|
|
3439
|
+
* `ref` must pin the version: "name@version" (the outcome is a fact about
|
|
3440
|
+
* one immutable version, so there is no active-version reading to guess).
|
|
3441
|
+
* A task the build has not settled — or a name the corpus never contained
|
|
3442
|
+
* — answers 404 `task_not_found`.
|
|
3443
|
+
*/
|
|
3444
|
+
getTaskBuild(ref: string, taskName: string): Promise<TaskBuild>;
|
|
3445
|
+
/**
|
|
3446
|
+
* Pre-flight a local corpus BEFORE publishing (dry run): collect only the
|
|
3447
|
+
* metadata files (each task's task.toml + the optional dataset.toml —
|
|
3448
|
+
* kilobytes), run the import's own toml-decidable guards and per-provider
|
|
3449
|
+
* capability stamps server-side, and answer per-task verdicts with the
|
|
3450
|
+
* importer's would-refuse sentences. Nothing is written and no corpus
|
|
3451
|
+
* byte moves. `evolve dataset publish --dir` runs this automatically;
|
|
3452
|
+
* `evolve dataset check <dir>` is the standalone verb.
|
|
3453
|
+
*/
|
|
3454
|
+
preflight(input: PreflightDatasetInput): Promise<DatasetPreflight>;
|
|
3455
|
+
/**
|
|
3456
|
+
* Publish a dataset version (asynchronous server-side import) from a git
|
|
3457
|
+
* source pinned to a ref, or a local corpus directory. Returns immediately;
|
|
3458
|
+
* poll with getImport()/watchImport().
|
|
3459
|
+
*/
|
|
3460
|
+
publish(input: PublishDatasetInput, options?: PublishDatasetOptions): Promise<DatasetImport>;
|
|
3461
|
+
/** Get an import job's status (failure, warnings, and task_count when available) */
|
|
3462
|
+
getImport(id: string): Promise<DatasetImport>;
|
|
3463
|
+
/**
|
|
3464
|
+
* Watch a publish to its settled end: poll getImport() until the import is
|
|
3465
|
+
* terminal, then confirm the version against the dataset detail. Under
|
|
3466
|
+
* build-then-READY, COMPLETED means the version is READY — the build
|
|
3467
|
+
* settled with at least one task ready (the partial-publish model; each
|
|
3468
|
+
* provider builds its boot artifact lazily at the first trial) and, on an
|
|
3469
|
+
* owner-stamped dataset,
|
|
3470
|
+
* already active — so the settle phase is normally the single confirming
|
|
3471
|
+
* read; against a mid-deploy older server it keeps polling until the
|
|
3472
|
+
* version reaches READY, ARCHIVED, or FAILED. A failed build rides the
|
|
3473
|
+
* returned import's `failure`. Throws ImportSettleError("settle_timeout")
|
|
3474
|
+
* when settleTimeoutMs elapses before the version settles.
|
|
3475
|
+
*/
|
|
3476
|
+
watchImport(id: string, options?: WatchImportOptions): Promise<DatasetImport>;
|
|
3477
|
+
/**
|
|
3478
|
+
* List the caller's own imports, newest first (cursor-paged). This is how
|
|
3479
|
+
* you find an import again after losing the id publish() returned. Await
|
|
3480
|
+
* for one page, or `for await` to walk them all. { status } filters on the
|
|
3481
|
+
* import vocabulary; { dataset } narrows to one dataset name.
|
|
3482
|
+
*/
|
|
3483
|
+
listImports(options?: ListImportsOptions): DatasetImportList;
|
|
3484
|
+
/**
|
|
3485
|
+
* Download the ORIGINAL corpus package one of your own dataset versions was
|
|
3486
|
+
* published from — the gzipped tarball you uploaded, or, for a git publish,
|
|
3487
|
+
* the checked-out tree packed at import time. `ref` is "name" (the active
|
|
3488
|
+
* version's package) or "name@version".
|
|
3489
|
+
*
|
|
3490
|
+
* OWNER ONLY. This is the one call that returns task files, and it returns
|
|
3491
|
+
* them only to the account that owns the dataset; a platform-curated dataset
|
|
3492
|
+
* has no owner, so nobody can download it. Someone else's dataset is a plain
|
|
3493
|
+
* not-found, never a 403.
|
|
3494
|
+
*
|
|
3495
|
+
* The server verifies the stored bytes against their recorded sha256 before
|
|
3496
|
+
* sending anything and echoes the digest; the client re-checks the digest
|
|
3497
|
+
* and the Content-Length, so a successful call is byte-identical to what was
|
|
3498
|
+
* published. A version published before packages were retained has none
|
|
3499
|
+
* (`package_not_retained`, distinct from "not found").
|
|
3500
|
+
*
|
|
3501
|
+
* Default: Buffer. { to } saves into a directory and returns the file path.
|
|
3502
|
+
* { stream: true } returns the raw response stream.
|
|
3503
|
+
*/
|
|
3504
|
+
download(ref: string): Promise<Buffer>;
|
|
3505
|
+
download(ref: string, options: {
|
|
3506
|
+
to: string;
|
|
3507
|
+
}): Promise<string>;
|
|
3508
|
+
download(ref: string, options: {
|
|
3509
|
+
stream: true;
|
|
3510
|
+
}): Promise<ReadableStream<Uint8Array>>;
|
|
3511
|
+
download(ref: string, options?: DownloadDatasetOptions): Promise<Buffer | string | ReadableStream<Uint8Array>>;
|
|
3512
|
+
/**
|
|
3513
|
+
* Update dataset settings. The only settable field is
|
|
3514
|
+
* `upstream_auto_import`: automatically import a new version when the
|
|
3515
|
+
* dataset's upstream git ref moves. Refused (upstream_not_watchable) when
|
|
3516
|
+
* the dataset has no moving git ref to follow, and dataset_not_owned on a
|
|
3517
|
+
* platform-curated dataset. Returns the updated dataset.
|
|
3518
|
+
*/
|
|
3519
|
+
update(name: string, patch: DatasetPatch): Promise<Dataset>;
|
|
3520
|
+
/**
|
|
3521
|
+
* Activate a READY version you own: bare-name job references resolve to it
|
|
3522
|
+
* from then on. A publish activates its own version when it lands, so this
|
|
3523
|
+
* verb is for re-pointing the default — a rollback to an older READY
|
|
3524
|
+
* version, or choosing between several. Refused with `version_not_ready`
|
|
3525
|
+
* while the publish is still building and `version_not_activatable` for a
|
|
3526
|
+
* FAILED or ARCHIVED version, which can never be activated.
|
|
3527
|
+
*/
|
|
3528
|
+
activate(name: string, version: string): Promise<Dataset>;
|
|
3529
|
+
/**
|
|
3530
|
+
* Delete a dataset you own, with every version, task, and archived
|
|
3531
|
+
* solution. Refused (dataset_in_use) while any job still references it — a
|
|
3532
|
+
* dataset is never deleted out from under a job that measured against it,
|
|
3533
|
+
* and `err.details.sampleJobIds` names the jobs blocking it. A platform
|
|
3534
|
+
* dataset is refused with dataset_not_owned; a name you cannot see is a
|
|
3535
|
+
* plain not-found.
|
|
3536
|
+
*/
|
|
3537
|
+
delete(name: string): Promise<void>;
|
|
3538
|
+
}
|
|
3539
|
+
/** Client for the caller's own private (bring-your-own) agents */
|
|
3540
|
+
interface AgentsClient {
|
|
3541
|
+
/**
|
|
3542
|
+
* Register a private agent. Provide either an install script
|
|
3543
|
+
* (`{ install_script }`) or a local directory (`{ directory }`), never both.
|
|
3544
|
+
* The name is then usable in job `agents[].name` like a built-in.
|
|
3545
|
+
*/
|
|
3546
|
+
create(input: AgentInput): Promise<Agent>;
|
|
3547
|
+
/**
|
|
3548
|
+
* List the caller's registered agents (cursor-paged). Await the result for
|
|
3549
|
+
* one page, or `for await` it to walk them all.
|
|
3550
|
+
*/
|
|
3551
|
+
list(options?: ListAgentsOptions): AgentList;
|
|
3552
|
+
/** Get one registered agent by name */
|
|
3553
|
+
get(name: string): Promise<Agent>;
|
|
3554
|
+
/** Delete a registered agent. Past jobs keep their recorded agent. */
|
|
3555
|
+
delete(name: string): Promise<void>;
|
|
3556
|
+
/**
|
|
3557
|
+
* Register or replace an agent in ONE call, under the name you give.
|
|
3558
|
+
*
|
|
3559
|
+
* Use this instead of delete()+create() to change an existing registration:
|
|
3560
|
+
* the pair leaves a window where the agent does not exist, and anything
|
|
3561
|
+
* naming it in that window fails for a change that was only ever meant to be
|
|
3562
|
+
* an edit. This is a full replacement, not a patch — every field comes from
|
|
3563
|
+
* this call, and an omitted `env` becomes empty.
|
|
3564
|
+
*/
|
|
3565
|
+
upsert(name: string, input: AgentUpsertInput): Promise<Agent>;
|
|
3566
|
+
}
|
|
3567
|
+
/**
|
|
3568
|
+
* One skill uploaded to the platform. Immutable content: the digest is the
|
|
3569
|
+
* identity (Harbor's skill digest recipe over the folder), and jobs reference
|
|
3570
|
+
* it as `upload:<id>` in `agents[].skills`.
|
|
3571
|
+
*/
|
|
3572
|
+
interface SkillUpload {
|
|
3573
|
+
id: string;
|
|
3574
|
+
/** Folder name = the name the harness sees when mounted. */
|
|
3575
|
+
name: string;
|
|
3576
|
+
/** Content digest, "sha256:<hex>" — Harbor's recipe. */
|
|
3577
|
+
digest: string;
|
|
3578
|
+
size_bytes: number;
|
|
3579
|
+
/** First heading / description line lifted from SKILL.md, or null. */
|
|
3580
|
+
description: string | null;
|
|
3581
|
+
/** The `upload:<id>` reference to put in `agents[].skills`. */
|
|
3582
|
+
ref: string;
|
|
3583
|
+
created_at: string;
|
|
3584
|
+
}
|
|
3585
|
+
type SkillUploadPage = Page<SkillUpload>;
|
|
3586
|
+
interface SkillUploadList extends Awaitable<SkillUploadPage>, AsyncIterable<SkillUpload> {
|
|
3587
|
+
}
|
|
3588
|
+
interface ListSkillsOptions extends PageOptions {
|
|
3589
|
+
}
|
|
3590
|
+
/** Client for platform-stored skills (uploads referenced as `upload:<id>`). */
|
|
3591
|
+
interface SkillsClient {
|
|
3592
|
+
/**
|
|
3593
|
+
* Upload a local skill folder (must contain SKILL.md, or be a root whose
|
|
3594
|
+
* child directories each contain SKILL.md — Harbor's discovery law; a root
|
|
3595
|
+
* uploads each child as its own skill and resolves to the full list).
|
|
3596
|
+
* Re-uploading identical content under the same name answers the existing
|
|
3597
|
+
* record — uploads are content-addressed, never duplicated. A skill NAME
|
|
3598
|
+
* is a moving pointer: every upload makes its record the name's current
|
|
3599
|
+
* one (different content = new record, pointer moves; old records keep
|
|
3600
|
+
* their immutable `upload:<id>` handles), and `name:<skill-name>` in
|
|
3601
|
+
* `agents[].skills` resolves through it at job create.
|
|
3602
|
+
*/
|
|
3603
|
+
upload(directory: string): Promise<SkillUpload[]>;
|
|
3604
|
+
/** List the caller's uploaded skills (cursor-paged). */
|
|
3605
|
+
list(options?: ListSkillsOptions): SkillUploadList;
|
|
3606
|
+
/**
|
|
3607
|
+
* Get one uploaded skill, including its SKILL.md text. Takes a record id,
|
|
3608
|
+
* or `name:<skill-name>` — the moving name pointer, answered with its
|
|
3609
|
+
* CURRENT record (unknown names are the typed `skill_name_not_found`).
|
|
3610
|
+
*/
|
|
3611
|
+
get(id: string): Promise<SkillUpload & {
|
|
3612
|
+
skill_md: string | null;
|
|
3613
|
+
}>;
|
|
3614
|
+
/**
|
|
3615
|
+
* Delete an uploaded skill. Refused while a non-terminal job references it;
|
|
3616
|
+
* finished jobs keep their recorded locks either way.
|
|
3617
|
+
*/
|
|
3618
|
+
delete(id: string): Promise<void>;
|
|
3619
|
+
}
|
|
3620
|
+
/** Client for hosted jobs */
|
|
3621
|
+
interface JobsClient {
|
|
3622
|
+
/**
|
|
3623
|
+
* Start a job over one or more catalog datasets. Each dataset selector may
|
|
3624
|
+
* carry glob task filters; every agent arm must name a model. Supports
|
|
3625
|
+
* Idempotency-Key.
|
|
3626
|
+
*/
|
|
3627
|
+
start(input: JobCreate, options?: StartJobOptions): Promise<Job>;
|
|
3628
|
+
/** Get one job */
|
|
3629
|
+
get(id: string): Promise<Job>;
|
|
3630
|
+
/**
|
|
3631
|
+
* List the caller's jobs, newest first (cursor-paged). Await the
|
|
3632
|
+
* result for one page, or `for await` it to walk every job across
|
|
3633
|
+
* cursor pages transparently.
|
|
3634
|
+
*/
|
|
3635
|
+
list(options?: ListJobsOptions): JobList;
|
|
3636
|
+
/**
|
|
3637
|
+
* List a job's trials (cursor-paged; { status } filters, e.g. to
|
|
3638
|
+
* the failed trials). Await the result for one page, or `for await` it to
|
|
3639
|
+
* walk every trial across cursor pages transparently.
|
|
3640
|
+
*/
|
|
3641
|
+
trials(id: string, options?: ListTrialsOptions): TrialList;
|
|
3642
|
+
/**
|
|
3643
|
+
* Per-task rollup of a job (cursor-paged): one row per distinct task with
|
|
3644
|
+
* its trial tally, mean reward, and cost. Sits between the job body and
|
|
3645
|
+
* the trial list so a caller need not fetch every trial to see which
|
|
3646
|
+
* tasks are dragging.
|
|
3647
|
+
*/
|
|
3648
|
+
tasks(id: string, options?: ListJobTasksOptions): JobTaskRollupList;
|
|
3649
|
+
/**
|
|
3650
|
+
* Watch a job's event stream (SSE). Replays from the beginning,
|
|
3651
|
+
* resumes with Last-Event-ID on reconnect (exponential backoff), and
|
|
3652
|
+
* finishes on the terminal event.
|
|
3653
|
+
*
|
|
3654
|
+
* The returned handle is dual-use: `await client.watch(id)` resolves with the
|
|
3655
|
+
* final Job, or `for await (const event of client.watch(id))` iterates
|
|
3656
|
+
* the events. The `onEvent` callback still fires in both forms.
|
|
3657
|
+
*/
|
|
3658
|
+
watch(id: string, options?: WatchJobOptions): JobWatch;
|
|
3659
|
+
/** Request cancellation. Idempotent; a terminal job is a no-op. */
|
|
3660
|
+
cancel(id: string): Promise<Job>;
|
|
3661
|
+
/**
|
|
3662
|
+
* Resume a terminal job: a NEW linked job holding fresh trials for the
|
|
3663
|
+
* source's failed and stopped work (`source_jobs` records
|
|
3664
|
+
* `action: "resume"`); the source is never mutated.
|
|
3665
|
+
* `request.filter_error_types` selects which failures to resume by
|
|
3666
|
+
* `exception_info.exception_type`; omitted, the default set includes
|
|
3667
|
+
* stopped trials. Supports Idempotency-Key.
|
|
3668
|
+
*/
|
|
3669
|
+
resume(id: string, request?: ResumeRequest, options?: StartJobOptions): Promise<Job>;
|
|
3670
|
+
/**
|
|
3671
|
+
* MANUAL retry: a NEW linked job holding fresh trials for caller-SELECTED
|
|
3672
|
+
* trials of the source (`source_jobs` records `action: "retry"`); the
|
|
3673
|
+
* source is never mutated. The request selects — `trial_ids` XOR
|
|
3674
|
+
* `failed_only`, omitted = every trial of the (terminal) job. Retry
|
|
3675
|
+
* differs from resume on purpose: resume answers "finish what broke",
|
|
3676
|
+
* retry answers "run THESE again" — a scored trial is a legitimate target.
|
|
3677
|
+
* In trial_ids mode the job may still be running; each named trial must be
|
|
3678
|
+
* settled. Supports Idempotency-Key (fingerprint over the RESOLVED
|
|
3679
|
+
* selection, namespaced to this verb).
|
|
3680
|
+
*/
|
|
3681
|
+
retry(id: string, request?: RetryRequest, options?: StartJobOptions): Promise<Job>;
|
|
3682
|
+
/**
|
|
3683
|
+
* Regrade a terminal job: re-run the verifier of every REGRADABLE trial
|
|
3684
|
+
* against its recorded inputs, in fresh separate verifier boxes. The agent
|
|
3685
|
+
* phase is never re-run and the source trials are never modified. THE
|
|
3686
|
+
* RESPONSE IS A JOB — a regrade is an ordinary job whose `source_jobs`
|
|
3687
|
+
* records `action: "regrade"` and whose `is_regrade` is true; view it with
|
|
3688
|
+
* get(). `request` narrows the set by statuses and/or task.
|
|
3689
|
+
*/
|
|
3690
|
+
regrade(id: string, request?: RegradeRequest): Promise<Job>;
|
|
3691
|
+
/**
|
|
3692
|
+
* Analyze a terminal job's trial traces (rubric-driven, Harbor's `harbor
|
|
3693
|
+
* analyze`), server-side: for each trial the analyzer agent reads the
|
|
3694
|
+
* trial's Harbor-shape tree plus its original task and rules every rubric
|
|
3695
|
+
* criterion, storing the result on the trial (`Trial.analysis`) and the
|
|
3696
|
+
* aggregate on the job (`stats.analysis`). THE RESPONSE IS THE JOB, its
|
|
3697
|
+
* analyses enqueued, and it returns AT ONCE — `stats.analysis.n_pending`
|
|
3698
|
+
* counts the queued batch; analyses are not a separate resource. Follow
|
|
3699
|
+
* them with watchAnalysis(), or poll the job's trials. This is also the
|
|
3700
|
+
* RE-analysis path: calling again (same job, different rubric or model)
|
|
3701
|
+
* runs a fresh wave once the previous one has settled. `request` omitted
|
|
3702
|
+
* (or `{}`) means the defaults: openrouter/deepseek/deepseek-v4.1-flash
|
|
3703
|
+
* at its per-model effort over the platform's default analyze rubric.
|
|
3704
|
+
* CANCELLED trials
|
|
3705
|
+
* are never analyzed.
|
|
3706
|
+
*/
|
|
3707
|
+
analyze(id: string, request?: AnalyzeConfigInput): Promise<Job>;
|
|
3708
|
+
/**
|
|
3709
|
+
* Follow a job's analysis wave to its settled end: polls the job until
|
|
3710
|
+
* `stats.analysis` reports nothing pending, and resolves with the final
|
|
3711
|
+
* Job — the interval doubling from `pollIntervalMs` while the tally
|
|
3712
|
+
* stands still (30-s ceiling, the job watch's own) and resetting on every
|
|
3713
|
+
* change. `onStats` fires on every observed change of the analysis tally
|
|
3714
|
+
* (including the first one seen). Per-trial results then ride the job's
|
|
3715
|
+
* trials (`Trial.analysis`). A null tally is tolerated and watched
|
|
3716
|
+
* through — it is the enqueue race right after an accepted analyze() —
|
|
3717
|
+
* so on a job that was NEVER analyzed this polls indefinitely: call it
|
|
3718
|
+
* after analyze(), as `evolve analyze --watch` does. It is the MANUAL wave's
|
|
3719
|
+
* companion, not the embedded trigger's: on a still-RUNNING job created
|
|
3720
|
+
* with `analyze`, `n_pending` can touch 0 between trial settles, so the
|
|
3721
|
+
* watch can return before every trial has been analyzed.
|
|
3722
|
+
*/
|
|
3723
|
+
watchAnalysis(id: string, options?: WatchAnalysisOptions): Promise<Job>;
|
|
3724
|
+
/**
|
|
3725
|
+
* Side-by-side comparison of 2-10 owned jobs: per-job
|
|
3726
|
+
* aggregates plus a per-task matrix with disagreement rows first.
|
|
3727
|
+
*/
|
|
3728
|
+
compare(ids: string[]): Promise<CompareResponse>;
|
|
3729
|
+
/**
|
|
3730
|
+
* Download a terminal job's results as one .tar.gz in the standard
|
|
3731
|
+
* job-directory layout (deterministic bytes): extracts to `job-<id>/` with
|
|
3732
|
+
* config.json, lock.json, result.json (stats incl. pass_at_k) and job.log,
|
|
3733
|
+
* and per trial its config.json, lock.json, result.json (step_results on
|
|
3734
|
+
* multi-step trials), trial.log, agent/trajectory.json (the normalized
|
|
3735
|
+
* ATIF trajectory), the harness stdout stream at Harbor's tee name for the
|
|
3736
|
+
* harness (agent/claude-code.txt, agent/codex.txt, ...), agent/stderr.log,
|
|
3737
|
+
* agent/trace-parsed.jsonl, the captured agent home at its real names
|
|
3738
|
+
* (agent/.claude/, agent/.claude.json, agent/.codex/, agent/.kimi-code/, …)
|
|
3739
|
+
* with the capture record agent/agent-home.json beside it and Harbor's own
|
|
3740
|
+
* copies of the subtrees its adapters keep (agent/sessions/,
|
|
3741
|
+
* agent/qwen-sessions/, agent/opencode/),
|
|
3742
|
+
* verifier/test-stdout.txt, verifier/reward.json, the raw
|
|
3743
|
+
* verifier/reward.txt (only when the grader wrote one),
|
|
3744
|
+
* steps/<name>/verifier/reward.json (multi-step trials only),
|
|
3745
|
+
* exception.txt, and artifacts/ with its always-present manifest.json —
|
|
3746
|
+
* absent artifacts are absent files. Default: Buffer — verified against
|
|
3747
|
+
* the response's Content-Length and, when the server states one, its
|
|
3748
|
+
* digest. { to } saves to a directory (temp-then-rename, same
|
|
3749
|
+
* verification) and returns the file path. { stream: true } returns the
|
|
3750
|
+
* raw response stream, the one shape the caller must verify themselves.
|
|
3751
|
+
*/
|
|
3752
|
+
download(id: string): Promise<Buffer>;
|
|
3753
|
+
download(id: string, options: {
|
|
3754
|
+
to: string;
|
|
3755
|
+
}): Promise<string>;
|
|
3756
|
+
download(id: string, options: {
|
|
3757
|
+
stream: true;
|
|
3758
|
+
}): Promise<ReadableStream<Uint8Array>>;
|
|
3759
|
+
download(id: string, options?: DownloadJobOptions): Promise<Buffer | string | ReadableStream<Uint8Array>>;
|
|
3760
|
+
/**
|
|
3761
|
+
* Upload a Harbor job directory for ingest as a first-class TERMINAL job
|
|
3762
|
+
* — Harbor's `harbor upload` in reverse, taking their CLI's own input (a
|
|
3763
|
+
* `job_dir` with result.json + config.json at its root, one subdirectory
|
|
3764
|
+
* per trial; the same gate applies here, client-side, with their refusal
|
|
3765
|
+
* sentences). `source` is that directory — packed to a TEMPORARY FILE on
|
|
3766
|
+
* disk with the same deterministic packer every upload route here uses,
|
|
3767
|
+
* never into memory — or a ready-packed `.tar.gz` of one, streamed
|
|
3768
|
+
* byte-for-byte (the platform's own download() produces exactly this
|
|
3769
|
+
* format); or `{ archive_url }`, a public https URL of the archive the
|
|
3770
|
+
* server fetches itself (no bytes ride the request). Archives over
|
|
3771
|
+
* 256 MiB ride the resumable session door automatically (a dropped link
|
|
3772
|
+
* resumes from the last acknowledged chunk), exactly as datasets().publish()
|
|
3773
|
+
* does; the switch is invisible.
|
|
3774
|
+
*
|
|
3775
|
+
* THE RESPONSE IS THE JOB IMPORT, not the job: the door only moves the
|
|
3776
|
+
* archive into storage and answers 202; a worker ingests it off the
|
|
3777
|
+
* request path and settles the import COMPLETED (`job_id` names the
|
|
3778
|
+
* created Job — COMPLETED on creation, a record not a run: resume, retry
|
|
3779
|
+
* and regrade refuse it `job_uploaded`; analyze() works on it unchanged)
|
|
3780
|
+
* or FAILED with a typed `failure` (the same codes the door once answered
|
|
3781
|
+
* synchronously — `not_a_job_dir`, `invalid_trial`, `job_already_uploaded`
|
|
3782
|
+
* naming the existing job, ...). Follow it with watchImport(). `dataset`
|
|
3783
|
+
* links the uploaded trials to a published dataset version by task name.
|
|
3784
|
+
* The one ceiling on an archive is what the object store can land
|
|
3785
|
+
* (`GET /api/meta` `limits.uploads.job_archive_bytes`, published with
|
|
3786
|
+
* its source) — no trial-count, per-file or per-tree cap exists (no
|
|
3787
|
+
* invented number, owner 2026-09-13).
|
|
3788
|
+
*/
|
|
3789
|
+
upload(source: string | {
|
|
3790
|
+
archive_url: string;
|
|
3791
|
+
}, options?: UploadJobOptions): Promise<JobImport>;
|
|
3792
|
+
/** One job import by id — owner-only (`job_import_not_found`, 404, for anyone else's). */
|
|
3793
|
+
getImport(id: string): Promise<JobImport>;
|
|
3794
|
+
/**
|
|
3795
|
+
* Watch a job import to its terminal state — COMPLETED (read the Job at
|
|
3796
|
+
* `job_id`) or FAILED (`failure` says why). Polls getImport() at
|
|
3797
|
+
* `pollIntervalMs` (default 2 s); a 429/503 mid-watch is a delay, not an
|
|
3798
|
+
* outcome. `onStatus` fires on every observed status change (the
|
|
3799
|
+
* receiving flip included); `onProgress` on every observed change of the
|
|
3800
|
+
* worker's phase record.
|
|
3801
|
+
*/
|
|
3802
|
+
watchImport(id: string, options?: WatchJobImportOptions): Promise<JobImport>;
|
|
3803
|
+
/**
|
|
3804
|
+
* List your own job imports, newest first (cursor-paged): await one page,
|
|
3805
|
+
* or for-await to walk them all. This is how an import id is found again
|
|
3806
|
+
* after the one upload() returned was lost.
|
|
3807
|
+
*/
|
|
3808
|
+
listImports(options?: ListJobImportsOptions): JobImportList;
|
|
3809
|
+
/**
|
|
3810
|
+
* Permanently delete one of your jobs — trials, trace events, analyses and
|
|
3811
|
+
* every stored trace object included (Harbor's `harbor hub job delete`:
|
|
3812
|
+
* "Permanently delete Hub jobs you own, including their trials"). Works on
|
|
3813
|
+
* uploaded and native jobs alike; deleting an uploaded job frees its
|
|
3814
|
+
* duplicate lock, so delete-then-reupload is the replace path.
|
|
3815
|
+
*
|
|
3816
|
+
* CREATOR-ONLY: org members may operate a job (cancel, retry), never
|
|
3817
|
+
* destroy its record — a member who did not create it is refused
|
|
3818
|
+
* (`org_forbidden`, 403). TERMINAL ONLY — never a delete under a live
|
|
3819
|
+
* worker: a QUEUED/RUNNING/CANCELLING job refuses `job_not_terminal`
|
|
3820
|
+
* (409; cancel first), a queued or running analysis wave refuses
|
|
3821
|
+
* `analysis_already_running` (409), and a live regrade derived from this
|
|
3822
|
+
* job refuses `job_not_terminal` with the regrade jobs to wait for in
|
|
3823
|
+
* `details.regrade_job_ids`. A regrade job id is itself not deletable
|
|
3824
|
+
* here (`job_not_found`, 404) — a regrade's results are deleted from the
|
|
3825
|
+
* traces surface. What stays: regrade JOB rows and `source_jobs` history,
|
|
3826
|
+
* which keep naming the deleted id; the model gateway's own ledger
|
|
3827
|
+
* remains the billing truth.
|
|
3828
|
+
*
|
|
3829
|
+
* The response is the receipt: what was destroyed, counted.
|
|
3830
|
+
*/
|
|
3831
|
+
delete(id: string): Promise<JobDeleteResult>;
|
|
3832
|
+
/**
|
|
3833
|
+
* Grep the parsed trace of EVERY trial of the job in one server-side pass.
|
|
3834
|
+
* `q` is the trace filter's grammar: a case-insensitive POSIX regex over
|
|
3835
|
+
* each event's type and serialized content, where a plain string is a
|
|
3836
|
+
* plain substring. Items are per-trial groups (exact count + the first few
|
|
3837
|
+
* matching events), ordered by trial id; page with { cursor }. An empty
|
|
3838
|
+
* page means no matches anywhere — a normal answer.
|
|
3839
|
+
*/
|
|
3840
|
+
grep(id: string, q: string, options?: GrepJobOptions): Promise<JobGrepPage>;
|
|
3841
|
+
}
|
|
3842
|
+
/**
|
|
3843
|
+
* The trace route's `?stream=` selectors, in the contract's own order —
|
|
3844
|
+
* `trace-parsed` (the parsed event trace, the same answer as omitting
|
|
3845
|
+
* `stream`) followed by the raw-artifact vocabulary. `trace-atif` is the
|
|
3846
|
+
* SERVED normalized trajectory (Harbor's ATIF v1.7); `trajectory` is a
|
|
3847
|
+
* DIFFERENT artifact — the harness's own native session file, in the
|
|
3848
|
+
* vocabulary ahead of its server wave (the server answers not-found for it
|
|
3849
|
+
* until that wave lands). A runtime value (not only a type) so a drift gate
|
|
3850
|
+
* can hold it to the spec's enum, and the CLI can build its `--stream`
|
|
3851
|
+
* validation from the same list instead of a second copy.
|
|
3852
|
+
*/
|
|
3853
|
+
declare const TRIAL_ARTIFACT_STREAMS: readonly ["trace-parsed", "verifier", "trace-stdout", "trace-stderr", "trace-atif", "trajectory", "agent-home"];
|
|
3854
|
+
/** One `?stream=` selector on the trace route. */
|
|
3855
|
+
type TrialArtifactStream = (typeof TRIAL_ARTIFACT_STREAMS)[number];
|
|
3856
|
+
/** Client for globally addressable trials — no job id in any signature */
|
|
3857
|
+
interface TrialsClient {
|
|
3858
|
+
/**
|
|
3859
|
+
* Get one trial by its globally addressable id. The body carries `job_id`
|
|
3860
|
+
* as the reverse pointer; `exception_info.exception_message` is untruncated
|
|
3861
|
+
* here, unlike list rows.
|
|
3862
|
+
*/
|
|
3863
|
+
get(trialId: string): Promise<Trial>;
|
|
3864
|
+
/** Get one page of a trial's trace; resume with { cursor: page.nextCursor } */
|
|
3865
|
+
trace(trialId: string, options?: TraceOptions): Promise<TraceEventPage>;
|
|
3866
|
+
/**
|
|
3867
|
+
* Iterate a trial's trace events, fetching pages under the hood until
|
|
3868
|
+
* the currently available trace is drained. Resume later by passing the
|
|
3869
|
+
* last seen seq as { cursor }.
|
|
3870
|
+
*/
|
|
3871
|
+
traceEvents(trialId: string, options?: TraceOptions): AsyncIterableIterator<TraceEvent>;
|
|
3872
|
+
/**
|
|
3873
|
+
* One RAW trace artifact, by the trace route's ?stream= selector.
|
|
3874
|
+
* "verifier" | "trace-stdout" | "trace-stderr" answer the log text;
|
|
3875
|
+
* "trace-atif" answers the normalized trajectory — Harbor's ATIF v1.7
|
|
3876
|
+
* document as JSON text, built server-side from the stored parsed trace
|
|
3877
|
+
* (the same document jobs.download() places at Harbor's own path
|
|
3878
|
+
* agent/trajectory.json); "trajectory" is the reserved harness-native
|
|
3879
|
+
* session file, refused not-found by the server until its wave lands;
|
|
3880
|
+
* "agent-home" answers the utf8 TEXT VIEW of the CLI's captured home
|
|
3881
|
+
* (subagent transcripts included), keyed by sandbox path: a file that is
|
|
3882
|
+
* not UTF-8 text is left out and named in the capture record that rides
|
|
3883
|
+
* the same map as "/agent-home.json"; a home over the server's whole-read
|
|
3884
|
+
* ceiling is refused 413 invalid_input (param "format") — this SDK has no
|
|
3885
|
+
* bytes door for the home, the job archive (jobs().download) carries it
|
|
3886
|
+
* whole. Null = never stored
|
|
3887
|
+
* (a normal answer, not an error). "trace-parsed" is not an
|
|
3888
|
+
* artifact — the parsed event trace rides trace()/traceEvents().
|
|
3889
|
+
*/
|
|
3890
|
+
artifact(trialId: string, stream: Exclude<TrialArtifactStream, "trace-parsed" | "agent-home">): Promise<string | null>;
|
|
3891
|
+
artifact(trialId: string, stream: "agent-home"): Promise<Record<string, string> | null>;
|
|
3892
|
+
/**
|
|
3893
|
+
* Regrade one settled trial: re-run its verifier against its recorded
|
|
3894
|
+
* inputs in a fresh separate verifier box. Refused
|
|
3895
|
+
* (regrade_source_ineligible) for shared-mode or pre-persistence trials.
|
|
3896
|
+
* THE RESPONSE IS A JOB — a one-trial regrade job with `source_jobs`
|
|
3897
|
+
* recording the provenance.
|
|
3898
|
+
*/
|
|
3899
|
+
regrade(trialId: string): Promise<Job>;
|
|
3900
|
+
/**
|
|
3901
|
+
* Run ONE settled trial again. THE RESPONSE IS A JOB — a one-trial retry
|
|
3902
|
+
* job inheriting the source job's config, with `source_jobs` recording
|
|
3903
|
+
* `action: "retry"`; the source trial is immutable. The same operation as
|
|
3904
|
+
* jobs.retry(jobId, {trial_ids: [trialId]}) — one selection rule, one
|
|
3905
|
+
* fingerprint — kept as its own door because the trial is what you are
|
|
3906
|
+
* holding. The source JOB may still be running; the trial must be settled
|
|
3907
|
+
* (`trial_not_settled` otherwise). Supports Idempotency-Key.
|
|
3908
|
+
*/
|
|
3909
|
+
retry(trialId: string, options?: StartJobOptions): Promise<Job>;
|
|
3910
|
+
/**
|
|
3911
|
+
* Stop selected in-flight trials without cancelling their job: each trial's
|
|
3912
|
+
* sandbox is killed and the trial is settled with its spend read from the
|
|
3913
|
+
* gateway. Ids may be eval trials and trace analyses, freely mixed — what
|
|
3914
|
+
* each id is gets resolved server-side; a stopped analysis settles `failed`
|
|
3915
|
+
* (failure phase `stopped`) and is reported under `stopped_analyses`. Only
|
|
3916
|
+
* the caller's own work; ids belonging to someone else are reported in
|
|
3917
|
+
* `not_found` (existence is never leaked). Idempotent — already-terminal
|
|
3918
|
+
* ids are reported as such and left untouched.
|
|
3919
|
+
*/
|
|
3920
|
+
stop(trialIds: string[]): Promise<StopResponse>;
|
|
3921
|
+
/**
|
|
3922
|
+
* List the trial's ENTIRE stored file tree — the read-only-filesystem law:
|
|
3923
|
+
* session files, verifier log, raw agent streams, live chunks, everything
|
|
3924
|
+
* the platform stored, as {path, size_bytes} rows sorted by path. Read any
|
|
3925
|
+
* row with file(). An empty page is a normal answer.
|
|
3926
|
+
*/
|
|
3927
|
+
files(trialId: string, options?: ListTrialFilesOptions): Promise<TrialFilePage>;
|
|
3928
|
+
/**
|
|
3929
|
+
* RAW BYTES of one stored file, by the path files() names — byte fidelity,
|
|
3930
|
+
* no translation. `range` reads a slice ({ start, end } inclusive,
|
|
3931
|
+
* { start } to the end, or { suffix } for the last N bytes) so a huge log
|
|
3932
|
+
* tails without shipping whole. A path the tree does not hold surfaces as
|
|
3933
|
+
* the API's typed 404.
|
|
3934
|
+
*/
|
|
3935
|
+
file(trialId: string, path: string, range?: TrialFileRange): Promise<Buffer>;
|
|
3936
|
+
}
|
|
3937
|
+
/**
|
|
3938
|
+
* The stored artifacts an analysis run OWNS under its own id — the analyzer's
|
|
3939
|
+
* raw process streams and its session home (the executor stores exactly these
|
|
3940
|
+
* three at settle). A runtime value like TRIAL_ARTIFACT_STREAMS, for the same
|
|
3941
|
+
* reason: the CLI builds its `--stream` validation from this list instead of
|
|
3942
|
+
* a second copy. `verifier` and `trace-atif` are deliberately NOT members: an
|
|
3943
|
+
* analysis never has them, and the server refuses them typed rather than
|
|
3944
|
+
* answering a null that would read as "not stored".
|
|
3945
|
+
*/
|
|
3946
|
+
declare const ANALYSIS_ARTIFACT_STREAMS: readonly ["trace-stdout", "trace-stderr", "agent-home"];
|
|
3947
|
+
/** One stored-artifact selector of an analysis run. */
|
|
3948
|
+
type AnalysisArtifactStream = (typeof ANALYSIS_ARTIFACT_STREAMS)[number];
|
|
3949
|
+
/** Options for analyses().transcript(). */
|
|
3950
|
+
interface AnalysisTranscriptOptions {
|
|
3951
|
+
/**
|
|
3952
|
+
* Skip the first N events — the feed's own resume grammar ("everything
|
|
3953
|
+
* after the N events I already hold"). An analysis's seqs are allocated
|
|
3954
|
+
* densely from 0, so N is also the seq the returned events start at.
|
|
3955
|
+
* A non-negative integer; anything else is refused client-side.
|
|
3956
|
+
*/
|
|
3957
|
+
since?: number;
|
|
3958
|
+
}
|
|
3959
|
+
/**
|
|
3960
|
+
* One analysis run's transcript: the ANALYZER's own parsed events — never the
|
|
3961
|
+
* analyzed trial's agent trace — plus the identity facts the feed serves
|
|
3962
|
+
* around them. Unlike a trial's trace there is no server-side paging: one
|
|
3963
|
+
* read answers everything after `since`, and `total` counts ALL stored rows,
|
|
3964
|
+
* so `events.length < total` with `since: 0` can only mean rows arrived
|
|
3965
|
+
* between count and read (a live analysis).
|
|
3966
|
+
*/
|
|
3967
|
+
interface AnalysisTranscript {
|
|
3968
|
+
id: string;
|
|
3969
|
+
/** The trial this analysis read — the walk back to the analyzed work. */
|
|
3970
|
+
analyzed_trial_id: string | null;
|
|
3971
|
+
/** The analyzed trial's job. */
|
|
3972
|
+
job_id: string | null;
|
|
3973
|
+
/** The analyzed task's key. */
|
|
3974
|
+
task_name: string | null;
|
|
3975
|
+
/** The model the analyzer ran. */
|
|
3976
|
+
model_name: string | null;
|
|
3977
|
+
/** Where the ANALYZER's own box ran — never the analyzed arm's. */
|
|
3978
|
+
sandbox_provider: string | null;
|
|
3979
|
+
sandbox_id: string | null;
|
|
3980
|
+
/** True once the analysis has settled (completed or failed). */
|
|
3981
|
+
is_ended: boolean;
|
|
3982
|
+
/** ALL stored rows for this analysis, independent of `since`. */
|
|
3983
|
+
total: number;
|
|
3984
|
+
/**
|
|
3985
|
+
* The events after `since`, in TraceEvent shape. The feed serves the bare
|
|
3986
|
+
* parsed payloads; `seq` is synthesized as since+index (sound because seqs
|
|
3987
|
+
* are dense from 0) and `type` by the viewer's own one extraction.
|
|
3988
|
+
*/
|
|
3989
|
+
events: TraceEvent[];
|
|
3990
|
+
/**
|
|
3991
|
+
* The gateway meter's per-call lines for the analyzer's key (spec
|
|
3992
|
+
* GatewayUsageEvent), in time order, as `usage` TraceEvents with `seq` in
|
|
3993
|
+
* the gateway band (GATEWAY_TRACE_SEQ_BASE + index). Served whole on every
|
|
3994
|
+
* read: they ride beside `events`, never inside the seq timeline.
|
|
3995
|
+
*/
|
|
3996
|
+
gateway_calls: TraceEvent[];
|
|
3997
|
+
/**
|
|
3998
|
+
* The server's write instant of each event's row, one per entry of
|
|
3999
|
+
* `events`, index-aligned (the contract's SessionTranscript.storedAt):
|
|
4000
|
+
* present on every row-served page (an empty page carries an empty list),
|
|
4001
|
+
* absent when the transcript was served from its file, where no write
|
|
4002
|
+
* instant exists. It places the gateway meter's calls under the harness's
|
|
4003
|
+
* steps for harnesses whose lines carry no clock of their own (codex, kimi,
|
|
4004
|
+
* qwen); a reader that does not place calls needs nothing from it.
|
|
4005
|
+
*/
|
|
4006
|
+
stored_at?: string[];
|
|
4007
|
+
}
|
|
4008
|
+
/**
|
|
4009
|
+
* Client for analysis runs — the analyzer's own transcript, verdict document,
|
|
4010
|
+
* and stored artifacts, all globally addressable by analysis id.
|
|
4011
|
+
*
|
|
4012
|
+
* DELIBERATELY OFF-CONTRACT: these three reads ride the dashboard's traces
|
|
4013
|
+
* feed (`/api/traces/trials/{id}/events` and `…/artifacts`), which is NOT in
|
|
4014
|
+
* spec/openapi.yaml — `traces` is outside the prefixes the platform's drift
|
|
4015
|
+
* gate walks (swarm_dashboard `__tests__/api/spec-drift-gate.test.ts`
|
|
4016
|
+
* CONTRACT_PREFIXES), and the one precedent for a transcript door living
|
|
4017
|
+
* off-contract is that gate's RUNTIME_INTERNAL_ROUTES: `api/sessions/[id]/
|
|
4018
|
+
* events`, the session transcript feed, is enumerated there by recorded
|
|
4019
|
+
* exemption — both SDKs' sessions client reads it (`sessions().transcript()`
|
|
4020
|
+
* / `events()`), and its `gatewayCalls` shape is named in the contract's
|
|
4021
|
+
* GatewayUsageEvent prose, not as an operation. RECORDED TENSION: whether
|
|
4022
|
+
* that feed and this one join the contract as operations (spec + both SDK
|
|
4023
|
+
* shadows) is an open ruling, not something settled here. The contract-side
|
|
4024
|
+
* verdict stays where it always was — `Trial.analysis` on the trial body;
|
|
4025
|
+
* this client adds the reads the contract does not carry today.
|
|
4026
|
+
*/
|
|
4027
|
+
interface AnalysesClient {
|
|
4028
|
+
/**
|
|
4029
|
+
* Every analysis you may read, newest first (cursor-paged) — the catalog
|
|
4030
|
+
* of trace-analysis runs, `GET /api/analyses`. Each row is the wire's
|
|
4031
|
+
* TrialAnalysis carrying the trial, job, and task it judged, so a
|
|
4032
|
+
* headless round is list → get each. `{ scope, job, status }` narrow it;
|
|
4033
|
+
* await the handle for one page, or `for await` it to walk every page.
|
|
4034
|
+
* Every row listed under either scope resolves on every read: the per-run
|
|
4035
|
+
* doors open to the job's creator and to every member of its organization,
|
|
4036
|
+
* the law jobs().get() and trials().get() follow. An id you may not read
|
|
4037
|
+
* answers exactly as one that does not exist — `analysis_not_found` (404)
|
|
4038
|
+
* from `get`, `artifact` and `download`; `transcript` alone rides the
|
|
4039
|
+
* feed's species-blind events door, which answers `trial_not_found`.
|
|
4040
|
+
*/
|
|
4041
|
+
list(options?: ListAnalysesOptions): AnalysisList;
|
|
4042
|
+
/**
|
|
4043
|
+
* The verdict document — the wire's TrialAnalysis, statuses and typed
|
|
4044
|
+
* failure included, for EVERY analysis (not only completed ones). The same
|
|
4045
|
+
* object the analyzed trial serves as `Trial.analysis` when this analysis
|
|
4046
|
+
* is its latest; this door answers for earlier analyses too.
|
|
4047
|
+
*/
|
|
4048
|
+
get(analysisId: string): Promise<TrialAnalysis>;
|
|
4049
|
+
/**
|
|
4050
|
+
* The analyzer's own transcript (see AnalysisTranscript). An id the feed
|
|
4051
|
+
* resolves to a trial or a regrade — the route answers those first —
|
|
4052
|
+
* refuses with an error naming the species rather than handing back the
|
|
4053
|
+
* wrong run's events.
|
|
4054
|
+
*/
|
|
4055
|
+
transcript(analysisId: string, options?: AnalysisTranscriptOptions): Promise<AnalysisTranscript>;
|
|
4056
|
+
/**
|
|
4057
|
+
* One stored artifact by selector: the analyzer's raw stdout/stderr answer
|
|
4058
|
+
* the text, "agent-home" the sandbox-path → text map of its session home.
|
|
4059
|
+
* Null = never stored (a normal answer, not an error): a QUEUED analysis,
|
|
4060
|
+
* or one whose box died before the settle stored anything. Like
|
|
4061
|
+
* transcript(), an id the feed resolves to a trial or a regrade refuses
|
|
4062
|
+
* with an error — the feed's stored selectors would answer for either
|
|
4063
|
+
* species, so the analysis-only ?what=analysis door is resolved first and
|
|
4064
|
+
* the wrong run's bytes are never served.
|
|
4065
|
+
*/
|
|
4066
|
+
artifact(analysisId: string, stream: Exclude<AnalysisArtifactStream, "agent-home">): Promise<string | null>;
|
|
4067
|
+
artifact(analysisId: string, stream: "agent-home"): Promise<Record<string, string> | null>;
|
|
4068
|
+
/**
|
|
4069
|
+
* Download the analysis run as Harbor's WRAPPER-trial folder in one
|
|
4070
|
+
* `.tar.gz` (`GET /api/analyses/{analysisId}/download` — on the
|
|
4071
|
+
* contract, unlike the three feed reads above). The archive extracts to
|
|
4072
|
+
* one directory named as Harbor names the wrapper trial
|
|
4073
|
+
* (`analyze-<analyzed trial dir>__<7 chars>/`): config.json, lock.json,
|
|
4074
|
+
* result.json, trial.log, exception.txt (an infrastructure failure only),
|
|
4075
|
+
* agent/claude-code.txt (Harbor's tee name for claude-code),
|
|
4076
|
+
* agent/stderr.log, agent/trace-parsed.jsonl, the captured home at its
|
|
4077
|
+
* real names with agent/agent-home.json beside it and Harbor's copy at
|
|
4078
|
+
* agent/sessions/, verifier/{test-stdout.txt,reward.txt,reward.json}
|
|
4079
|
+
* when the validator ruled (reward 1 = a valid analysis.json, 0 = it was
|
|
4080
|
+
* refused), and artifacts/manifest.json with artifacts/analysis.json (the
|
|
4081
|
+
* validated {summary, checks}) on a completed run — absent artifacts are
|
|
4082
|
+
* absent files; data the platform does not hold is left out, never
|
|
4083
|
+
* faked. Same three delivery shapes and the same integrity checks as
|
|
4084
|
+
* jobs().download(). 404 `analysis_not_found` for an id you cannot read;
|
|
4085
|
+
* 409 `analysis_not_terminal` while the run is queued or running.
|
|
4086
|
+
*/
|
|
4087
|
+
download(analysisId: string): Promise<Buffer>;
|
|
4088
|
+
download(analysisId: string, options: {
|
|
4089
|
+
to: string;
|
|
4090
|
+
}): Promise<string>;
|
|
4091
|
+
download(analysisId: string, options: {
|
|
4092
|
+
stream: true;
|
|
4093
|
+
}): Promise<ReadableStream<Uint8Array>>;
|
|
4094
|
+
download(analysisId: string, options?: DownloadJobOptions): Promise<Buffer | string | ReadableStream<Uint8Array>>;
|
|
4095
|
+
}
|
|
4096
|
+
/**
|
|
4097
|
+
* Task-check configuration — Harbor's `harbor check` vocabulary (their
|
|
4098
|
+
* cli/analyze.py:84-148 check_command), the spec's `CheckConfigInput`. The
|
|
4099
|
+
* rubric-agent trio is the analyze door's, under the same rules
|
|
4100
|
+
* (`AnalyzeConfigInput` states them; refusals name `check.*`): `model_name`
|
|
4101
|
+
* (Harbor's check default is `claude-sonnet-4-6`; this platform's is the
|
|
4102
|
+
* analyzer's `openrouter/deepseek/deepseek-v4.1-flash` — one roster, one
|
|
4103
|
+
* default for both rubric
|
|
4104
|
+
* agents, a recorded deviation), `rubric` (the default is the platform's
|
|
4105
|
+
* check rubric, eleven criteria), and
|
|
4106
|
+
* `prompt` (the TEXT of Harbor's `-p/--prompt` file, replacing the platform's
|
|
4107
|
+
* default check body and rendered with `{task_path}`, `{file_tree}`,
|
|
4108
|
+
* `{criteria_guidance}`; the output contract is appended after it exactly
|
|
4109
|
+
* as Harbor appends it). `reasoning_effort` and `sandbox_provider` are the
|
|
4110
|
+
* platform's two hosted knobs, exactly as on the analyze door.
|
|
4111
|
+
*
|
|
4112
|
+
* Which tasks, and how wide, are Harbor's own check options with their
|
|
4113
|
+
* exact names: `n_concurrent` (`-n/--n-concurrent`), `include_task_names`
|
|
4114
|
+
* (`-i/--include-task-name`, repeatable glob), `exclude_task_names`
|
|
4115
|
+
* (`-x/--exclude-task-name`), `n_tasks` (`-l/--n-tasks`) — applied in
|
|
4116
|
+
* checker.py's order (:132-138: include, exclude, then the cap) over the
|
|
4117
|
+
* sorted task directory names, Python fnmatch globs against the NAME.
|
|
4118
|
+
* Harbor's `-a/--agent`, `--job-name`, `-o/--jobs-dir`, `-k/--n-attempts`
|
|
4119
|
+
* and the local-runner kwargs are not on this surface; the contract records
|
|
4120
|
+
* each with its reason.
|
|
4121
|
+
*/
|
|
4122
|
+
interface CheckConfigInput {
|
|
4123
|
+
/** Model the checker agent runs (Harbor's `-m/--model`); must be on the claude roster (`GET /api/meta`). */
|
|
4124
|
+
model_name?: string;
|
|
4125
|
+
/** The rubric (Harbor's `-r/--rubric` file as its `{criteria}` object); default: the platform's check rubric (eleven criteria). */
|
|
4126
|
+
rubric?: Rubric;
|
|
4127
|
+
/** The prompt template — the TEXT of Harbor's `-p/--prompt` file. */
|
|
4128
|
+
prompt?: string;
|
|
4129
|
+
/** Reasoning effort the checker runs at (the arms' vocabulary; a hosted extension). */
|
|
4130
|
+
reasoning_effort?: string;
|
|
4131
|
+
/** The provider whose sandbox the checker boots (a hosted extension; the job lineup). */
|
|
4132
|
+
sandbox_provider?: EvalSandboxProvider;
|
|
4133
|
+
/** How many of this check's tasks run at once — Harbor's `-n/--n-concurrent`; beneath the organization's `max_concurrent_analyses`. */
|
|
4134
|
+
n_concurrent?: number;
|
|
4135
|
+
/** Only check task directories whose name matches one of these globs — Harbor's `-i`, repeatable. */
|
|
4136
|
+
include_task_names?: string[];
|
|
4137
|
+
/** Skip task directories whose name matches one of these globs — Harbor's `-x`, repeatable. */
|
|
4138
|
+
exclude_task_names?: string[];
|
|
4139
|
+
/** At most this many task directories, after the globs — Harbor's `-l/--n-tasks`. */
|
|
4140
|
+
n_tasks?: number;
|
|
4141
|
+
}
|
|
4142
|
+
/**
|
|
4143
|
+
* What checks().create() takes: WHERE the tasks are, plus the
|
|
4144
|
+
* CheckConfigInput knobs. Two sources, one at a time:
|
|
4145
|
+
*
|
|
4146
|
+
* { directory } — a local directory (one task directory, or a directory of
|
|
4147
|
+
* task directories: Harbor's `PATH`, tarred and streamed
|
|
4148
|
+
* from disk; a directory only, as Harbor's check is — their
|
|
4149
|
+
* checker.py:125-130 refuses any PATH that is not one; no
|
|
4150
|
+
* ready-packed archive form).
|
|
4151
|
+
* { dataset } — a PUBLISHED dataset, `name` (its active version) or
|
|
4152
|
+
* `name@version` — the hosted form, a recorded deviation
|
|
4153
|
+
* (Harbor's check takes a local path only): the checker
|
|
4154
|
+
* reads the version's retained task package, and the job
|
|
4155
|
+
* page's CHECK tab lists these checks by task. Nothing is
|
|
4156
|
+
* uploaded.
|
|
4157
|
+
*/
|
|
4158
|
+
interface CreateCheckInput extends CheckConfigInput {
|
|
4159
|
+
source: {
|
|
4160
|
+
directory: string;
|
|
4161
|
+
} | {
|
|
4162
|
+
dataset: string;
|
|
4163
|
+
};
|
|
4164
|
+
/** Client-side upload progress (sent bytes, total bytes), from the stream itself. Directory source only. */
|
|
4165
|
+
onUploadProgress?: (sentBytes: number, totalBytes: number) => void;
|
|
4166
|
+
}
|
|
4167
|
+
/**
|
|
4168
|
+
* What was checked, by identity (spec CheckSource) — two forms told apart by
|
|
4169
|
+
* `type`: `archive` (the uploaded tar's sha256 and compressed `bytes`;
|
|
4170
|
+
* `dataset` null) or `dataset` (the resolved `name@version` and the sha256
|
|
4171
|
+
* of the version's retained task package; `bytes` null — a package's size is
|
|
4172
|
+
* not a recorded fact, so the wire invents none).
|
|
4173
|
+
*/
|
|
4174
|
+
interface CheckSource {
|
|
4175
|
+
type: "archive" | "dataset";
|
|
4176
|
+
/** SHA-256 of the archive as uploaded, or of the version's retained task package. */
|
|
4177
|
+
sha256: string;
|
|
4178
|
+
/** The archive's compressed size on the archive form; null on the dataset form. */
|
|
4179
|
+
bytes: number | null;
|
|
4180
|
+
/** The resolved `name@version` on the dataset form; null on the archive form. */
|
|
4181
|
+
dataset: string | null;
|
|
4182
|
+
}
|
|
4183
|
+
/**
|
|
4184
|
+
* One task's quality check — Harbor's QualityCheckResult shape (their
|
|
4185
|
+
* cli/quality_checker/models.py:31-35: `task_name`, `checks` keyed by
|
|
4186
|
+
* criterion, `cost_usd`), its checks extended by the result schema and the
|
|
4187
|
+
* derived `label` and `executed` beside them, plus the hosted provenance: its own id, the check
|
|
4188
|
+
* it belongs to, its lifecycle (the analysis ladder's four lowercase words),
|
|
4189
|
+
* the bounded attempt count, and a typed `failure` in place of Harbor's
|
|
4190
|
+
* `error` string (the TrialAnalysis rule).
|
|
4191
|
+
*
|
|
4192
|
+
* `checks` is the FLAT object the platform's validate.py accepts (one key per
|
|
4193
|
+
* rubric criterion, each `{outcome, explanation, evidence}`, four outcome
|
|
4194
|
+
* words — no summary; analyze has
|
|
4195
|
+
* one, check does not). `cost_usd` is the checker agent's OWN metered spend;
|
|
4196
|
+
* null when nothing was measured, never a fabricated 0.
|
|
4197
|
+
*/
|
|
4198
|
+
interface TaskCheck {
|
|
4199
|
+
id: string;
|
|
4200
|
+
check_id: string;
|
|
4201
|
+
/** The task directory's name (Harbor's task_name). */
|
|
4202
|
+
task_name: string;
|
|
4203
|
+
/** `queued` | `running` | `completed` | `failed` — the analysis ladder, on a task check. */
|
|
4204
|
+
status: AnalysisStatus;
|
|
4205
|
+
/** One entry per rubric criterion, keys exactly the frozen criteria. Null until completed. */
|
|
4206
|
+
checks: Record<string, AnalysisCheck> | null;
|
|
4207
|
+
/** The derived label (CheckLabel states the rule); null until completed, and null under a custom rubric. */
|
|
4208
|
+
label: CheckLabel | null;
|
|
4209
|
+
/**
|
|
4210
|
+
* Whether the box ran the task's environment: true when none of the five
|
|
4211
|
+
* run-based criteria (reference_solution_is_valid,
|
|
4212
|
+
* verifier_rejects_non_solutions, environment_builds_and_runs,
|
|
4213
|
+
* verification_is_stable, limits_allow_the_task) is unknown, so a
|
|
4214
|
+
* reading-only `no_problem_found` is never mistaken for a run. Null
|
|
4215
|
+
* exactly when `label` is null.
|
|
4216
|
+
*/
|
|
4217
|
+
executed: boolean | null;
|
|
4218
|
+
cost_usd: number | null;
|
|
4219
|
+
/** 1, or 2 when the one automatic re-run fired (a run that produced no valid check-result.json, the missing file included, is re-run once — the analyze verb's hosted rule; a run cut by its budget is not that class: it settles `failed` with phase `timeout` at once and is never re-run). */
|
|
4220
|
+
attempts: number;
|
|
4221
|
+
/** Non-null exactly when status is `failed`. */
|
|
4222
|
+
failure: AnalysisFailure | null;
|
|
4223
|
+
created_at: string;
|
|
4224
|
+
/** When it settled; null while queued or running. */
|
|
4225
|
+
finished_at: string | null;
|
|
4226
|
+
}
|
|
4227
|
+
/**
|
|
4228
|
+
* One task quality check — Harbor's CheckReport (`results`, and `cost_usd`
|
|
4229
|
+
* its `total_cost_usd`: the sum of measured task costs, null when none was
|
|
4230
|
+
* measured) plus the hosted record: the run's own id and lifecycle
|
|
4231
|
+
* (CheckStatus), its source, and the policy every task ran under, frozen at
|
|
4232
|
+
* accept.
|
|
4233
|
+
*/
|
|
4234
|
+
interface Check {
|
|
4235
|
+
id: string;
|
|
4236
|
+
status: CheckStatus;
|
|
4237
|
+
source: CheckSource;
|
|
4238
|
+
model_name: string;
|
|
4239
|
+
/** The effort every task's checker ran at — named at create, or the model's default, resolved at accept. */
|
|
4240
|
+
reasoning_effort: string;
|
|
4241
|
+
rubric: Rubric;
|
|
4242
|
+
/** The prompt template as stored; null = the platform's default check body. */
|
|
4243
|
+
prompt: string | null;
|
|
4244
|
+
sandbox_provider: EvalSandboxProvider;
|
|
4245
|
+
/** Harbor's -n as stored; null = the organization's ceiling alone. */
|
|
4246
|
+
n_concurrent: number | null;
|
|
4247
|
+
/** The include globs as stored; empty = no include filter. */
|
|
4248
|
+
include_task_names: string[];
|
|
4249
|
+
/** The exclude globs as stored; empty = no exclude filter. */
|
|
4250
|
+
exclude_task_names: string[];
|
|
4251
|
+
/** Harbor's -l/--n-tasks as stored; null = no cap. */
|
|
4252
|
+
n_tasks: number | null;
|
|
4253
|
+
/** One entry per task directory checked, sorted by task name. */
|
|
4254
|
+
results: TaskCheck[];
|
|
4255
|
+
cost_usd: number | null;
|
|
4256
|
+
created_at: string;
|
|
4257
|
+
/** When the last task settled; null until every task has. */
|
|
4258
|
+
finished_at: string | null;
|
|
4259
|
+
}
|
|
4260
|
+
/** Options for checks().watch() */
|
|
4261
|
+
interface WatchCheckOptions {
|
|
4262
|
+
/** Called on every observed change of the check's per-task statuses, with the check body the observation came from. */
|
|
4263
|
+
onProgress?: (check: Check) => void;
|
|
4264
|
+
/** Abort the watch (rejects with the abort reason) */
|
|
4265
|
+
signal?: AbortSignal;
|
|
4266
|
+
/** Initial poll interval (default: 2000ms); doubles while nothing changes, up to 30 s, and snaps back on every change. */
|
|
4267
|
+
pollIntervalMs?: number;
|
|
4268
|
+
}
|
|
4269
|
+
/**
|
|
4270
|
+
* One task check's transcript: the CHECKER's own parsed events plus the
|
|
4271
|
+
* identity facts the feed serves around them — the AnalysisTranscript's
|
|
4272
|
+
* shape (its docs state the read grammar: no server-side paging, `total`
|
|
4273
|
+
* counts ALL stored rows, `since` resumes), with the check record and the
|
|
4274
|
+
* dataset ref in place of the analyzed trial and its job.
|
|
4275
|
+
*/
|
|
4276
|
+
interface TaskCheckTranscript {
|
|
4277
|
+
id: string;
|
|
4278
|
+
/** The check record (Harbor's CheckReport) this task's result belongs to. */
|
|
4279
|
+
check_id: string | null;
|
|
4280
|
+
/** The resolved `name@version` when the check came from a published dataset; null for an uploaded archive. */
|
|
4281
|
+
dataset: string | null;
|
|
4282
|
+
/** The checked task's name. */
|
|
4283
|
+
task_name: string | null;
|
|
4284
|
+
/** The model the checker ran. */
|
|
4285
|
+
model_name: string | null;
|
|
4286
|
+
/** Where the CHECKER's own box ran. */
|
|
4287
|
+
sandbox_provider: string | null;
|
|
4288
|
+
sandbox_id: string | null;
|
|
4289
|
+
/** True once the task check has settled (completed or failed). */
|
|
4290
|
+
is_ended: boolean;
|
|
4291
|
+
/** ALL stored rows for this task check, independent of `since`. */
|
|
4292
|
+
total: number;
|
|
4293
|
+
events: TraceEvent[];
|
|
4294
|
+
/** The gateway meter's per-call lines for the checker's key (the AnalysisTranscript's field, same law). */
|
|
4295
|
+
gateway_calls: TraceEvent[];
|
|
4296
|
+
/** The server's write instant of each event's row, one per entry of `events`, index-aligned (the AnalysisTranscript's field, same law); absent when the transcript was served from its file. */
|
|
4297
|
+
stored_at?: string[];
|
|
4298
|
+
}
|
|
4299
|
+
/**
|
|
4300
|
+
* Client for task quality checks — Harbor's `harbor check <PATH>`, hosted
|
|
4301
|
+
* (`POST /api/checks`, `GET /api/checks`, `GET /api/checks/{checkId}`).
|
|
4302
|
+
* Created via the standalone `checks()` factory; requires EVOLVE_API_KEY
|
|
4303
|
+
* (or `{ apiKey }` in config).
|
|
4304
|
+
*
|
|
4305
|
+
* `create()` uploads the task directory (or the directory of task
|
|
4306
|
+
* directories), or names a published dataset, and returns AT ONCE with the
|
|
4307
|
+
* accepted Check — one `results` entry per task, each `queued` — exactly as
|
|
4308
|
+
* Harbor's hosted launch submits and returns; `watch()` is the follow, a
|
|
4309
|
+
* separate poll on purpose. The task bytes are never modified: Harbor's own
|
|
4310
|
+
* rule for its check.
|
|
4311
|
+
*
|
|
4312
|
+
* EACH TASK'S RESULT IS A RUN OF ITS OWN, read like an analysis run (owner
|
|
4313
|
+
* ruling 2026-09-09): `task()`, `transcript()` and `artifact()` take the
|
|
4314
|
+
* TASK CHECK id (`Check.results[].id`) and ride the same off-contract feed
|
|
4315
|
+
* doors AnalysesClient's reads ride (its doc records the tension), under the
|
|
4316
|
+
* check's own access law — the creator and the owning organization's
|
|
4317
|
+
* members; an id you may not read is 404 `trial_not_found`, the code every
|
|
4318
|
+
* door a task check rides speaks (the feed's ?what=analysis selector alone
|
|
4319
|
+
* answers `analysis_not_found`).
|
|
4320
|
+
*/
|
|
4321
|
+
interface ChecksClient {
|
|
4322
|
+
/** Upload a task directory (or a directory of them), or name a published dataset, and start the check. Returns the accepted Check (202). */
|
|
4323
|
+
create(input: CreateCheckInput): Promise<Check>;
|
|
4324
|
+
/** The check with its per-task results — for every status. 404 `check_not_found` for an id you cannot read. */
|
|
4325
|
+
get(checkId: string): Promise<Check>;
|
|
4326
|
+
/** Every check you may read, newest first (cursor-paged); `{ scope, status }` narrow it. */
|
|
4327
|
+
list(options?: ListChecksOptions): CheckList;
|
|
4328
|
+
/** Poll a check until every task settled; resolves with the final Check. */
|
|
4329
|
+
watch(checkId: string, options?: WatchCheckOptions): Promise<Check>;
|
|
4330
|
+
/**
|
|
4331
|
+
* One task's result by TASK CHECK id — the wire's TaskCheck for every
|
|
4332
|
+
* status, typed failure included; the same document the feed downloads
|
|
4333
|
+
* as check-result.json (Harbor's name for the checker's deliverable). A
|
|
4334
|
+
* trial, regrade or analysis id refuses typed at the door, never answering
|
|
4335
|
+
* with another run's document.
|
|
4336
|
+
*/
|
|
4337
|
+
task(taskCheckId: string): Promise<TaskCheck>;
|
|
4338
|
+
/** The checker's own transcript (see TaskCheckTranscript); a wrong-species id refuses with the species named. */
|
|
4339
|
+
transcript(taskCheckId: string, options?: AnalysisTranscriptOptions): Promise<TaskCheckTranscript>;
|
|
4340
|
+
/**
|
|
4341
|
+
* One stored artifact by selector — the checker's raw stdout/stderr, or
|
|
4342
|
+
* "agent-home" as the sandbox-path → text map of its session home (the
|
|
4343
|
+
* analysis run's own three selectors). Null = never stored, a normal
|
|
4344
|
+
* answer; a wrong-species id refuses (the ?what=task-check door is
|
|
4345
|
+
* resolved first, so another run's bytes are never served).
|
|
4346
|
+
*/
|
|
4347
|
+
artifact(taskCheckId: string, stream: Exclude<AnalysisArtifactStream, "agent-home">): Promise<string | null>;
|
|
4348
|
+
artifact(taskCheckId: string, stream: "agent-home"): Promise<Record<string, string> | null>;
|
|
4349
|
+
/**
|
|
4350
|
+
* Download in one `.tar.gz` (`GET /api/checks/{checkId}/download`, on
|
|
4351
|
+
* the contract) EITHER the whole check — pass the CHECK id: the archive
|
|
4352
|
+
* extracts to `check-<id>/`, Harbor's check job folder, with
|
|
4353
|
+
* `check_report.json` (their CheckReport: `results`, one per task —
|
|
4354
|
+
* task_name, the flat checks, cost_usd, error — plus total_cost_usd) and
|
|
4355
|
+
* one wrapper-trial folder per task check, named as Harbor names a trial
|
|
4356
|
+
* (`check-<task>__<7 chars>/`) — OR one task check's folder alone: pass
|
|
4357
|
+
* the TASK CHECK id (`Check.results[].id`). Each folder is Harbor's
|
|
4358
|
+
* TrialPaths for the checker's run: config.json, lock.json, result.json,
|
|
4359
|
+
* trial.log, exception.txt (an infrastructure failure only),
|
|
4360
|
+
* agent/claude-code.txt, agent/stderr.log, agent/trace-parsed.jsonl, the
|
|
4361
|
+
* captured home at its real names with agent/agent-home.json and Harbor's
|
|
4362
|
+
* copy at agent/sessions/, verifier/{test-stdout.txt,reward.txt,
|
|
4363
|
+
* reward.json} when the validator ruled (reward 1 = a valid
|
|
4364
|
+
* check-result.json, 0 = it was refused), artifacts/manifest.json and
|
|
4365
|
+
* artifacts/check-result.json (the validated flat checks) on a completed
|
|
4366
|
+
* run — absent artifacts are absent files; data the platform does not
|
|
4367
|
+
* hold is left out, never faked. Same three delivery shapes and integrity
|
|
4368
|
+
* checks as jobs().download(). An id that is neither a check nor a task
|
|
4369
|
+
* check you can read is 404 `check_not_found` naming both forms; an
|
|
4370
|
+
* unsettled check or task check is 409 `check_not_terminal`.
|
|
4371
|
+
*/
|
|
4372
|
+
download(id: string): Promise<Buffer>;
|
|
4373
|
+
download(id: string, options: {
|
|
4374
|
+
to: string;
|
|
4375
|
+
}): Promise<string>;
|
|
4376
|
+
download(id: string, options: {
|
|
4377
|
+
stream: true;
|
|
4378
|
+
}): Promise<ReadableStream<Uint8Array>>;
|
|
4379
|
+
download(id: string, options?: DownloadJobOptions): Promise<Buffer | string | ReadableStream<Uint8Array>>;
|
|
4380
|
+
}
|
|
4381
|
+
/** A key descriptor. The secret is never returned. */
|
|
4382
|
+
interface ApiKey {
|
|
4383
|
+
id: string;
|
|
4384
|
+
label: string | null;
|
|
4385
|
+
created_at: string;
|
|
4386
|
+
last_used_at: string | null;
|
|
4387
|
+
}
|
|
4388
|
+
/** Who the caller is and the key they used. */
|
|
4389
|
+
interface AuthStatus {
|
|
4390
|
+
user_id: string;
|
|
4391
|
+
email: string | null;
|
|
4392
|
+
key: ApiKey;
|
|
4393
|
+
}
|
|
4394
|
+
/** Client for caller identity. */
|
|
4395
|
+
interface AuthClient {
|
|
4396
|
+
/** Identify the caller and the API key in use. */
|
|
4397
|
+
status(): Promise<AuthStatus>;
|
|
4398
|
+
}
|
|
4399
|
+
/** The caller's role in an organization: owner manages, member reads and runs. */
|
|
4400
|
+
type OrgRole = "owner" | "member";
|
|
4401
|
+
/** An organization the caller belongs to (`GET /api/orgs` item; Harbor's `auth org list` row). */
|
|
4402
|
+
interface Organization {
|
|
4403
|
+
org_id: string;
|
|
4404
|
+
/** URL-safe handle, globally unique — the `<slug>` every org verb takes. */
|
|
4405
|
+
slug: string;
|
|
4406
|
+
display_name: string;
|
|
4407
|
+
/** True for the auto-created personal org — the invisible default owner of everything created without naming an org. */
|
|
4408
|
+
personal: boolean;
|
|
4409
|
+
/** The CALLER'S role; present when the read implies membership. */
|
|
4410
|
+
role?: OrgRole;
|
|
4411
|
+
created_at: string;
|
|
4412
|
+
}
|
|
4413
|
+
/**
|
|
4414
|
+
* An organization's ceilings, every one EFFECTIVE — the value the platform
|
|
4415
|
+
* administrator set, else the fleet default. `0` means paused: creates are
|
|
4416
|
+
* refused `quota_exceeded`, queued work waits. Set only from the platform
|
|
4417
|
+
* administrator's dashboard session; the SDK reads them.
|
|
4418
|
+
*/
|
|
4419
|
+
interface OrgQuota {
|
|
4420
|
+
/** Trials running at once (RUNNING or SCORING); work beyond it waits. */
|
|
4421
|
+
max_concurrent_trials: number;
|
|
4422
|
+
/** Trials waiting in the queue — the one refusal (`quota_exceeded`) on job create. */
|
|
4423
|
+
max_queued_trials: number;
|
|
4424
|
+
/** Dataset imports a worker holds at once; further imports wait. */
|
|
4425
|
+
max_concurrent_imports: number;
|
|
4426
|
+
/** Rubric-agent runs in flight at once fleet-wide — trace analyses AND task quality checks under ONE count; further runs of either kind wait. */
|
|
4427
|
+
max_concurrent_analyses: number;
|
|
4428
|
+
/** Managed-agent sessions open at once (recorded and read back; not yet enforced by the box-create doors). */
|
|
4429
|
+
max_concurrent_sessions: number;
|
|
4430
|
+
/** Model spend allowed this UTC calendar month (the gateway's window, reset on the 1st), USD; null = no monthly budget. */
|
|
4431
|
+
monthly_budget_usd: number | null;
|
|
4432
|
+
/** Sandboxes of this organization in flight on e2b at once — trials, trace analyses, regrade verifiers and managed sessions together; work beyond it waits; 0 pauses the organization on that provider; fleet default = the platform's own e2b ceiling. */
|
|
4433
|
+
max_concurrent_sandboxes_e2b: number;
|
|
4434
|
+
/** The same ceiling on daytona: the organization's sandboxes in flight there at once; work beyond it waits; 0 pauses the organization on daytona; fleet default = the platform's own daytona ceiling. */
|
|
4435
|
+
max_concurrent_sandboxes_daytona: number;
|
|
4436
|
+
/** The same ceiling on modal: the organization's sandboxes in flight there at once; work beyond it waits; 0 pauses the organization on modal; fleet default = the platform's own modal ceiling. */
|
|
4437
|
+
max_concurrent_sandboxes_modal: number;
|
|
4438
|
+
}
|
|
4439
|
+
/** The live load beside the ceilings — what `evolve auth org show` prints as N/M. */
|
|
4440
|
+
interface OrgUsage {
|
|
4441
|
+
in_flight_trials: number;
|
|
4442
|
+
queued_trials: number;
|
|
4443
|
+
in_flight_imports: number;
|
|
4444
|
+
/** Rubric-agent runs RUNNING now — trace analyses AND task quality checks, the one count `max_concurrent_analyses` bounds. */
|
|
4445
|
+
in_flight_analyses: number;
|
|
4446
|
+
/** Sessions not yet ended; always 0 on a shared org (sessions carry no organization). */
|
|
4447
|
+
active_sessions: number;
|
|
4448
|
+
/**
|
|
4449
|
+
* The gateway's own month-to-date meter for the organization — the number
|
|
4450
|
+
* `monthly_budget_usd` is enforced against (UTC calendar month, reset on the
|
|
4451
|
+
* 1st) — as the platform last copied it. null = no copy the platform may
|
|
4452
|
+
* serve (never copied, or the month rolled and the gateway has not reset
|
|
4453
|
+
* yet); never 0 for "unknown".
|
|
4454
|
+
*/
|
|
4455
|
+
month_spend_usd: number | null;
|
|
4456
|
+
/** When the gateway answered the copy above (ISO 8601); null exactly when `month_spend_usd` is. */
|
|
4457
|
+
month_spend_as_of: string | null;
|
|
4458
|
+
}
|
|
4459
|
+
/** One organization in depth (`GET /api/orgs/{org}`): the row, the member count, its quota and usage. */
|
|
4460
|
+
interface OrganizationDetail extends Organization {
|
|
4461
|
+
member_count: number;
|
|
4462
|
+
quota: OrgQuota;
|
|
4463
|
+
usage: OrgUsage;
|
|
4464
|
+
}
|
|
4465
|
+
/**
|
|
4466
|
+
* Client for the caller's organizations — the read pair. Creating, renaming,
|
|
4467
|
+
* deleting, members and invite links are served by the API and stay outside
|
|
4468
|
+
* the SDK until a wave asks for them; quotas are set only from the platform
|
|
4469
|
+
* administrator's dashboard session, so no SDK method could ever set one.
|
|
4470
|
+
*/
|
|
4471
|
+
interface OrgsClient {
|
|
4472
|
+
/** Every organization the caller belongs to, personal first (`GET /api/orgs`). */
|
|
4473
|
+
list(): Promise<Organization[]>;
|
|
4474
|
+
/** One organization by slug (or id): role, member count, quota, usage (`GET /api/orgs/{org}`). */
|
|
4475
|
+
get(org: string): Promise<OrganizationDetail>;
|
|
4476
|
+
}
|
|
4477
|
+
/**
|
|
4478
|
+
* Every error code the hosted API can return, as a closed list.
|
|
4479
|
+
*
|
|
4480
|
+
* This exists so a typo cannot compile. `err.code === "insufficient_creidts"`
|
|
4481
|
+
* used to typecheck (code was `string`) and then silently never match, which is
|
|
4482
|
+
* the worst shape a bug can take: the branch looks handled and never runs.
|
|
4483
|
+
*
|
|
4484
|
+
* It mirrors the ErrorCode enum in spec/openapi.yaml and is published verbatim
|
|
4485
|
+
* at GET /api/meta as `error_codes`. A server newer than this SDK may send a
|
|
4486
|
+
* code that is not listed here — `EvolveApiError.code` widens to string for
|
|
4487
|
+
* exactly that case, so an unknown code is still readable, just not narrowable.
|
|
4488
|
+
*
|
|
4489
|
+
* Held to the spec by hosted-error-codes.json at the package root, the
|
|
4490
|
+
* checked-in copy both SDKs assert against; the list drifted silently before
|
|
4491
|
+
* that file existed. Adding a code means editing the spec, that file, this
|
|
4492
|
+
* list, and the Python pair.
|
|
4493
|
+
*/
|
|
4494
|
+
declare const HOSTED_ERROR_CODES: readonly ["missing_authorization", "invalid_api_key", "read_only_key", "credential_service_unavailable", "rate_limited", "insufficient_credits", "quota_exceeded", "invalid_json", "invalid_input", "invalid_limit", "invalid_status", "invalid_visibility", "invalid_cursor", "invalid_after", "invalid_format", "invalid_ids", "invalid_multipart", "idempotency_key_reused", "dataset_not_found", "dataset_version_not_found", "dataset_name_taken", "dataset_in_use", "dataset_not_owned", "dataset_import_in_progress", "upstream_not_watchable", "no_active_version", "version_not_ready", "version_not_activatable", "unknown_task_names", "no_tasks", "task_not_found", "task_failed_to_build", "upload_session_not_found", "upload_offset_mismatch", "upload_chunk_digest_mismatch", "upload_incomplete", "upload_archive_digest_mismatch", "upload_session_failed", "too_many_concurrent_upload_chunks", "agent_not_found", "agent_name_taken", "agent_name_reserved", "agent_invalid_name", "agent_source_required", "agent_source_conflict", "agent_invalid_env", "agent_too_large", "agent_limit_reached", "skill_not_found", "skill_name_not_found", "skill_ref_invalid", "skill_unresolvable", "skill_invalid", "skill_in_use", "skill_too_large", "skill_limit_reached", "too_many_concurrent_skill_uploads", "secret_not_found", "secret_ambiguous", "secret_brokered_unsupported", "secret_exists", "secret_not_attached", "agent_version_not_found", "agent_version_unresolvable", "agent_kwarg_unsupported", "agent_config_unsupported", "agent_config_key_refused", "agent_preset_unsupported", "provider_unsupported", "job_not_found", "job_not_terminal", "no_failed_trials", "trial_not_found", "trial_not_settled", "concurrent_update", "regrade_source_ineligible", "no_regradable_trials", "invalid_rubric", "analysis_already_running", "analysis_not_found", "analysis_not_terminal", "check_not_found", "check_not_terminal", "no_checkable_tasks", "too_many_concurrent_check_uploads", "no_analyzable_trials", "not_a_job_dir", "invalid_trial", "trial_too_large", "upload_too_large", "job_uploaded", "job_already_uploaded", "too_many_concurrent_job_uploads", "job_import_not_found", "import_not_found", "import_too_large", "too_many_concurrent_imports", "invalid_archive", "unpinned_git_ref", "hub_package_not_found", "hub_unreachable", "package_not_retained", "package_corrupt", "package_missing", "too_many_concurrent_package_downloads", "org_not_found", "org_slug_taken", "org_forbidden", "org_personal_immutable", "org_last_owner", "org_in_use", "org_member_not_found", "invite_not_found", "invite_invalid", "internal_error"];
|
|
4495
|
+
/** One of the API's stable error codes. */
|
|
4496
|
+
type HostedErrorCode = (typeof HOSTED_ERROR_CODES)[number];
|
|
4497
|
+
/** True when `value` is a code this SDK version knows about (narrowing guard). */
|
|
4498
|
+
declare function isHostedErrorCode(value: unknown): value is HostedErrorCode;
|
|
4499
|
+
/** A closed vocabulary a client renders, with the members that end it. */
|
|
4500
|
+
interface StatusVocabulary {
|
|
4501
|
+
values: string[];
|
|
4502
|
+
/** Members after which nothing more happens — a watcher may stop here. */
|
|
4503
|
+
terminal: string[];
|
|
4504
|
+
}
|
|
4505
|
+
/**
|
|
4506
|
+
* The `effort_support` vocabulary, published as a runtime list so the drift
|
|
4507
|
+
* gate can hold this union to the contract's enum — and so a client can
|
|
4508
|
+
* narrow an unknown string. The server serves exactly these members.
|
|
4509
|
+
*/
|
|
4510
|
+
declare const AGENT_EFFORT_SUPPORT_VALUES: readonly ["level", "binary", "none"];
|
|
4511
|
+
/** What an agent does with `agents[].reasoning_effort` — see AgentCapability. */
|
|
4512
|
+
type AgentEffortSupport = (typeof AGENT_EFFORT_SUPPORT_VALUES)[number];
|
|
4513
|
+
/** One model alias an agent offers, for a picker's option list. */
|
|
4514
|
+
interface AgentModelOption {
|
|
4515
|
+
alias: string;
|
|
4516
|
+
model_id: string;
|
|
4517
|
+
description: string | null;
|
|
4518
|
+
}
|
|
4519
|
+
/** One built-in agent's declared capabilities. */
|
|
4520
|
+
interface AgentCapability {
|
|
4521
|
+
name: string;
|
|
4522
|
+
/** false = registered but the agent phase must refuse it, with `reason` set. */
|
|
4523
|
+
runnable: boolean;
|
|
4524
|
+
/** Why not, when `runnable` is false. Null otherwise. */
|
|
4525
|
+
reason: string | null;
|
|
4526
|
+
/**
|
|
4527
|
+
* What the evolve SDK runs when no model is named. This API REQUIRES an
|
|
4528
|
+
* explicit model (limits.job.model_required), so treat it as the sensible
|
|
4529
|
+
* pre-selection for a picker — the server never fills it in for you.
|
|
4530
|
+
*/
|
|
4531
|
+
default_model: string | null;
|
|
4532
|
+
/** Known model aliases for this agent — the picker's option list. */
|
|
4533
|
+
models: AgentModelOption[];
|
|
4534
|
+
/**
|
|
4535
|
+
* What this agent does with `agents[].reasoning_effort`:
|
|
4536
|
+
* 'level' the value reaches the agent CLI as a level
|
|
4537
|
+
* 'binary' thinking on/off only — a level outside
|
|
4538
|
+
* limits.job.binary_effort_values is refused at create
|
|
4539
|
+
* 'none' no effort input at all; naming one is refused at create
|
|
4540
|
+
* Published so a builder greys the control out instead of discovering the
|
|
4541
|
+
* refusal after a POST.
|
|
4542
|
+
*/
|
|
4543
|
+
effort_support: AgentEffortSupport;
|
|
4544
|
+
/** The pinned default stored when a create request omits the effort; null for 'none'. */
|
|
4545
|
+
default_effort: string | null;
|
|
4546
|
+
/** Whether job agents[].version may pin this agent. */
|
|
4547
|
+
version_pinnable: boolean;
|
|
4548
|
+
/**
|
|
4549
|
+
* Whether job `agents[].kwargs.config` reaches this agent — native
|
|
4550
|
+
* agent-settings support (Harbor's SUPPORTS_CONFIG). Declaring a config for
|
|
4551
|
+
* an agent without it is refused `agent_config_unsupported`.
|
|
4552
|
+
*/
|
|
4553
|
+
supports_config?: boolean;
|
|
4554
|
+
/**
|
|
4555
|
+
* The named settings presets this agent can guarantee (`no-internet`,
|
|
4556
|
+
* `pinned-context`). Declaring `agents[].preset` outside this list is
|
|
4557
|
+
* refused `agent_preset_unsupported`. Absent on older servers = none
|
|
4558
|
+
* advertised.
|
|
4559
|
+
*/
|
|
4560
|
+
presets?: string[];
|
|
4561
|
+
/**
|
|
4562
|
+
* Newest published version, for a "your pin is out of date" badge. Null
|
|
4563
|
+
* means "not known right now", never "up to date".
|
|
4564
|
+
*/
|
|
4565
|
+
latest_version?: string | null;
|
|
4566
|
+
}
|
|
4567
|
+
/** One sandbox provider, its ceilings, and what it refuses. */
|
|
4568
|
+
interface ProviderCapability {
|
|
4569
|
+
name: string;
|
|
4570
|
+
default: boolean;
|
|
4571
|
+
sizing: {
|
|
4572
|
+
max_cpus: number;
|
|
4573
|
+
max_memory_mb: number;
|
|
4574
|
+
max_storage_mb: number;
|
|
4575
|
+
storage: "sized" | "fixed";
|
|
4576
|
+
};
|
|
4577
|
+
/**
|
|
4578
|
+
* GPU capability of this provider for eval boxes right now. When
|
|
4579
|
+
* `supported` is false, `degrades_to` names where a GPU job stamped here
|
|
4580
|
+
* actually runs (modal) and `reason` is the same sentence the trial
|
|
4581
|
+
* records; daytona's answer comes from a live org-quota probe that fails
|
|
4582
|
+
* closed, and `source` says whether it was measured ('live-quota') or is
|
|
4583
|
+
* the conservative fallback. Absent on servers predating the field.
|
|
4584
|
+
*/
|
|
4585
|
+
gpus?: {
|
|
4586
|
+
supported: boolean;
|
|
4587
|
+
/** Per-container allocation ceiling; import refuses tasks above it. */
|
|
4588
|
+
max_gpus: number;
|
|
4589
|
+
degrades_to?: "modal";
|
|
4590
|
+
reason?: string;
|
|
4591
|
+
source?: "live-quota" | "fallback" | "provider-constant";
|
|
4592
|
+
};
|
|
4593
|
+
refuses: {
|
|
4594
|
+
capability: string;
|
|
4595
|
+
reason: string;
|
|
4596
|
+
}[];
|
|
4597
|
+
}
|
|
4598
|
+
/**
|
|
4599
|
+
* One managed sandbox door and whether this deployment serves it — a
|
|
4600
|
+
* different question from ProviderCapability, which is about the eval lane.
|
|
4601
|
+
* A managed sandbox is one the caller drives directly holding nothing but an
|
|
4602
|
+
* Evolve key.
|
|
4603
|
+
*/
|
|
4604
|
+
interface ManagedProviderCapability {
|
|
4605
|
+
name: string;
|
|
4606
|
+
/**
|
|
4607
|
+
* The operator config this door reads is present. NOT a health check: it
|
|
4608
|
+
* says nothing about whether the pass-through behind the door is deployed
|
|
4609
|
+
* or the credential behind it is valid.
|
|
4610
|
+
*/
|
|
4611
|
+
configured: boolean;
|
|
4612
|
+
/** Config this door reads, so an operator sees what to set. */
|
|
4613
|
+
requires_config: string[];
|
|
4614
|
+
/** The subset of `requires_config` missing right now — empty when configured. */
|
|
4615
|
+
missing_config: string[];
|
|
4616
|
+
/** A full SDK agent session can run on this door. */
|
|
4617
|
+
agent_sessions: boolean;
|
|
4618
|
+
/** Why not, when `agent_sessions` is false. Null otherwise. */
|
|
4619
|
+
agent_sessions_reason: string | null;
|
|
4620
|
+
}
|
|
4621
|
+
/**
|
|
4622
|
+
* The capability document: everything a client would otherwise hardcode.
|
|
4623
|
+
*
|
|
4624
|
+
* Public and cacheable — no API key needed, so a signed-out page can populate
|
|
4625
|
+
* its own agent picker. `schema_version` bumps when a FIELD changes meaning,
|
|
4626
|
+
* never when a value changes.
|
|
4627
|
+
*/
|
|
4628
|
+
interface CapabilityDocument {
|
|
4629
|
+
schema_version: number;
|
|
4630
|
+
/** Built-in agents and their declared capabilities. */
|
|
4631
|
+
agents: AgentCapability[];
|
|
4632
|
+
/** Rules a bring-your-own agent registration must satisfy. */
|
|
4633
|
+
agent_registration: {
|
|
4634
|
+
name_pattern: string;
|
|
4635
|
+
max_name_length: number;
|
|
4636
|
+
max_run_command_length: number;
|
|
4637
|
+
max_install_script_length: number;
|
|
4638
|
+
max_env_entries: number;
|
|
4639
|
+
max_per_user: number;
|
|
4640
|
+
max_upload_bytes: number;
|
|
4641
|
+
/** Built-in names a registration may not reuse. */
|
|
4642
|
+
reserved_names: string[];
|
|
4643
|
+
/** Env keys the platform owns; declaring one is refused at registration. */
|
|
4644
|
+
reserved_env_keys: string[];
|
|
4645
|
+
};
|
|
4646
|
+
sandbox_providers: ProviderCapability[];
|
|
4647
|
+
/**
|
|
4648
|
+
* Fleet-wide cap on concurrently in-flight trials of GPU-declaring tasks
|
|
4649
|
+
* (platform-paid GPU compute). Queued GPU trials past the cap wait for a
|
|
4650
|
+
* slot. Absent on servers predating the field.
|
|
4651
|
+
*/
|
|
4652
|
+
gpu_concurrency_cap?: number;
|
|
4653
|
+
/** The managed doors this deployment serves, and what each can carry. */
|
|
4654
|
+
managed_providers: ManagedProviderCapability[];
|
|
4655
|
+
/** Constraints that hold on EVERY provider. */
|
|
4656
|
+
platform_constraints: {
|
|
4657
|
+
capability: string;
|
|
4658
|
+
reason: string;
|
|
4659
|
+
}[];
|
|
4660
|
+
network_modes: string[];
|
|
4661
|
+
/** Every status vocabulary on the surface, with terminal members. */
|
|
4662
|
+
statuses: {
|
|
4663
|
+
job: StatusVocabulary;
|
|
4664
|
+
trial: StatusVocabulary;
|
|
4665
|
+
import: StatusVocabulary;
|
|
4666
|
+
dataset_version: StatusVocabulary;
|
|
4667
|
+
};
|
|
4668
|
+
/**
|
|
4669
|
+
* The trace analyzer's roster and defaults: the model an omitted
|
|
4670
|
+
* `analyze.model_name` takes, the efforts `analyze.reasoning_effort`
|
|
4671
|
+
* accepts (the claude harness's — the analyzer IS that harness), and
|
|
4672
|
+
* every roster model with the effort an omitted `reasoning_effort` takes
|
|
4673
|
+
* for it. Absent on servers predating the field.
|
|
4674
|
+
*/
|
|
4675
|
+
analyze?: {
|
|
4676
|
+
default_model: string;
|
|
4677
|
+
reasoning_efforts: string[];
|
|
4678
|
+
models: {
|
|
4679
|
+
alias: string;
|
|
4680
|
+
model_id: string;
|
|
4681
|
+
default_reasoning_effort: string;
|
|
4682
|
+
}[];
|
|
4683
|
+
};
|
|
4684
|
+
limits: {
|
|
4685
|
+
/**
|
|
4686
|
+
* What a client must know to send a good job: the concurrency bound
|
|
4687
|
+
* and the fleet defaults. No fan-out ceiling is published because none
|
|
4688
|
+
* exists (no invented number, owner 2026-09-13): `n_attempts`, the
|
|
4689
|
+
* agent count, the trial matrix, `retry.max_retries` and every timeout
|
|
4690
|
+
* multiplier are unbounded, as in Harbor.
|
|
4691
|
+
*/
|
|
4692
|
+
job: {
|
|
4693
|
+
n_concurrent_trials: {
|
|
4694
|
+
default: number;
|
|
4695
|
+
max: number;
|
|
4696
|
+
};
|
|
4697
|
+
default_max_trial_spend_usd: number;
|
|
4698
|
+
/**
|
|
4699
|
+
* Applied to retry.max_retries when a create request omits it — the
|
|
4700
|
+
* auto-retry fleet default (Harbor's local default is 0; here
|
|
4701
|
+
* infrastructure errors retry automatically).
|
|
4702
|
+
*/
|
|
4703
|
+
default_max_retries: number;
|
|
4704
|
+
default_sandbox_provider: string;
|
|
4705
|
+
default_sizing: {
|
|
4706
|
+
cpus: number;
|
|
4707
|
+
memory_mb: number;
|
|
4708
|
+
storage_mb: number;
|
|
4709
|
+
};
|
|
4710
|
+
/** Every agent must name a model; the server applies no default. */
|
|
4711
|
+
model_required: boolean;
|
|
4712
|
+
/**
|
|
4713
|
+
* Phase wall-clocks a task INHERITS when its own config declares none —
|
|
4714
|
+
* a task that declares its own always wins, so these fill in rather than
|
|
4715
|
+
* cap. Published because nothing else here says how long a trial may run.
|
|
4716
|
+
*/
|
|
4717
|
+
default_agent_timeout_sec: number;
|
|
4718
|
+
default_verifier_timeout_sec: number;
|
|
4719
|
+
/** Values agents[].reasoning_effort accepts, and the one an omitted effort takes. */
|
|
4720
|
+
reasoning_efforts: string[];
|
|
4721
|
+
default_reasoning_effort: string;
|
|
4722
|
+
};
|
|
4723
|
+
compare: {
|
|
4724
|
+
min_ids: number;
|
|
4725
|
+
max_ids: number;
|
|
4726
|
+
};
|
|
4727
|
+
pagination: {
|
|
4728
|
+
collections: {
|
|
4729
|
+
default: number;
|
|
4730
|
+
max: number;
|
|
4731
|
+
};
|
|
4732
|
+
dataset_tasks: {
|
|
4733
|
+
default: number;
|
|
4734
|
+
max: number;
|
|
4735
|
+
};
|
|
4736
|
+
trace_events: {
|
|
4737
|
+
default: number;
|
|
4738
|
+
max: number;
|
|
4739
|
+
};
|
|
4740
|
+
};
|
|
4741
|
+
/**
|
|
4742
|
+
* The ONE physical ceiling on an uploaded dataset, job or check archive
|
|
4743
|
+
* — what the object store can land (S3's 10,000 parts of the store
|
|
4744
|
+
* move's part size; the local store's free disk in local mode) —
|
|
4745
|
+
* published with its source. No archive cap, trial-count cap, per-file
|
|
4746
|
+
* or per-tree cap of the platform's exists (owner 2026-09-13).
|
|
4747
|
+
*/
|
|
4748
|
+
uploads: {
|
|
4749
|
+
dataset_archive_bytes: number;
|
|
4750
|
+
/** The same ceiling, for a job archive. */
|
|
4751
|
+
job_archive_bytes: number;
|
|
4752
|
+
/** Where the archive ceiling was read from — S3's multipart limits, or the local store's statfs reading. */
|
|
4753
|
+
archive_bytes_source: string;
|
|
4754
|
+
agent_tarball_bytes: number;
|
|
4755
|
+
/** Compressed cap on one uploaded skill tarball (`skill_too_large` past it). */
|
|
4756
|
+
skill_archive_bytes: number;
|
|
4757
|
+
/** Uploaded-skill records one caller may hold (`skill_limit_reached` past it). */
|
|
4758
|
+
skill_uploads_per_user: number;
|
|
4759
|
+
};
|
|
4760
|
+
dataset_names: {
|
|
4761
|
+
pattern: string;
|
|
4762
|
+
max_name_length: number;
|
|
4763
|
+
max_version_length: number;
|
|
4764
|
+
max_git_url_length: number;
|
|
4765
|
+
max_git_ref_length: number;
|
|
4766
|
+
max_git_path_length: number;
|
|
4767
|
+
};
|
|
4768
|
+
/** How many items an error MESSAGE names before "and N more". */
|
|
4769
|
+
max_items_named_in_error_message: number;
|
|
4770
|
+
};
|
|
4771
|
+
/** The ImportWarning codes the platform can attach to an import. */
|
|
4772
|
+
import_warning_codes: string[];
|
|
4773
|
+
/** The closed error-code union, enumerated at runtime. */
|
|
4774
|
+
error_codes: string[];
|
|
4775
|
+
}
|
|
4776
|
+
|
|
4777
|
+
export { type AnalysisList as $, type AgentInput as A, type AgentDatasetStats as B, type Check as C, type DatasetVersion as D, type AgentEffortSupport as E, type AgentInfo as F, type GatewayUsageEvent as G, type HostedErrorCode as H, type AgentList as I, type JobCreate as J, type AgentModelOption as K, type AgentPage as L, type AgentResult as M, type AgentSource as N, type OrgsClient as O, type PublishDatasetInput as P, type AgentSourceInput as Q, type Rubric as R, type SkillsClient as S, type Trial as T, type UsageReading as U, type AgentUpsertInput as V, type AnalysisArtifactStream as W, type AnalysisCheck as X, type AnalysisEvidence as Y, type AnalysisFailure as Z, type AnalysisLabel as _, type TrialAnalysis as a, type JobImportPhaseProgress as a$, type AnalysisPage as a0, type AnalysisStatus as a1, type AnalysisTranscript as a2, type AnalysisTranscriptOptions as a3, type AnalyzeConfig as a4, type AnalyzeConfigInput as a5, type ApiKey as a6, type AttemptPhase as a7, type AuthStatus as a8, type Awaitable as a9, type DownloadJobOptions as aA, EVAL_SANDBOX_PROVIDERS as aB, type Agent as aC, type ModelInfo as aD, type EvalSandboxProvider as aE, type StepResult as aF, type ExceptionInfo as aG, GATEWAY_TRACE_SEQ_BASE as aH, type GatewayUsage as aI, type GetDatasetOptions as aJ, type GrepJobOptions as aK, HOSTED_ERROR_CODES as aL, type ImportPhase as aM, type ImportPhaseProgress as aN, type ImportWarning as aO, type InfraFailureSignature as aP, JOB_LIST_SCOPES as aQ, type JobAnalysisStats as aR, type JobDeleteResult as aS, type JobFailure as aT, type JobGrepGroup as aU, type JobGrepPage as aV, type JobImport as aW, type JobImportFailure as aX, type JobImportList as aY, type JobImportPage as aZ, type JobImportPhaseName as a_, CHECK_STATUSES as aa, type CheckLabel as ab, type CompareCell as ac, type CompareCoverage as ad, type CompareJobAggregate as ae, type CompareResponse as af, type CompareTaskRow as ag, type Dataset as ah, type DatasetImportFailure as ai, type DatasetImportList as aj, type DatasetImportPage as ak, type DatasetImportStatus as al, type DatasetList as am, type DatasetPage as an, type DatasetPatch as ao, type DatasetPreflight as ap, type DatasetRef as aq, type DatasetSelector as ar, type DatasetSource as as, type DatasetVersionArchiveSource as at, type DatasetVersionArchiveUrlSource as au, type DatasetVersionGitSource as av, type DatasetVersionHubSource as aw, type DatasetVersionSource as ax, type DatasetVersionState as ay, type DownloadDatasetOptions as az, type DatasetImport as b, type TimingInfo as b$, type JobImportProgress as b0, type JobImportSkippedTrial as b1, type JobImportSource as b2, type JobList as b3, type JobListScope as b4, type JobPage as b5, type JobStats as b6, type JobStatus as b7, type JobTaskLink as b8, type JobTaskRollup as b9, type PreflightManifestVerdict as bA, type PreflightTaskVerdict as bB, type ProviderCapability as bC, type PublishDatasetOptions as bD, type RegradeRequest as bE, type ResumeRequest as bF, type RetryRequest as bG, type RubricCriterion as bH, type SkillLock as bI, type SkillUpload as bJ, type SkillUploadList as bK, type SkillUploadPage as bL, type SourceJob as bM, type SpendSource as bN, type StartJobOptions as bO, type StatusVocabulary as bP, type StopResponse as bQ, TASK_LINKED_BY as bR, TASK_LINK_REASONS as bS, TRIAL_ARTIFACT_STREAMS as bT, TRIAL_STATUSES as bU, type Task as bV, type TaskCheckTranscript as bW, type TaskLinkReason as bX, type TaskLinkedBy as bY, type TaskNote as bZ, type TaskProviderVerdict as b_, type JobTaskRollupList as ba, type JobTaskRollupPage as bb, type JobWatch as bc, type JudgeResult as bd, type ListAgentsOptions as be, type ListAnalysesOptions as bf, type ListDatasetsOptions as bg, type ListImportsOptions as bh, type ListJobImportsOptions as bi, type ListJobTasksOptions as bj, type ListJobsOptions as bk, type ListSkillsOptions as bl, type ListTrialFilesOptions as bm, type ListTrialsOptions as bn, type ManagedProviderCapability as bo, type OrgQuota as bp, type OrgRole as bq, type OrgUsage as br, type Organization as bs, type OrganizationDetail as bt, type Page as bu, type PageOptions as bv, type PassAtKGroup as bw, type PassAtKPoint as bx, type PreflightDatasetInput as by, type PreflightDeferredCheck as bz, type JobEvent as c, type TraceEventPage as c0, type TraceOptions as c1, type TrialArtifactStream as c2, type TrialCounts as c3, type TrialFile as c4, type TrialFilePage as c5, type TrialFileRange as c6, type TrialGpuCost as c7, type TrialList as c8, type TrialPage as c9, type TrialStatus as ca, type TrialStatusTally as cb, type TrialTaskLink as cc, type TrialUploadProvenance as cd, type UploadJobOptions as ce, type UploadProvenance as cf, type UpstreamStatus as cg, type VerifierEnvironmentMode as ch, type VerifierResult as ci, type WatchAnalysisOptions as cj, type WatchImportOptions as ck, type WatchJobImportOptions as cl, type WatchJobOptions as cm, gatewayUsageOf as cn, isHostedErrorCode as co, passAtK as cp, type DatasetFailedTask as d, type DatasetImportProgress as e, type JobSecretInline as f, type JobSecretRef as g, type TraceEvent as h, type Job as i, type TaskCheck as j, type DatasetsClient as k, type AgentsClient as l, type JobsClient as m, type TrialsClient as n, type AnalysesClient as o, type ChecksClient as p, type CapabilityDocument as q, type HostedClientConfig as r, type AuthClient as s, AGENT_EFFORT_SUPPORT_VALUES as t, ANALYSIS_ARTIFACT_STREAMS as u, ANALYSIS_STATUSES as v, type ActiveDataset as w, type AgentArm as x, type AgentArmInput as y, type AgentCapability as z };
|