@error-bar/mcp 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +155 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +58 -0
- package/dist/client.d.ts +38 -0
- package/dist/client.js +62 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.js +5 -0
- package/dist/manifest-types.d.ts +32 -0
- package/dist/manifest-types.js +1 -0
- package/dist/manifest.d.ts +2 -0
- package/dist/manifest.js +2042 -0
- package/dist/server.d.ts +21 -0
- package/dist/server.js +135 -0
- package/package.json +47 -0
package/dist/manifest.js
ADDED
|
@@ -0,0 +1,2042 @@
|
|
|
1
|
+
export const OPERATIONS = [
|
|
2
|
+
{
|
|
3
|
+
"name": "list_alerts",
|
|
4
|
+
"method": "GET",
|
|
5
|
+
"path": "/v1/alerts",
|
|
6
|
+
"summary": "List every alert this workspace has fired, newest first, with the payload the notification carried, so a pipeline can react to quality, cost, or drift events without reading a mailbox.",
|
|
7
|
+
"scope": "read",
|
|
8
|
+
"query": [
|
|
9
|
+
{
|
|
10
|
+
"name": "kind",
|
|
11
|
+
"type": "string",
|
|
12
|
+
"description": "Filter to one alert kind.",
|
|
13
|
+
"enum": [
|
|
14
|
+
"error_rate",
|
|
15
|
+
"latency_p90",
|
|
16
|
+
"balance_low",
|
|
17
|
+
"judge_drift",
|
|
18
|
+
"quality_low",
|
|
19
|
+
"model_shift",
|
|
20
|
+
"quality_gate"
|
|
21
|
+
]
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
"name": "since",
|
|
25
|
+
"type": "string",
|
|
26
|
+
"description": "ISO-8601 datetime; only alerts fired at or after this instant. 400 if unparseable."
|
|
27
|
+
},
|
|
28
|
+
{
|
|
29
|
+
"name": "limit",
|
|
30
|
+
"type": "integer",
|
|
31
|
+
"description": "Page size, integer 1..200.",
|
|
32
|
+
"default": 50
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
"name": "cursor",
|
|
36
|
+
"type": "string",
|
|
37
|
+
"description": "Opaque cursor from a previous response's next_cursor (the id of the last alert on that page). Resumes after that alert."
|
|
38
|
+
}
|
|
39
|
+
],
|
|
40
|
+
"responseSummary": "{ alerts: [{ id, kind, fired_at (ISO), payload (JSON object: criterion, model, rates, reason as applicable) }], next_cursor: string|null }",
|
|
41
|
+
"notes": "Keyset pagination: pass next_cursor back as cursor until it is null. 400 when since is not an ISO date or limit is outside 1..200. Cache-Control: no-store."
|
|
42
|
+
},
|
|
43
|
+
{
|
|
44
|
+
"name": "list_aliases",
|
|
45
|
+
"method": "GET",
|
|
46
|
+
"path": "/v1/aliases",
|
|
47
|
+
"summary": "List this workspace's model aliases (stable names your code calls) with their current target, canary split, quality-gate config, evidence policy and the eval run that authorized the current routing.",
|
|
48
|
+
"scope": "read",
|
|
49
|
+
"responseSummary": "{ object: \"list\", data: [{ id, name, target_model, canary_model, canary_percent, description, gate_criterion_id, gate_mode (\"recommend\"|\"auto\"), gate_min_samples, gate_rollback_threshold, gate_window_hours, gate_verdict ({decision, reason, canary, incumbent, acted}|null), gate_verdict_at, model_version_id, require_evidence, last_evidence_run_id, created_at, updated_at }] }",
|
|
50
|
+
"notes": "Sorted by name ascending. last_evidence_run_id is null when the routing predates the evidence policy or went through as an audited override."
|
|
51
|
+
},
|
|
52
|
+
{
|
|
53
|
+
"name": "upsert_alias",
|
|
54
|
+
"method": "PUT",
|
|
55
|
+
"path": "/v1/aliases",
|
|
56
|
+
"summary": "Create or repoint a model alias by name (idempotent upsert) so production traffic moves to a new model without a redeploy; optionally attach a canary split, a quality gate, or an evidence-required policy.",
|
|
57
|
+
"scope": "aliases:write",
|
|
58
|
+
"body": [
|
|
59
|
+
{
|
|
60
|
+
"name": "name",
|
|
61
|
+
"type": "string",
|
|
62
|
+
"description": "Alias name, 3..64 chars of letters/digits/dots/dashes/underscores, must start and end alphanumeric, no '/'. Upsert key within the workspace.",
|
|
63
|
+
"required": true
|
|
64
|
+
},
|
|
65
|
+
{
|
|
66
|
+
"name": "target_model",
|
|
67
|
+
"type": "string",
|
|
68
|
+
"description": "Model id that receives the main share of traffic. Must be an available model or the call fails with \"Model '<id>' is not available.\"",
|
|
69
|
+
"required": true
|
|
70
|
+
},
|
|
71
|
+
{
|
|
72
|
+
"name": "canary_model",
|
|
73
|
+
"type": "string",
|
|
74
|
+
"description": "Model id for the canary arm (nullable). Must differ from target_model and be an available model. Required (non-null) whenever canary_percent > 0."
|
|
75
|
+
},
|
|
76
|
+
{
|
|
77
|
+
"name": "canary_percent",
|
|
78
|
+
"type": "integer",
|
|
79
|
+
"description": "Integer 0..100 share of traffic sent to canary_model. Default 0. Ignored (stored as 0) when canary_model is null."
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
"name": "description",
|
|
83
|
+
"type": "string",
|
|
84
|
+
"description": "Free-text note, max 200 chars (nullable)."
|
|
85
|
+
},
|
|
86
|
+
{
|
|
87
|
+
"name": "gate_criterion_id",
|
|
88
|
+
"type": "string",
|
|
89
|
+
"description": "Id of a criterion in this workspace that scores both arms online. Null = no gate. The criterion must have been aligned at least once."
|
|
90
|
+
},
|
|
91
|
+
{
|
|
92
|
+
"name": "gate_mode",
|
|
93
|
+
"type": "string",
|
|
94
|
+
"description": "\"recommend\" (default) only surfaces verdicts; \"auto\" lets the gate repoint the alias itself and requires a trustworthy, non-drift-flagged, request-unit judge that did not train the destination model.",
|
|
95
|
+
"enum": [
|
|
96
|
+
"recommend",
|
|
97
|
+
"auto"
|
|
98
|
+
]
|
|
99
|
+
},
|
|
100
|
+
{
|
|
101
|
+
"name": "gate_min_samples",
|
|
102
|
+
"type": "integer",
|
|
103
|
+
"description": "Scored requests both arms need before a verdict. Integer 10..1000, default 50."
|
|
104
|
+
},
|
|
105
|
+
{
|
|
106
|
+
"name": "gate_rollback_threshold",
|
|
107
|
+
"type": "number",
|
|
108
|
+
"description": "Roll back when the canary's upper CI bound on pass rate is below this. Number 0..1, default 0.7."
|
|
109
|
+
},
|
|
110
|
+
{
|
|
111
|
+
"name": "gate_window_hours",
|
|
112
|
+
"type": "integer",
|
|
113
|
+
"description": "Trailing window of online scores a verdict is computed over. Integer 1..720, default 168."
|
|
114
|
+
},
|
|
115
|
+
{
|
|
116
|
+
"name": "require_evidence",
|
|
117
|
+
"type": "boolean",
|
|
118
|
+
"description": "Evidence policy. Omitted = leave the existing alias's setting unchanged (false on create). When on, a repoint that sends traffic to a model it is not already reaching is refused unless a finished comparison in the last 30 days proves the destination against the incumbent."
|
|
119
|
+
},
|
|
120
|
+
{
|
|
121
|
+
"name": "override_reason",
|
|
122
|
+
"type": "string",
|
|
123
|
+
"description": "Audited escape hatch for the evidence policy: a written justification of at least 10 characters lets the repoint through and records an audit event. Blank/missing is NOT an override. Shorter than 10 chars is a 400."
|
|
124
|
+
}
|
|
125
|
+
],
|
|
126
|
+
"responseSummary": "200 with the alias object: { id, name, target_model, canary_model, canary_percent, description, gate_criterion_id, gate_mode, gate_min_samples, gate_rollback_threshold, gate_window_hours, gate_verdict, gate_verdict_at, model_version_id, require_evidence, last_evidence_run_id, created_at, updated_at }",
|
|
127
|
+
"notes": "MOVES PRODUCTION TRAFFIC: the gateway resolves aliases within ~10s. Requires the key's minting user to be workspace OWNER/ADMIN (403 otherwise). Same status 200 whether created or updated. 412 Precondition Failed (code precondition_failed) when the evidence policy refuses the repoint; a brand-new alias is never blocked by the policy. 400 for schema failures, canary_percent > 0 without canary_model, canary equal to target, unavailable model, gate criterion never aligned, or auto-mode eligibility refusals (judge not trustworthy, drift-flagged, trace-unit, or judge trained the destination). 404 \"Gate criterion not found\". Billing always follows the model that actually ran; an alias is routing only."
|
|
128
|
+
},
|
|
129
|
+
{
|
|
130
|
+
"name": "delete_alias",
|
|
131
|
+
"method": "DELETE",
|
|
132
|
+
"path": "/v1/aliases/{id}",
|
|
133
|
+
"summary": "Remove a model alias; requests still using that name will fail afterwards, so this is a cutover step, not cleanup.",
|
|
134
|
+
"scope": "aliases:write",
|
|
135
|
+
"pathParams": [
|
|
136
|
+
{
|
|
137
|
+
"name": "id",
|
|
138
|
+
"type": "string",
|
|
139
|
+
"description": "The alias id (from GET /v1/aliases), not its name."
|
|
140
|
+
}
|
|
141
|
+
],
|
|
142
|
+
"responseSummary": "200 { ok: true }",
|
|
143
|
+
"notes": "OWNER/ADMIN only (403). 404 \"Alias not found\" when the id is not in this workspace. Audited with the model it pointed at."
|
|
144
|
+
},
|
|
145
|
+
{
|
|
146
|
+
"name": "export_audit_log",
|
|
147
|
+
"method": "GET",
|
|
148
|
+
"path": "/v1/audit/export",
|
|
149
|
+
"summary": "Export this workspace's audit rows with their hash-chain fields (seq, prev_hash, row_hash) as CSV or JSON, so a recipient can verify a later export reproduces the same hashes.",
|
|
150
|
+
"scope": "read",
|
|
151
|
+
"query": [
|
|
152
|
+
{
|
|
153
|
+
"name": "since",
|
|
154
|
+
"type": "string",
|
|
155
|
+
"description": "ISO-8601 datetime lower bound (inclusive) on the row timestamp. 400 if unparseable."
|
|
156
|
+
},
|
|
157
|
+
{
|
|
158
|
+
"name": "until",
|
|
159
|
+
"type": "string",
|
|
160
|
+
"description": "ISO-8601 datetime upper bound (inclusive). 400 if unparseable."
|
|
161
|
+
},
|
|
162
|
+
{
|
|
163
|
+
"name": "format",
|
|
164
|
+
"type": "string",
|
|
165
|
+
"description": "Output format.",
|
|
166
|
+
"enum": [
|
|
167
|
+
"csv",
|
|
168
|
+
"json"
|
|
169
|
+
],
|
|
170
|
+
"default": "csv"
|
|
171
|
+
},
|
|
172
|
+
{
|
|
173
|
+
"name": "limit",
|
|
174
|
+
"type": "integer",
|
|
175
|
+
"description": "Max rows, positive integer; silently capped at 50000.",
|
|
176
|
+
"default": 50000
|
|
177
|
+
}
|
|
178
|
+
],
|
|
179
|
+
"responseSummary": "format=json: { rows: [{ id, timestamp, event_type, category, status, actor_id, actor_email, actor_role, target_type, target_id, description, seq, prev_hash, row_hash }], truncated: boolean }. format=csv: text/csv attachment (Content-Disposition audit-<workspaceId>.csv) with header row seq,timestamp,event_type,category,status,actor_id,actor_email,actor_role,target_type,target_id,description,prev_hash,row_hash,id; header X-Truncated: true when the limit cut the result.",
|
|
180
|
+
"notes": "Rows ordered by timestamp then seq ascending. Returns an empty set (not an error) if the log store is unavailable. Cache-Control: no-store.",
|
|
181
|
+
"raw": true
|
|
182
|
+
},
|
|
183
|
+
{
|
|
184
|
+
"name": "list_audit_tombstones",
|
|
185
|
+
"method": "GET",
|
|
186
|
+
"path": "/v1/audit/tombstones",
|
|
187
|
+
"summary": "List acknowledged audit-chain gaps (tombstones) with the recorded reason for each lost slot, so a known loss can be distinguished from tampering.",
|
|
188
|
+
"scope": "read",
|
|
189
|
+
"responseSummary": "{ tombstones: [{ seq (integer), reason, created_at (ISO) }] } sorted by seq ascending",
|
|
190
|
+
"notes": "The audit chain is platform-global, so this list is the same for every workspace. Cache-Control: no-store."
|
|
191
|
+
},
|
|
192
|
+
{
|
|
193
|
+
"name": "create_audit_tombstone",
|
|
194
|
+
"method": "POST",
|
|
195
|
+
"path": "/v1/audit/tombstones",
|
|
196
|
+
"summary": "Acknowledge a verified audit-chain gap with a written reason so the integrity check stops reporting it as unexplained; a platform-admin repair action, never a way to hide a gap.",
|
|
197
|
+
"scope": "platform:write",
|
|
198
|
+
"body": [
|
|
199
|
+
{
|
|
200
|
+
"name": "seq",
|
|
201
|
+
"type": "integer",
|
|
202
|
+
"description": "Positive integer chain sequence number. Must appear as a problem of kind \"gap\" in the latest stored verification (see GET /v1/audit/verify) or the call is refused.",
|
|
203
|
+
"required": true
|
|
204
|
+
},
|
|
205
|
+
{
|
|
206
|
+
"name": "reason",
|
|
207
|
+
"type": "string",
|
|
208
|
+
"description": "Why the slot was lost. Trimmed; at least 10 characters; stored up to 500 characters. Upserting an existing seq replaces the reason.",
|
|
209
|
+
"required": true
|
|
210
|
+
}
|
|
211
|
+
],
|
|
212
|
+
"responseSummary": "201 { seq, reason, created_at }",
|
|
213
|
+
"notes": "403 unless the key's minting user has the platform-level admin role (workspace OWNER/ADMIN is not enough). 400 for invalid JSON, wrong types, seq <= 0, reason under 10 chars, or a seq that is not a gap in the latest verification (message names when that verification ran, or that none has run yet)."
|
|
214
|
+
},
|
|
215
|
+
{
|
|
216
|
+
"name": "get_audit_verification",
|
|
217
|
+
"method": "GET",
|
|
218
|
+
"path": "/v1/audit/verify",
|
|
219
|
+
"summary": "Read the latest nightly whole-chain integrity verification of the audit log (ok flag, rows checked, head seq, problems found, acknowledged gaps) plus the tombstone list, as the platform's integrity statement.",
|
|
220
|
+
"scope": "read",
|
|
221
|
+
"responseSummary": "{ verification: { ran_at, ok, checked_rows, head_seq, problems: [{ seq, kind, detail }], acknowledged (count) } | null, tombstones: [{ seq, reason, created_at }], statement: string describing the hashing scheme }",
|
|
222
|
+
"notes": "verification is null until the first nightly run has stored a result. The chain is platform-global; your own rows' hashes come from GET /v1/audit/export. Cache-Control: no-store."
|
|
223
|
+
},
|
|
224
|
+
{
|
|
225
|
+
"name": "list_batches",
|
|
226
|
+
"method": "GET",
|
|
227
|
+
"path": "/v1/batches",
|
|
228
|
+
"summary": "List this workspace's batch inference jobs, newest first, with status, request counts and billed cost.",
|
|
229
|
+
"scope": "read",
|
|
230
|
+
"responseSummary": "A bare JSON array (no envelope) of up to 100 batch objects: { id, nebius_batch_id (upstream batch id), endpoint, status (VALIDATING|IN_PROGRESS|FINALIZING|COMPLETED|FAILED|EXPIRED|CANCELLING|CANCELLED), request_total, request_completed, request_failed, completion_window, billed_cost_usd (number|null), created_at, output_file_id, error_file_id, error }",
|
|
231
|
+
"notes": "Feature-flagged: every /v1/batches route returns 404 { error: \"Batch inference is not enabled\" } while the batch flag is off (code default is off). Statuses here are the stored values; GET /v1/batches/{id} refreshes them live."
|
|
232
|
+
},
|
|
233
|
+
{
|
|
234
|
+
"name": "create_batch",
|
|
235
|
+
"method": "POST",
|
|
236
|
+
"path": "/v1/batches",
|
|
237
|
+
"summary": "Submit an asynchronous, discounted batch of inference requests from a previously uploaded JSONL file, for workloads that can wait up to the completion window.",
|
|
238
|
+
"scope": "platform:write",
|
|
239
|
+
"body": [
|
|
240
|
+
{
|
|
241
|
+
"name": "input_file_id",
|
|
242
|
+
"type": "string",
|
|
243
|
+
"description": "Id of a file uploaded via /v1/files with purpose \"batch\" containing the request JSONL. Must belong to this workspace (or be the input of one of its past batches); otherwise 404 \"Input file not found\".",
|
|
244
|
+
"required": true
|
|
245
|
+
},
|
|
246
|
+
{
|
|
247
|
+
"name": "endpoint",
|
|
248
|
+
"type": "string",
|
|
249
|
+
"description": "The API route every line in the file targets.",
|
|
250
|
+
"required": true,
|
|
251
|
+
"enum": [
|
|
252
|
+
"/v1/chat/completions",
|
|
253
|
+
"/v1/completions",
|
|
254
|
+
"/v1/embeddings"
|
|
255
|
+
]
|
|
256
|
+
},
|
|
257
|
+
{
|
|
258
|
+
"name": "model",
|
|
259
|
+
"type": "string",
|
|
260
|
+
"description": "A representative model id from the file; it is the billing-rate basis. Must have configured pricing or the call is refused with 400.",
|
|
261
|
+
"required": true
|
|
262
|
+
},
|
|
263
|
+
{
|
|
264
|
+
"name": "completion_window",
|
|
265
|
+
"type": "string",
|
|
266
|
+
"description": "How long the batch may take, e.g. \"24h\".",
|
|
267
|
+
"default": "24h"
|
|
268
|
+
}
|
|
269
|
+
],
|
|
270
|
+
"responseSummary": "201 with the batch object: { id, nebius_batch_id, endpoint, status, request_total, request_completed, request_failed, completion_window, billed_cost_usd, created_at, output_file_id, error_file_id, error }",
|
|
271
|
+
"notes": "MONEY: the wallet must hold at least $0.10 of available runway to submit (402 otherwise); the batch discount and markup are frozen at submit time and the job is billed on completion. OWNER/ADMIN only (403). 400 when input_file_id/endpoint/model is missing or the body is not JSON. 503 when batch creation is temporarily unavailable upstream (the input file stays uploaded; retry later). 404 while the batch feature flag is off.",
|
|
272
|
+
"spends": true
|
|
273
|
+
},
|
|
274
|
+
{
|
|
275
|
+
"name": "get_batch",
|
|
276
|
+
"method": "GET",
|
|
277
|
+
"path": "/v1/batches/{id}",
|
|
278
|
+
"summary": "Fetch one batch job with its status, request counts and output/error file ids refreshed live from the processing backend, to poll for completion.",
|
|
279
|
+
"scope": "read",
|
|
280
|
+
"pathParams": [
|
|
281
|
+
{
|
|
282
|
+
"name": "id",
|
|
283
|
+
"type": "string",
|
|
284
|
+
"description": "The batch id from POST /v1/batches or GET /v1/batches (the platform id, not the upstream batch id)."
|
|
285
|
+
}
|
|
286
|
+
],
|
|
287
|
+
"responseSummary": "{ id, nebius_batch_id, endpoint, status, request_total, request_completed, request_failed, completion_window, billed_cost_usd, created_at, output_file_id, error_file_id, error }",
|
|
288
|
+
"notes": "Best-effort live reconciliation: if the upstream status lookup fails the stored row is returned unchanged. Billing still happens in the background reconciler, not on this read. 404 \"Batch not found\" outside the workspace; 404 while the batch flag is off."
|
|
289
|
+
},
|
|
290
|
+
{
|
|
291
|
+
"name": "cancel_batch",
|
|
292
|
+
"method": "POST",
|
|
293
|
+
"path": "/v1/batches/{id}/cancel",
|
|
294
|
+
"summary": "Request cancellation of an in-flight batch job.",
|
|
295
|
+
"scope": "platform:write",
|
|
296
|
+
"pathParams": [
|
|
297
|
+
{
|
|
298
|
+
"name": "id",
|
|
299
|
+
"type": "string",
|
|
300
|
+
"description": "The platform batch id."
|
|
301
|
+
}
|
|
302
|
+
],
|
|
303
|
+
"responseSummary": "200 with the updated batch object (status typically CANCELLING or CANCELLED)",
|
|
304
|
+
"notes": "OWNER/ADMIN only (403). 404 \"Batch not found\". Audited. 404 while the batch feature flag is off. Work already completed before cancellation may still be billed."
|
|
305
|
+
},
|
|
306
|
+
{
|
|
307
|
+
"name": "list_criteria",
|
|
308
|
+
"method": "GET",
|
|
309
|
+
"path": "/v1/criteria",
|
|
310
|
+
"summary": "List this workspace's judge criteria with their calibration metrics (TPR/TNR/kappa with intervals), trust verdict, drift status and online-monitoring config, to see which judges are proven enough to gate on.",
|
|
311
|
+
"scope": "read",
|
|
312
|
+
"responseSummary": "{ object: \"list\", data: [{ id, name, description, judge_prompt, judge_model, status, source, unit (\"request\"|\"trace\"), population (tag), population_family, online_enabled, online_percent, online_cap_usd, online_spent_usd, tier (aligned|weak|misaligned|unmeasured), trust (trustworthy|misaligned|under-measured|borderline|unmeasured), fail_grades_needed, pass_grades_needed, tpr_ci ([lo,hi]|null), tnr_ci, drift_status (ok|flagged), drift_signal (stale|quality_drop|null), drift_reason, drift_checked_at, tpr, tnr, kappa, alignment_n, aligned_at, created_at }] }",
|
|
313
|
+
"notes": "trust is what every gate reads; tier is the legacy point-estimate badge. drift_status is derived (a fresh calibration supersedes a cached flag)."
|
|
314
|
+
},
|
|
315
|
+
{
|
|
316
|
+
"name": "create_criterion",
|
|
317
|
+
"method": "POST",
|
|
318
|
+
"path": "/v1/criteria",
|
|
319
|
+
"summary": "Create a judge criterion (a rubric prompt run by a judge model) that can score traffic online and be calibrated against human labels.",
|
|
320
|
+
"scope": "evals:write",
|
|
321
|
+
"body": [
|
|
322
|
+
{
|
|
323
|
+
"name": "name",
|
|
324
|
+
"type": "string",
|
|
325
|
+
"description": "Unique within the workspace, trimmed, 1..80 chars. Duplicate name is a 400.",
|
|
326
|
+
"required": true
|
|
327
|
+
},
|
|
328
|
+
{
|
|
329
|
+
"name": "description",
|
|
330
|
+
"type": "string",
|
|
331
|
+
"description": "Optional note, max 500 chars (nullable)."
|
|
332
|
+
},
|
|
333
|
+
{
|
|
334
|
+
"name": "judge_prompt",
|
|
335
|
+
"type": "string",
|
|
336
|
+
"description": "The rubric the judge model applies, trimmed, 10..4000 chars.",
|
|
337
|
+
"required": true
|
|
338
|
+
},
|
|
339
|
+
{
|
|
340
|
+
"name": "judge_model",
|
|
341
|
+
"type": "string",
|
|
342
|
+
"description": "Model id that runs the judgment. Must be an available model (400 \"Judge model '<id>' is not available.\").",
|
|
343
|
+
"required": true
|
|
344
|
+
},
|
|
345
|
+
{
|
|
346
|
+
"name": "unit",
|
|
347
|
+
"type": "string",
|
|
348
|
+
"description": "What one verdict covers: \"request\" judges one exchange, \"trace\" judges a whole agent run. CREATE-ONLY; cannot be changed later.",
|
|
349
|
+
"enum": [
|
|
350
|
+
"request",
|
|
351
|
+
"trace"
|
|
352
|
+
],
|
|
353
|
+
"default": "request"
|
|
354
|
+
},
|
|
355
|
+
{
|
|
356
|
+
"name": "population",
|
|
357
|
+
"type": "string",
|
|
358
|
+
"description": "Request tag this criterion judges online AND calibrates against (one binding). Max 64 chars; \"\" = all traffic."
|
|
359
|
+
},
|
|
360
|
+
{
|
|
361
|
+
"name": "population_family",
|
|
362
|
+
"type": "string",
|
|
363
|
+
"description": "Auto-detected traffic segment (a `family` value from GET /v1/logs facets, 16 hex chars or \"none\") scoping the same binding. Max 32 chars; \"\" = no segment scope."
|
|
364
|
+
}
|
|
365
|
+
],
|
|
366
|
+
"responseSummary": "201 with the criterion object (same shape as list items): id, name, description, judge_prompt, judge_model, status, source, unit, population, population_family, online_* fields, tier, trust, *_ci, drift_*, tpr, tnr, kappa, alignment_n, aligned_at, created_at",
|
|
367
|
+
"notes": "OWNER/ADMIN only (403). Creating does not spend; judging (align, online monitoring) does. Body keys are snake_case exactly as listed; other criterion knobs (coverage, pre-checks, contract rules) are not settable through this endpoint."
|
|
368
|
+
},
|
|
369
|
+
{
|
|
370
|
+
"name": "suggest_criteria",
|
|
371
|
+
"method": "POST",
|
|
372
|
+
"path": "/v1/criteria/suggest",
|
|
373
|
+
"summary": "Clusters the workspace's written failure critiques into up to 5 DRAFT judge criteria, one per failure mode — use it after grading a batch of fails with reasons to bootstrap criteria you then review and align.",
|
|
374
|
+
"scope": "evals:write",
|
|
375
|
+
"body": [
|
|
376
|
+
{
|
|
377
|
+
"name": "judge_model",
|
|
378
|
+
"type": "string",
|
|
379
|
+
"description": "Model used for the single clustering call (and set as judge_model on every draft). Defaults to the platform's recommended judge (Qwen/Qwen3-235B-A22B-Instruct-2507). Must be a model available in the workspace's playground catalog, else 400. Whitespace-only values fall back to the default."
|
|
380
|
+
}
|
|
381
|
+
],
|
|
382
|
+
"responseSummary": "{ created: [<criterion objects, same snake_case shape as GET /v1/criteria: id, name, description, judge_prompt, judge_model, status ('draft'), source ('assist_suggested'), unit, population, population_family, online_*, tier, trust, fail_grades_needed, pass_grades_needed, tpr_ci, tnr_ci, drift_*, tpr, tnr, kappa, alignment_n, aligned_at, created_at>], critiques_used: <int>, skipped_duplicates: <int, proposed drafts whose name already existed> }.",
|
|
383
|
+
"notes": "A body-less POST (or invalid JSON) is valid and uses the default judge — there is no 400 for a missing body. Requires an OWNER/ADMIN minting user (403). 400 when fewer than 10 FAIL grades carry a critique (message includes the current count); only the 200 most recent critiques are considered. 400 if the model returns no parseable JSON array ('try again'). SPENDS THE WALLET: one metered clustering call (billed under assist:suggest). Drafts are never trusted by any gate until a human reviews them and runs an alignment; an existing criterion with the same name is skipped, never overwritten. Function maxDuration is 300s.",
|
|
384
|
+
"spends": true
|
|
385
|
+
},
|
|
386
|
+
{
|
|
387
|
+
"name": "list_criterion_templates",
|
|
388
|
+
"method": "GET",
|
|
389
|
+
"path": "/v1/criteria/templates",
|
|
390
|
+
"summary": "Lists the shipped judge-criterion templates (starting-point judge prompts grouped by use case) so a caller can instantiate one via POST /v1/criteria with an edited judge_prompt.",
|
|
391
|
+
"scope": "read",
|
|
392
|
+
"responseSummary": "{ object: 'list', data: [{ id: <slug e.g. 'no-fabrication', 'grounded-in-context', 'right-next-action', 'tool-use-sound', 'finishes-what-it-starts'>, use_case: <'Support & assistants'|'RAG & knowledge'|'Extraction & structured output'|'Data processing'|'Agents & tools'|'Any traffic'>, name, description, judge_prompt, unit: 'request'|'trace', universal: <bool, true = meaningful on any traffic, safe to leave unscoped> }] }. 11 templates as of this build.",
|
|
393
|
+
"notes": "Static and free; only authentication is required. A template is a starting point, not a truth — it still has to be aligned against the workspace's own labels. Task-specific (non-universal) templates should be scoped to the tag of the traffic they judge; unit 'trace' templates judge whole agent runs and need trace-scoped labels."
|
|
394
|
+
},
|
|
395
|
+
{
|
|
396
|
+
"name": "get_criterion",
|
|
397
|
+
"method": "GET",
|
|
398
|
+
"path": "/v1/criteria/{id}",
|
|
399
|
+
"summary": "Fetch one criterion with its current calibration metrics, trust verdict and the intervals it was derived from.",
|
|
400
|
+
"scope": "read",
|
|
401
|
+
"pathParams": [
|
|
402
|
+
{
|
|
403
|
+
"name": "id",
|
|
404
|
+
"type": "string",
|
|
405
|
+
"description": "Criterion id."
|
|
406
|
+
}
|
|
407
|
+
],
|
|
408
|
+
"responseSummary": "The criterion object: id, name, description, judge_prompt, judge_model, status, source, unit, population, population_family, online_enabled, online_percent, online_cap_usd, online_spent_usd, tier, trust, fail_grades_needed, pass_grades_needed, tpr_ci, tnr_ci, drift_status, drift_signal, drift_reason, drift_checked_at, tpr, tnr, kappa, alignment_n, aligned_at, created_at",
|
|
409
|
+
"notes": "404 \"Criterion not found\" when the id is not in this workspace. Derivations (trust, drift) match GET /v1/criteria exactly."
|
|
410
|
+
},
|
|
411
|
+
{
|
|
412
|
+
"name": "update_criterion",
|
|
413
|
+
"method": "PATCH",
|
|
414
|
+
"path": "/v1/criteria/{id}",
|
|
415
|
+
"summary": "Update a criterion's prompt, judge model, population scope, online-monitoring settings or lifecycle status; instrument changes void its calibration.",
|
|
416
|
+
"scope": "evals:write",
|
|
417
|
+
"pathParams": [
|
|
418
|
+
{
|
|
419
|
+
"name": "id",
|
|
420
|
+
"type": "string",
|
|
421
|
+
"description": "Criterion id."
|
|
422
|
+
}
|
|
423
|
+
],
|
|
424
|
+
"body": [
|
|
425
|
+
{
|
|
426
|
+
"name": "name",
|
|
427
|
+
"type": "string",
|
|
428
|
+
"description": "Trimmed, 1..80 chars."
|
|
429
|
+
},
|
|
430
|
+
{
|
|
431
|
+
"name": "description",
|
|
432
|
+
"type": "string",
|
|
433
|
+
"description": "Max 500 chars; null clears."
|
|
434
|
+
},
|
|
435
|
+
{
|
|
436
|
+
"name": "judge_prompt",
|
|
437
|
+
"type": "string",
|
|
438
|
+
"description": "Trimmed, 10..4000 chars. Changing it VOIDS tpr/tnr/kappa/aligned_at and deletes stored confusion rows."
|
|
439
|
+
},
|
|
440
|
+
{
|
|
441
|
+
"name": "judge_model",
|
|
442
|
+
"type": "string",
|
|
443
|
+
"description": "Model id. Changing it voids calibration."
|
|
444
|
+
},
|
|
445
|
+
{
|
|
446
|
+
"name": "unit",
|
|
447
|
+
"type": "string",
|
|
448
|
+
"description": "Accepted only if equal to the current unit; any change is refused with 400 (unit is create-only).",
|
|
449
|
+
"enum": [
|
|
450
|
+
"request",
|
|
451
|
+
"trace"
|
|
452
|
+
]
|
|
453
|
+
},
|
|
454
|
+
{
|
|
455
|
+
"name": "population",
|
|
456
|
+
"type": "string",
|
|
457
|
+
"description": "Request tag scope, max 64 chars; \"\" = all traffic. Changing it voids calibration."
|
|
458
|
+
},
|
|
459
|
+
{
|
|
460
|
+
"name": "population_family",
|
|
461
|
+
"type": "string",
|
|
462
|
+
"description": "Traffic-segment scope (family value from logs facets), max 32 chars; \"\" clears. Changing it voids calibration."
|
|
463
|
+
},
|
|
464
|
+
{
|
|
465
|
+
"name": "online_enabled",
|
|
466
|
+
"type": "boolean",
|
|
467
|
+
"description": "Turn online monitoring on/off. When on, the judge scores a sample of fresh logged traffic and each judge call is billed as usage."
|
|
468
|
+
},
|
|
469
|
+
{
|
|
470
|
+
"name": "online_percent",
|
|
471
|
+
"type": "integer",
|
|
472
|
+
"description": "Percent of fresh in-scope traffic to judge, integer 1..100."
|
|
473
|
+
},
|
|
474
|
+
{
|
|
475
|
+
"name": "online_cap_usd",
|
|
476
|
+
"type": "number",
|
|
477
|
+
"description": "Weekly online-judging spend ceiling in USD, 0..100000; 0 = uncapped. Money config; does not void calibration."
|
|
478
|
+
},
|
|
479
|
+
{
|
|
480
|
+
"name": "status",
|
|
481
|
+
"type": "string",
|
|
482
|
+
"description": "Lifecycle. \"retired\" removes the criterion from online scoring and pickers.",
|
|
483
|
+
"enum": [
|
|
484
|
+
"draft",
|
|
485
|
+
"retired"
|
|
486
|
+
]
|
|
487
|
+
}
|
|
488
|
+
],
|
|
489
|
+
"responseSummary": "200 with the updated criterion object (same shape as GET /v1/criteria/{id})",
|
|
490
|
+
"notes": "OWNER/ADMIN only (403). 404 \"Criterion not found\". 400 for schema failures, a unit change, or an invalid status. MONEY: enabling online monitoring spends the wallet on judge calls, capped weekly by online_cap_usd. Voided calibration means trust becomes \"unmeasured\" until POST /align is run again."
|
|
491
|
+
},
|
|
492
|
+
{
|
|
493
|
+
"name": "delete_criterion",
|
|
494
|
+
"method": "DELETE",
|
|
495
|
+
"path": "/v1/criteria/{id}",
|
|
496
|
+
"summary": "Permanently delete a criterion and its calibration history.",
|
|
497
|
+
"scope": "evals:write",
|
|
498
|
+
"pathParams": [
|
|
499
|
+
{
|
|
500
|
+
"name": "id",
|
|
501
|
+
"type": "string",
|
|
502
|
+
"description": "Criterion id."
|
|
503
|
+
}
|
|
504
|
+
],
|
|
505
|
+
"responseSummary": "200 { id, deleted: true }",
|
|
506
|
+
"notes": "OWNER/ADMIN only (403). 404 \"Criterion not found\" when not in this workspace. Aliases gated on this criterion lose their gate."
|
|
507
|
+
},
|
|
508
|
+
{
|
|
509
|
+
"name": "run_criterion_alignment",
|
|
510
|
+
"method": "POST",
|
|
511
|
+
"path": "/v1/criteria/{id}/align",
|
|
512
|
+
"summary": "Calibrate a criterion by re-judging every in-scope human-labeled trace and measuring agreement (TPR/TNR with Wilson intervals, Cohen's kappa), which is what earns a judge the trust needed to gate on it.",
|
|
513
|
+
"scope": "evals:write",
|
|
514
|
+
"pathParams": [
|
|
515
|
+
{
|
|
516
|
+
"name": "id",
|
|
517
|
+
"type": "string",
|
|
518
|
+
"description": "Criterion id."
|
|
519
|
+
}
|
|
520
|
+
],
|
|
521
|
+
"responseSummary": "Small sets (<= 50 labels) run synchronously and return the report: { scope_tag, scope_family, tag_breakdown: [{ tag, n }], mixed_population, excluded_other_cause, unattributed_fails, metrics: { n, tpr, tpr_ci, tnr, tnr_ci, kappa }, tier, thin_alignment_set, skipped, holdout: { tune_n, report_n }|null, one_class_note, disagreements: [{ request_id, judge_verdict, human_verdict }] }. Larger sets return { queued: true, total_labels } and the report is built in the background over the following minutes (poll GET /v1/criteria/{id}/alignment).",
|
|
522
|
+
"notes": "MONEY: spends the wallet like any judging (one judge call per label; a new run is new spend). OWNER/ADMIN only (403). 400 when fewer than 30 in-scope labels exist (message says how many you have and how to label more), when a run is already in progress (\"An alignment run is already in progress for this criterion.\"), or when fewer than 30 labels could actually be judged. 404 \"Criterion not found\". Route maxDuration is 300s.",
|
|
523
|
+
"spends": true
|
|
524
|
+
},
|
|
525
|
+
{
|
|
526
|
+
"name": "get_criterion_alignment",
|
|
527
|
+
"method": "GET",
|
|
528
|
+
"path": "/v1/criteria/{id}/alignment",
|
|
529
|
+
"summary": "Read the persistent report from the criterion's last calibration run: metrics with intervals, population breakdown, threshold sweep, and every judge/human disagreement with the human's critique and a response excerpt. Free; it never re-judges.",
|
|
530
|
+
"scope": "read",
|
|
531
|
+
"pathParams": [
|
|
532
|
+
{
|
|
533
|
+
"name": "id",
|
|
534
|
+
"type": "string",
|
|
535
|
+
"description": "Criterion id."
|
|
536
|
+
}
|
|
537
|
+
],
|
|
538
|
+
"responseSummary": "{ aligned_at, scope_tag, tag_breakdown: [{ tag, n }], mixed_population, tier, thin_alignment_set, metrics: { n, tpr, tpr_ci, tnr, tnr_ci, kappa }, threshold_analysis: { half (\"tune\"|\"all\"), sweep: { n, ungraded, argmax: metrics, best: { threshold, metrics, youden_j }|null, curve: [{ threshold, metrics, youden_j }], note }, report_check: { threshold, n, metrics, youden_j, argmax }|null }|null, agreements (count), disagreements: [{ request_id, judge_verdict, human_verdict, critique, tag, response_excerpt }] }",
|
|
539
|
+
"notes": "404 \"Criterion not found\". 400 \"This criterion has no alignment run yet\" when never calibrated or after a voiding edit; 400 \"Alignment run in progress — N labels judged so far\" while a background run is in flight (use it to poll). threshold_analysis is null for runs recorded before logprob grades were stored. All nested keys are snake_case."
|
|
540
|
+
},
|
|
541
|
+
{
|
|
542
|
+
"name": "auto_improve_criterion",
|
|
543
|
+
"method": "POST",
|
|
544
|
+
"path": "/v1/criteria/{id}/auto_improve",
|
|
545
|
+
"summary": "Runs one auto-improvement round on a judge criterion: mines the tune-half disagreements from its last alignment, rewrites the judge prompt coherently, and creates a successor DRAFT criterion with its alignment queued — use it when a calibrated judge still disagrees with your grades and you want a better candidate without hand-editing the prompt.",
|
|
546
|
+
"scope": "evals:write",
|
|
547
|
+
"pathParams": [
|
|
548
|
+
{
|
|
549
|
+
"name": "id",
|
|
550
|
+
"type": "string",
|
|
551
|
+
"description": "Criterion id (must belong to the key's workspace)."
|
|
552
|
+
}
|
|
553
|
+
],
|
|
554
|
+
"responseSummary": "201 with { criterion: <full criterion object, snake_case: id, name, description, judge_prompt, judge_model, status, source, unit, population, population_family, online_enabled, online_percent, online_cap_usd, online_spent_usd, tier, trust, fail_grades_needed, pass_grades_needed, tpr_ci, tnr_ci, drift_status, drift_signal, drift_reason, drift_checked_at, tpr, tnr, kappa, alignment_n, aligned_at, created_at>, tune_disagreements: <int, tune-half rows where judge and human disagreed>, alignment_queued: <bool> }. The returned criterion is the NEW successor (draft, metrics void), named '<parent name> (auto r2)' (round suffix increments), inheriting the parent's unit, judge model, tag and segment.",
|
|
555
|
+
"notes": "No request body is read. Requires an OWNER/ADMIN minting user (403 otherwise). 404 if the criterion is not in the workspace. 400 when: an alignment run is in progress; the last alignment has fewer than 80 judged rows (needs a holdout-scale run so the report half stays untouched); there are zero tune-half disagreements; or the rewriter returned an unusable prompt (nothing created, only the single rewriter call was spent). SPENDS THE WALLET: one metered rewriter call (billed under assist:iterate) plus the queued alignment run, which bills like any alignment. Deliberately single-round: loop it yourself once the successor's alignment lands; adoption (repoint monitoring, retire the parent) stays a human act. Function maxDuration is 300s.",
|
|
556
|
+
"spends": true
|
|
557
|
+
},
|
|
558
|
+
{
|
|
559
|
+
"name": "get_criterion_certificate",
|
|
560
|
+
"method": "GET",
|
|
561
|
+
"path": "/v1/criteria/{id}/certificate",
|
|
562
|
+
"summary": "Returns the signed judge certificate for a criterion — what was proven (confusion matrix, TPR/TNR/kappa with intervals, trust verdict), on which population, what voids it, and what enforcement refused involving it — for audits, evidence bundles, or proving a judge's calibration to a third party.",
|
|
563
|
+
"scope": "read",
|
|
564
|
+
"pathParams": [
|
|
565
|
+
{
|
|
566
|
+
"name": "id",
|
|
567
|
+
"type": "string",
|
|
568
|
+
"description": "Criterion id (must belong to the key's workspace)."
|
|
569
|
+
}
|
|
570
|
+
],
|
|
571
|
+
"responseSummary": "A JSON document whose keys are camelCase (NOT snake_case — it is emitted verbatim so its signature can be re-derived): signature: {alg:'HS256', key_id, value} | null (with unsigned: true when no signing secret is configured), criterionId, name, question (the judge prompt), unit ('request'|'trace'), judgeModel, issuedAt, calibration: {measured, matrix: {tp,fp,tn,fn}|null, metrics: {n, tpr, tprCi, tnr, tnrCi, kappa}|null, labels, holdoutActive (labels >= 80), alignedAt, goldenSet: {id, name, size, membershipHash, frozenAt, humanKappa, humanAgreement, humanKappaN, raterCount}|null}, trust: {trust: 'trustworthy'|'misaligned'|'under-measured'|'borderline'|'unmeasured', failGradesNeeded, passGradesNeeded, tprCi, tnrCi} (or just {trust:'unmeasured'}), population: {tag, segment, unit, statement}, validity: {driftStatus: 'ok'|'flagged', driftReason, driftCheckedAt, voidedBy: string[]}, enforcement: {windowDays: 90, refusalsInvolvingJudge, lastReason}.",
|
|
572
|
+
"notes": "Free (no judging). 404 if the criterion is not in the workspace. Response is Cache-Control: no-store. Hand the WHOLE JSON object to POST /v1/verify to check the signature later. An uncalibrated judge still returns a certificate that honestly says nothing is measured (calibration.measured=false, trust.trust='unmeasured'). enforcement counts refusal-ledger rows from the last 90 days whose subject is this criterion or whose reason names it."
|
|
573
|
+
},
|
|
574
|
+
{
|
|
575
|
+
"name": "scan_criterion_suspects",
|
|
576
|
+
"method": "POST",
|
|
577
|
+
"path": "/v1/criteria/{id}/scan",
|
|
578
|
+
"summary": "Judges a bounded batch of recent, not-yet-labeled traffic with this criterion and queues every FAIL as a pending suspect for human review in the dashboard's Review queue — the fastest way to grow a judge's failure-label set from live traffic.",
|
|
579
|
+
"scope": "evals:write",
|
|
580
|
+
"pathParams": [
|
|
581
|
+
{
|
|
582
|
+
"name": "id",
|
|
583
|
+
"type": "string",
|
|
584
|
+
"description": "Criterion id (must belong to the key's workspace)."
|
|
585
|
+
}
|
|
586
|
+
],
|
|
587
|
+
"responseSummary": "{ scanned: <int, items actually judged>, flagged: <int, FAIL verdicts queued as pending suspects> }. Both are 0 when no unlabeled, unscanned candidates exist in scope.",
|
|
588
|
+
"notes": "No request body is read. Requires an OWNER/ADMIN minting user (403). 404 if the criterion is not in the workspace. Request-unit criteria: takes the 100 most recent logged exchanges (scoped to the criterion's population segment when it has one), drops already-labeled and already-scanned rows (dismissed suspects never resurface), and judges at most 30. Trace-unit criteria: scans at most 10 COMPLETED agent runs from the last 7 days (quiet for 10 minutes), scoped to the criterion's tag and segment; requires a completed calibration (400 'Calibrate this judge first' otherwise) and has a pre-flight wallet gate of ~$0.10 per run (402 'Insufficient balance' before any spend). SPENDS THE WALLET: every judge call is metered as usage (billing prefix scan:). Suspects are adjudicated in the dashboard (accept = a real FAIL label; dismiss = never resurfaces). Function maxDuration is 300s.",
|
|
589
|
+
"spends": true
|
|
590
|
+
},
|
|
591
|
+
{
|
|
592
|
+
"name": "decontaminate_texts",
|
|
593
|
+
"method": "POST",
|
|
594
|
+
"path": "/v1/datasets/decontaminate",
|
|
595
|
+
"summary": "Checks a batch of texts against the public-benchmark contamination index (13-word shingles of well-known test splits) and reports which inputs share material with which benchmark — use it before training so later benchmark scores measure capability, not memorised answer keys.",
|
|
596
|
+
"scope": "evals:write",
|
|
597
|
+
"body": [
|
|
598
|
+
{
|
|
599
|
+
"name": "texts",
|
|
600
|
+
"type": "array",
|
|
601
|
+
"description": "Array of strings to check (each item must be a string). At most 5,000 per call. Positions in this array are the `index` values in the response.",
|
|
602
|
+
"required": true,
|
|
603
|
+
"items": "string"
|
|
604
|
+
}
|
|
605
|
+
],
|
|
606
|
+
"responseSummary": "{ checked: <int, texts actually checked; 0 when the index was unavailable>, index: { version, generatedAt (camelCase — passed through verbatim), benchmarks: [{id, name, rows}] } | null, hits: [{ index: <position in texts>, benchmark: <display name>, benchmark_id, matches: <shared shingle count> }], contaminated: <sorted unique int[] of input positions with any hit> }.",
|
|
607
|
+
"notes": "400 on invalid JSON, when `texts` is not an array of strings, or when it exceeds 5,000 items. `index: null` means the benchmark index was unavailable and NOTHING was checked — never treat that as clean. 'Clean' is a claim about THOSE benchmarks on THAT index date only. Free (no judging, no wallet spend). Response is Cache-Control: no-store. One text can produce multiple hits (one per benchmark it overlaps)."
|
|
608
|
+
},
|
|
609
|
+
{
|
|
610
|
+
"name": "create_dataset_from_logs",
|
|
611
|
+
"method": "POST",
|
|
612
|
+
"path": "/v1/datasets/from_logs",
|
|
613
|
+
"summary": "Curates logged gateway traffic into a managed training dataset (optionally with a disjoint eval holdout split), auto-dropping errored/truncated/empty/duplicate/human-failed/benchmark-contaminated exchanges and applying a chosen quality gate — use it to turn production logs into fine-tuning or eval data.",
|
|
614
|
+
"scope": "evals:write",
|
|
615
|
+
"body": [
|
|
616
|
+
{
|
|
617
|
+
"name": "name",
|
|
618
|
+
"type": "string",
|
|
619
|
+
"description": "Dataset name, 1-80 chars after trimming. Also used as the file name ('<name>.jsonl') and, with holdout_pct, the eval split is named '<name>-eval'.",
|
|
620
|
+
"required": true
|
|
621
|
+
},
|
|
622
|
+
{
|
|
623
|
+
"name": "holdout_pct",
|
|
624
|
+
"type": "number",
|
|
625
|
+
"description": "Percentage (0-50) of curated lines carved into a second, DISJOINT '<name>-eval' dataset linked back to the training set. 0/omitted = no eval split. Values above 50 are capped at 50."
|
|
626
|
+
},
|
|
627
|
+
{
|
|
628
|
+
"name": "decontaminate",
|
|
629
|
+
"type": "boolean",
|
|
630
|
+
"description": "Drop rows whose prompt shares a 13-word shingle with a public benchmark test split. Default true. Only an explicit boolean is honored. The provenance records what was checked; an unavailable index is recorded as 'not checked', never as clean."
|
|
631
|
+
},
|
|
632
|
+
{
|
|
633
|
+
"name": "filters",
|
|
634
|
+
"type": "object",
|
|
635
|
+
"description": "Which logged traffic feeds the build. Nested keys: model (string, exact model name), tag (string, the task label sent as X-Omnia-Tag), segment (string, an auto-detected traffic segment / prompt family as shown on GET /v1/logs rows), finish_reason (string), cache_hit (boolean), start (integer unix seconds, inclusive lower bound), end (integer unix seconds). Falsy values (empty string, 0) are ignored. Success-only is always enforced regardless of filters."
|
|
636
|
+
},
|
|
637
|
+
{
|
|
638
|
+
"name": "sources",
|
|
639
|
+
"type": "object",
|
|
640
|
+
"description": "Include-list of source models: { models: string[] }. Only exchanges served by these models feed the build; an empty array means no restriction. Each item must be a non-empty string, else 400 'sources.models must be an array of model names'."
|
|
641
|
+
},
|
|
642
|
+
{
|
|
643
|
+
"name": "quality",
|
|
644
|
+
"type": "object",
|
|
645
|
+
"description": "The quality ladder: { mode: 'cleaned'|'graded'|'judge', criterion_id?: string }. 'cleaned' (default) = mechanical curation only. 'graded' = keep only exchanges a human graded pass (free). 'judge' = a CALIBRATED judge (criterion_id REQUIRED; request-unit; trust 'trustworthy' or 'under-measured'; not drift-flagged; if segment-bound, filters.segment must equal its segment) keeps only passing conversations, judged at each conversation's terminal turn; human grades override the judge for free. mode must be a string, criterion_id a string when present.",
|
|
646
|
+
"enum": [
|
|
647
|
+
"cleaned",
|
|
648
|
+
"graded",
|
|
649
|
+
"judge"
|
|
650
|
+
]
|
|
651
|
+
}
|
|
652
|
+
],
|
|
653
|
+
"responseSummary": "201 with snake_case: { summary: { total: <rows fetched>, kept, dropped: { unparseable, errored, truncated, empty, duplicate, human_failed, contaminated }, folded: { folded_turns, conversations } }, quality: { mode, criterion_id?, criterion_name?, kappa?, human_pass_kept?, ungraded_excluded?, judged?, judge_passed?, judge_failed?, judge_unparsed?, judge_spend_usd? }, training_name, training_count, eval_name?: '<name>-eval', eval_count? }.",
|
|
654
|
+
"notes": "Requires an OWNER/ADMIN minting user for dataset creation (403, enforced in the dataset service). 400 (invalid_json) on unparseable JSON. Other 400s: name missing/over 80 chars; unknown quality mode; judge mode without criterion_id; judge criterion not found / trace-unit / misaligned / borderline / unmeasured / drift-flagged / segment-bound but build not scoped to that segment (each eligibility refusal is also written to the refusal ledger, kind dataset_judge_trust); no usable exchanges after curation (nothing created). Fetch is capped at 50,000 most recent matching rows. MONEY: judge mode is gated up front at ~$0.02 per conversation to judge (402 'Insufficient balance for judge gating' before any spend) and every judge call is then metered as usage; a scoring failure mid-run FAILS THE WHOLE BUILD (no dataset created) but rows already judged were billed (idempotent ids — retry does not re-bill). Human FAIL grades (or rows sharing an agent run with a trace-scoped FAIL) never enter a dataset in any mode. Multi-turn chats are folded into one weighted line per conversation. If the training set is created but the eval split fails, the response is a 400 that says the training dataset already exists. The eval split shares no example with the training set, so it is valid as an eval source (POST /v1/evals with sample_filters.dataset_id)."
|
|
655
|
+
},
|
|
656
|
+
{
|
|
657
|
+
"name": "list_dedicated_endpoints",
|
|
658
|
+
"method": "GET",
|
|
659
|
+
"path": "/v1/dedicated",
|
|
660
|
+
"summary": "Lists the workspace's dedicated (reserved-GPU) inference endpoints with live-reconciled status, frozen hourly price and unbilled cost accrued since the last meter — use it to monitor what is running and what it is costing.",
|
|
661
|
+
"scope": "read",
|
|
662
|
+
"responseSummary": "A bare JSON array (no {object:'list'} envelope) of endpoint objects, snake_case: { id, name, description, model_name, flavor_name, gpu_type, gpu_count, region, min_replicas, max_replicas, status (e.g. PENDING/STARTING/RUNNING/UPDATING/STOPPING/STOPPED/FAILED), enabled, hourly_rate_usd (customer sell price per GPU-hour, frozen at deploy), pending_cost_usd (GPU-hours accrued since last_metered_at while RUNNING, not yet billed), routing_key (the model name to send to the inference API to hit this endpoint), base_url, last_metered_at, created_at }. Internal margin fields are never returned.",
|
|
663
|
+
"notes": "Scope via requiredScopeFor is 'read' for GET; NOTE the dedicated routes use their own local apiKeyActor (app/api/v1/dedicated/_helpers.ts) which authenticates the key but does NOT enforce key scopes — any valid, unrevoked key passes. Deleted endpoints are excluded. Status/enabled/region are reconciled live from the control plane on every call (DB state served if reconcile fails). 400 on catalog/provider failure. Money: a RUNNING endpoint bills per GPU-hour (gpu_count x replicas x hourly_rate_usd) continuously; pending_cost_usd is what the next meter will charge."
|
|
664
|
+
},
|
|
665
|
+
{
|
|
666
|
+
"name": "create_dedicated_endpoint",
|
|
667
|
+
"method": "POST",
|
|
668
|
+
"path": "/v1/dedicated",
|
|
669
|
+
"summary": "Provisions a new dedicated inference endpoint (a model served on reserved GPUs at a frozen per-GPU-hour price) — use it for guaranteed capacity, custom fine-tuned weights, or predictable latency; billing starts as soon as it is running.",
|
|
670
|
+
"scope": "platform:write",
|
|
671
|
+
"body": [
|
|
672
|
+
{
|
|
673
|
+
"name": "name",
|
|
674
|
+
"type": "string",
|
|
675
|
+
"description": "Display name (trimmed, non-empty).",
|
|
676
|
+
"required": true
|
|
677
|
+
},
|
|
678
|
+
{
|
|
679
|
+
"name": "description",
|
|
680
|
+
"type": "string",
|
|
681
|
+
"description": "Optional description (trimmed)."
|
|
682
|
+
},
|
|
683
|
+
{
|
|
684
|
+
"name": "model_name",
|
|
685
|
+
"type": "string",
|
|
686
|
+
"description": "A template `name` from GET /v1/dedicated/templates. camelCase alias modelName also accepted (camelCase wins if both present).",
|
|
687
|
+
"required": true
|
|
688
|
+
},
|
|
689
|
+
{
|
|
690
|
+
"name": "flavor_name",
|
|
691
|
+
"type": "string",
|
|
692
|
+
"description": "A key of that template's `flavors` map. Alias: flavorName.",
|
|
693
|
+
"required": true
|
|
694
|
+
},
|
|
695
|
+
{
|
|
696
|
+
"name": "gpu_type",
|
|
697
|
+
"type": "string",
|
|
698
|
+
"description": "A key of the flavor's available_configurations.gpu_configurations. Alias: gpuType.",
|
|
699
|
+
"required": true
|
|
700
|
+
},
|
|
701
|
+
{
|
|
702
|
+
"name": "gpu_count",
|
|
703
|
+
"type": "integer",
|
|
704
|
+
"description": "Must be > 0 and in the GPU configuration's allowed_gpu_counts. Alias: gpuCount.",
|
|
705
|
+
"required": true
|
|
706
|
+
},
|
|
707
|
+
{
|
|
708
|
+
"name": "region",
|
|
709
|
+
"type": "string",
|
|
710
|
+
"description": "Must be in the GPU configuration's allowed_regions.",
|
|
711
|
+
"required": true
|
|
712
|
+
},
|
|
713
|
+
{
|
|
714
|
+
"name": "min_replicas",
|
|
715
|
+
"type": "integer",
|
|
716
|
+
"description": ">= 1. Alias: minReplicas. Sizes the prepay/wallet gate (min_replicas x gpu_count x hourly price x prepay hours).",
|
|
717
|
+
"required": true
|
|
718
|
+
},
|
|
719
|
+
{
|
|
720
|
+
"name": "max_replicas",
|
|
721
|
+
"type": "integer",
|
|
722
|
+
"description": ">= min_replicas and <= the configuration's max_replicas_allowed. Alias: maxReplicas.",
|
|
723
|
+
"required": true
|
|
724
|
+
},
|
|
725
|
+
{
|
|
726
|
+
"name": "custom_weights_id",
|
|
727
|
+
"type": "string",
|
|
728
|
+
"description": "Serve a fine-tuned model's merged weights: must start with 'model-artifact_' (the artifact id from a completed fine-tune), else 400. Alias: customWeightsId. Omit for stock base models."
|
|
729
|
+
},
|
|
730
|
+
{
|
|
731
|
+
"name": "fine_tuning_job_id",
|
|
732
|
+
"type": "string",
|
|
733
|
+
"description": "The source fine-tuning job to record on the endpoint, when deployed from one. Alias: fineTuningJobId."
|
|
734
|
+
}
|
|
735
|
+
],
|
|
736
|
+
"responseSummary": "201 with { id: <endpoint id> }. Poll GET /v1/dedicated/{id} for status and routing_key.",
|
|
737
|
+
"notes": "400 'Invalid JSON body' or 'Missing required field(s): ...' when any of name, model_name, flavor_name, gpu_type, gpu_count, region, min_replicas, max_replicas is absent/null (checked after alias lifting). Requires an OWNER/ADMIN minting user (403). 400 when the model/flavor/GPU/region/count combo is not in the catalog, replica range invalid, or no price is configured for the GPU/region. MONEY: 402 'Insufficient balance' unless the wallet covers at least DEDICATED_PREPAY_HOURS (default 1 hour) of runway at min_replicas x gpu_count x sell rate; the per-GPU-hour price is FROZEN on the endpoint at create time; GPU-hours are metered continuously while the endpoint is enabled and RUNNING — stop (PATCH enabled=false) or DELETE to stop billing. Scope note: the dedicated routes' local apiKeyActor does not enforce key scopes on this branch.",
|
|
738
|
+
"spends": true
|
|
739
|
+
},
|
|
740
|
+
{
|
|
741
|
+
"name": "list_dedicated_templates",
|
|
742
|
+
"method": "GET",
|
|
743
|
+
"path": "/v1/dedicated/templates",
|
|
744
|
+
"summary": "Returns the deployable model catalog for dedicated endpoints (model -> flavor -> GPU type -> allowed regions/counts/replica limits) plus this workspace's sell price per GPU-hour for every GPU/region combo — read it to build a valid POST /v1/dedicated request and estimate cost.",
|
|
745
|
+
"scope": "read",
|
|
746
|
+
"responseSummary": "{ templates: [{ name (use as model_name), type ('text2text'|'embedding'|'image2text'|...), metadata?: { huggingface_url?, vendor?, context_window_k?, size_b?, license?: {url?, name?} }, flavors?: { <flavor_name>: { quantization?, use_cases?, tags?, base_model_slug?, available_configurations?: { gpu_configurations?: { <gpu_type>: { allowed_regions: string[], allowed_gpu_counts: int[], max_replicas_allowed: int } } } } } }], prices: [{ gpu_type, region, price_per_gpu_hour_usd: number|null }] }. Template contents are the upstream catalog shape, already snake_case.",
|
|
747
|
+
"notes": "Scope via requiredScopeFor is 'read'; the local dedicated apiKeyActor does not enforce scopes. price_per_gpu_hour_usd is the customer price (base cost and margin are never returned); null means no price is configured for that combo yet and a deploy on it will be refused. 400 'Dedicated endpoints are not configured' or 'Failed to load dedicated endpoint catalog' on provider/config failure. Prices are quoted at request time; the price frozen on an endpoint is the one in effect when it is created. Free to call."
|
|
748
|
+
},
|
|
749
|
+
{
|
|
750
|
+
"name": "get_dedicated_endpoint",
|
|
751
|
+
"method": "GET",
|
|
752
|
+
"path": "/v1/dedicated/{id}",
|
|
753
|
+
"summary": "Returns one dedicated endpoint's current view (live status, frozen hourly price, unbilled accrued cost, routing key) — use it to poll a deploy until RUNNING or to check spend.",
|
|
754
|
+
"scope": "read",
|
|
755
|
+
"pathParams": [
|
|
756
|
+
{
|
|
757
|
+
"name": "id",
|
|
758
|
+
"type": "string",
|
|
759
|
+
"description": "Dedicated endpoint id (must belong to the key's workspace; deleted endpoints 404)."
|
|
760
|
+
}
|
|
761
|
+
],
|
|
762
|
+
"responseSummary": "A single endpoint object, snake_case: { id, name, description, model_name, flavor_name, gpu_type, gpu_count, region, min_replicas, max_replicas, status, enabled, hourly_rate_usd, pending_cost_usd, routing_key, base_url, last_metered_at, created_at }.",
|
|
763
|
+
"notes": "Scope via requiredScopeFor is 'read'; the local dedicated apiKeyActor does not enforce scopes. Implemented by listing the workspace's endpoints (live-reconciled) and picking the id, so it costs a full list call. 404 'Endpoint not found'."
|
|
764
|
+
},
|
|
765
|
+
{
|
|
766
|
+
"name": "update_dedicated_endpoint",
|
|
767
|
+
"method": "PATCH",
|
|
768
|
+
"path": "/v1/dedicated/{id}",
|
|
769
|
+
"summary": "Scales, starts/stops, renames, or changes the GPU configuration of a dedicated endpoint — use enabled=false to stop billing without deleting, or gpu_type/gpu_count to re-size (which re-freezes the price).",
|
|
770
|
+
"scope": "platform:write",
|
|
771
|
+
"pathParams": [
|
|
772
|
+
{
|
|
773
|
+
"name": "id",
|
|
774
|
+
"type": "string",
|
|
775
|
+
"description": "Dedicated endpoint id (must belong to the key's workspace)."
|
|
776
|
+
}
|
|
777
|
+
],
|
|
778
|
+
"body": [
|
|
779
|
+
{
|
|
780
|
+
"name": "name",
|
|
781
|
+
"type": "string",
|
|
782
|
+
"description": "New display name (trimmed)."
|
|
783
|
+
},
|
|
784
|
+
{
|
|
785
|
+
"name": "description",
|
|
786
|
+
"type": "string",
|
|
787
|
+
"description": "New description (trimmed)."
|
|
788
|
+
},
|
|
789
|
+
{
|
|
790
|
+
"name": "enabled",
|
|
791
|
+
"type": "boolean",
|
|
792
|
+
"description": "false = STOP the endpoint (runs a final meter for accrued GPU-hours, status STOPPING); true = START it (status STARTING, billing resumes when RUNNING). Omit to leave unchanged."
|
|
793
|
+
},
|
|
794
|
+
{
|
|
795
|
+
"name": "min_replicas",
|
|
796
|
+
"type": "integer",
|
|
797
|
+
"description": ">= 1; defaults to the current value. Alias: minReplicas."
|
|
798
|
+
},
|
|
799
|
+
{
|
|
800
|
+
"name": "max_replicas",
|
|
801
|
+
"type": "integer",
|
|
802
|
+
"description": ">= min_replicas; defaults to the current value. Alias: maxReplicas. Sending either replica field pushes the new scaling range."
|
|
803
|
+
},
|
|
804
|
+
{
|
|
805
|
+
"name": "gpu_type",
|
|
806
|
+
"type": "string",
|
|
807
|
+
"description": "Change GPU type (must be available for the endpoint's model/flavor in its region). Alias: gpuType. Triggers a price re-freeze + wallet gate + final meter."
|
|
808
|
+
},
|
|
809
|
+
{
|
|
810
|
+
"name": "gpu_count",
|
|
811
|
+
"type": "integer",
|
|
812
|
+
"description": "Change GPU count (must be in allowed_gpu_counts). Alias: gpuCount. Same re-freeze semantics as gpu_type."
|
|
813
|
+
}
|
|
814
|
+
],
|
|
815
|
+
"responseSummary": "{ ok: true } on success (no body data).",
|
|
816
|
+
"notes": "400 'Invalid JSON body'. Requires an OWNER/ADMIN minting user (403). 404 'Endpoint not found'. 400 on invalid replica range (max must be >= min >= 1), GPU not available for the model/region, disallowed GPU count, max replicas above the configuration limit, or no price configured. MONEY: a GPU change re-prices the endpoint at today's rate (new frozen hourly_rate_usd) and is gated at 402 unless the wallet covers 1 prepay hour at the new configuration; stopping (enabled=false) or any GPU change immediately meters and bills the GPU-hours accrued so far. Status becomes UPDATING (GPU change), STOPPING (enabled=false) or STARTING (enabled=true). Scope note: local dedicated apiKeyActor does not enforce key scopes.",
|
|
817
|
+
"spends": true
|
|
818
|
+
},
|
|
819
|
+
{
|
|
820
|
+
"name": "delete_dedicated_endpoint",
|
|
821
|
+
"method": "DELETE",
|
|
822
|
+
"path": "/v1/dedicated/{id}",
|
|
823
|
+
"summary": "Tears down a dedicated endpoint: meters and bills the GPU-hours accrued since the last meter, frees the reserved GPUs, and marks it DELETED — the way to permanently stop paying for an endpoint.",
|
|
824
|
+
"scope": "platform:write",
|
|
825
|
+
"pathParams": [
|
|
826
|
+
{
|
|
827
|
+
"name": "id",
|
|
828
|
+
"type": "string",
|
|
829
|
+
"description": "Dedicated endpoint id (must belong to the key's workspace)."
|
|
830
|
+
}
|
|
831
|
+
],
|
|
832
|
+
"responseSummary": "{ ok: true } on success.",
|
|
833
|
+
"notes": "Requires an OWNER/ADMIN minting user (403). 404 'Endpoint not found' (already-deleted endpoints also 404). MONEY: runs a final meter first (bills accrued GPU-hours), then releases the endpoint; soft-deleted (status DELETED, enabled=false) and excluded from later lists. Not reversible. Scope note: local dedicated apiKeyActor does not enforce key scopes."
|
|
834
|
+
},
|
|
835
|
+
{
|
|
836
|
+
"name": "list_refusals",
|
|
837
|
+
"method": "GET",
|
|
838
|
+
"path": "/v1/enforcement/refusals",
|
|
839
|
+
"summary": "Pages through the workspace's refusal ledger — every time enforcement stopped something (an alias repoint without evidence, an auto-gate or dataset build refusing an untrusted judge, a reward refusing a drift-flagged criterion, a round gate holding, an eval refused off its calibrated population) — newest first, for audit and compliance reporting.",
|
|
840
|
+
"scope": "read",
|
|
841
|
+
"query": [
|
|
842
|
+
{
|
|
843
|
+
"name": "since",
|
|
844
|
+
"type": "string",
|
|
845
|
+
"description": "ISO-8601 date/time; only refusals created at or after it. 400 'since must be an ISO date' if unparseable."
|
|
846
|
+
},
|
|
847
|
+
{
|
|
848
|
+
"name": "kind",
|
|
849
|
+
"type": "string",
|
|
850
|
+
"description": "Exact-match filter on refusal kind.",
|
|
851
|
+
"enum": [
|
|
852
|
+
"alias_config_gate",
|
|
853
|
+
"alias_evidence",
|
|
854
|
+
"alias_act_hold",
|
|
855
|
+
"dataset_judge_trust",
|
|
856
|
+
"reward_validation",
|
|
857
|
+
"round_gate",
|
|
858
|
+
"eval_population"
|
|
859
|
+
]
|
|
860
|
+
},
|
|
861
|
+
{
|
|
862
|
+
"name": "subject",
|
|
863
|
+
"type": "string",
|
|
864
|
+
"description": "Exact-match filter on subject identity, e.g. 'alias:prod-chat', 'criterion:<id>', 'run:<prefix>'."
|
|
865
|
+
},
|
|
866
|
+
{
|
|
867
|
+
"name": "cursor",
|
|
868
|
+
"type": "string",
|
|
869
|
+
"description": "Opaque cursor = the `next_cursor` (a refusal id) from the previous page; returns rows strictly after it in newest-first order."
|
|
870
|
+
},
|
|
871
|
+
{
|
|
872
|
+
"name": "limit",
|
|
873
|
+
"type": "integer",
|
|
874
|
+
"description": "Page size, integer 1..200 (400 otherwise).",
|
|
875
|
+
"default": 50
|
|
876
|
+
}
|
|
877
|
+
],
|
|
878
|
+
"responseSummary": "{ refusals: [{ id, kind, subject, reason (verbatim refusal message, truncated to 500 chars), created_at (ISO) }], next_cursor: <string|null — null on the last page> }.",
|
|
879
|
+
"notes": "Ordered by created_at desc, id desc. The ledger is append-only: a refusal later overridden is still listed. Cache-Control: no-store. Free."
|
|
880
|
+
},
|
|
881
|
+
{
|
|
882
|
+
"name": "list_env_tools",
|
|
883
|
+
"method": "GET",
|
|
884
|
+
"path": "/v1/env/tools",
|
|
885
|
+
"summary": "Lists the agent tools this workspace has declared for training environments (the egress allowlist) together with the per-workspace secret used to verify signed environment calls — use it to audit which endpoints and credentials environments may call.",
|
|
886
|
+
"scope": "read",
|
|
887
|
+
"responseSummary": "snake_case: { tools: [{ id, name, endpoint_url, auth_prefix (first 10 chars of the stored header + ellipsis, '(configured)' if undecryptable, or null), read_only, max_calls_per_episode, enabled, created_at }], egress_verification_secret: <hex HMAC secret; environment calls carry X-Omnia-Environment / X-Omnia-Timestamp / X-Omnia-Signature = HMAC_SHA256(secret, `${timestamp}.${rawBody}`)> }.",
|
|
888
|
+
"notes": "Gated behind the fineTuning feature flag: 404 'Fine-tuning is not enabled' when off (prod runs with it ON). Requires an OWNER/ADMIN minting user even for GET (403). Credentials (auth headers) are encrypted at rest and never returned — only the prefix. Tools are sorted by name."
|
|
889
|
+
},
|
|
890
|
+
{
|
|
891
|
+
"name": "register_env_tool",
|
|
892
|
+
"method": "POST",
|
|
893
|
+
"path": "/v1/env/tools",
|
|
894
|
+
"summary": "Registers (or updates, by name) an agent tool that training environments for this workspace are allowed to call — the explicit consent grant naming the https endpoint, its credential, whether it is read-only, and a per-episode call cap.",
|
|
895
|
+
"scope": "platform:write",
|
|
896
|
+
"body": [
|
|
897
|
+
{
|
|
898
|
+
"name": "name",
|
|
899
|
+
"type": "string",
|
|
900
|
+
"description": "Tool name, 1-64 chars of letters, digits, '_', '.', '-' (trimmed). Upsert key: re-posting the same name updates the registration and re-enables it.",
|
|
901
|
+
"required": true
|
|
902
|
+
},
|
|
903
|
+
{
|
|
904
|
+
"name": "endpointUrl",
|
|
905
|
+
"type": "string",
|
|
906
|
+
"description": "Absolute https URL the environment may call. Rejected (400) if not https, if it embeds username/password, or targets localhost, a private/loopback/link-local/CGNAT IPv4, IPv6 loopback/link-local/unique-local, IPv4-mapped private addresses, or a cloud metadata host. NOTE: camelCase key — no snake_case alias is accepted on this route.",
|
|
907
|
+
"required": true
|
|
908
|
+
},
|
|
909
|
+
{
|
|
910
|
+
"name": "authHeader",
|
|
911
|
+
"type": "string",
|
|
912
|
+
"description": "Full Authorization header value to send to the tool (e.g. 'Bearer sk-...'). Stored encrypted, never returned. On update, omit to keep the existing header; send '' to clear it. 400 if encrypted storage is not configured. camelCase only."
|
|
913
|
+
},
|
|
914
|
+
{
|
|
915
|
+
"name": "readOnly",
|
|
916
|
+
"type": "boolean",
|
|
917
|
+
"description": "Whether the tool is side-effect free. Defaults to true whenever omitted (including on update). camelCase only."
|
|
918
|
+
},
|
|
919
|
+
{
|
|
920
|
+
"name": "maxCallsPerEpisode",
|
|
921
|
+
"type": "integer",
|
|
922
|
+
"description": "Per-episode call cap, clamped to 1..500. Defaults to 20 whenever omitted (including on update). camelCase only."
|
|
923
|
+
}
|
|
924
|
+
],
|
|
925
|
+
"responseSummary": "201 with the tool view, snake_case: { id, name, endpoint_url, auth_prefix, read_only, max_calls_per_episode, enabled, created_at }.",
|
|
926
|
+
"notes": "Request body keys are camelCase (endpointUrl, authHeader, readOnly, maxCallsPerEpisode) while the response is snake_case — the route lifts no aliases. 400 'Invalid JSON body' or 'Missing required field(s): name, endpointUrl' when either is missing/empty. Gated behind the fineTuning feature flag (404 when off). Requires an OWNER/ADMIN minting user (403). Registration-time host validation only; DNS rebinding is not defended here. Writes an audit event. No money implication by itself."
|
|
927
|
+
},
|
|
928
|
+
{
|
|
929
|
+
"name": "delete_env_tool",
|
|
930
|
+
"method": "DELETE",
|
|
931
|
+
"path": "/v1/env/tools/{id}",
|
|
932
|
+
"summary": "Revokes a declared agent tool registration (consent withdrawal) so training environments can no longer call that endpoint.",
|
|
933
|
+
"scope": "platform:write",
|
|
934
|
+
"pathParams": [
|
|
935
|
+
{
|
|
936
|
+
"name": "id",
|
|
937
|
+
"type": "string",
|
|
938
|
+
"description": "Tool registration id (from GET /v1/env/tools; must belong to the key's workspace)."
|
|
939
|
+
}
|
|
940
|
+
],
|
|
941
|
+
"responseSummary": "{ ok: true } on success.",
|
|
942
|
+
"notes": "Gated behind the fineTuning feature flag (404 'Fine-tuning is not enabled' when off). Requires an OWNER/ADMIN minting user (403). 404 'Tool not found' when the id is not in the workspace. Hard delete; writes an audit event."
|
|
943
|
+
},
|
|
944
|
+
{
|
|
945
|
+
"name": "list_evals",
|
|
946
|
+
"method": "GET",
|
|
947
|
+
"path": "/v1/evals",
|
|
948
|
+
"summary": "List this workspace's eval runs (newest first, most recent 50) with status, progress and stored results, so a customer can see every comparison, criterion run and screening they have queued or finished.",
|
|
949
|
+
"scope": "read",
|
|
950
|
+
"responseSummary": "{object:\"list\", data:[run]} where run = {id, name, rubric, rubric_type, eval_kind (\"comparison\"|\"criterion\"), criterion_snapshot (frozen judge instrument on criterion runs, else null), baseline_model (a catalog id or \"__stored__\"), candidate_models (arm keys), arms:[{key, model, label|null, system:bool, tools:bool, n}], judge_model, sample_count, sample_filters:{model?, tag?, segment?, dataset_id?, trace_replay?, screening?}, status (PENDING|RUNNING|DONE|ERROR|CANCELLED), error|null, results (null until DONE; comparison: {sample_count, clipped_samples, baseline:{model, stored_answers, truncated, avg_latency_ms, eval_cost_micros}, per_candidate:[{model, arm, wins, losses, ties, failed, attempted, judged_share, unreportable, win_rate, ci95, inconclusive, truncated, avg_latency_ms, eval_cost_micros, savings_pct, replay?}], judge_cost_micros, screening?:{incumbent, token_shape, per_candidate:[{model, similarity:{matched, differed, unparsed, judged, match_rate, ci95}, est_usd_per_request, est_savings_pct, projected_monthly_usd, projected_monthly_savings_usd, ...placement}], recommendation}}; criterion: {eval_kind:\"criterion\", sample_count, clipped_samples, criterion, per_model:[{model, judged_pass, judged_fail, unparsed, truncated, observed_pass_rate, observed_ci, corrected_pass_rate|null, corrected_ci|null, avg_latency_ms, eval_cost_micros}], judge_youden, judge_cost_micros}), assertions|null, created_at (ISO), progress_ratio (0..1)}.",
|
|
951
|
+
"notes": "No pagination or filtering: always the 50 newest runs. All keys are snake_cased at the door (camelCase internally); model ids used as map keys pass through untouched. Read-only, no wallet spend."
|
|
952
|
+
},
|
|
953
|
+
{
|
|
954
|
+
"name": "create_eval",
|
|
955
|
+
"method": "POST",
|
|
956
|
+
"path": "/v1/evals",
|
|
957
|
+
"summary": "Queue an eval run — a pairwise model comparison, an absolute criterion (calibrated judge) run, or a one-click screening of cheaper models against your own logged traffic — so a customer can measure a model, prompt, tool or index change before shipping it.",
|
|
958
|
+
"scope": "evals:write",
|
|
959
|
+
"body": [
|
|
960
|
+
{
|
|
961
|
+
"name": "screening",
|
|
962
|
+
"type": "boolean",
|
|
963
|
+
"description": "Screening mode. When true every other field becomes an optional override and the server auto-fills like the dashboard's one-click: incumbent = your dominant logged model, baseline = its stored answers (\"__stored__\"), candidates = the cheapest model of each distinct family, judge family-checked, fixed quality rubric, sample_count = clamp(min(40, population), 5..500). Only name, sample_count, candidate_models (1..6 explicit picks, validated against the catalog: unknown id / the incumbent itself / duplicate / >6 / empty list = 400 naming the offender), judge_model, and sample_filters.dataset_id (screen an imported dataset's stored answers instead of logged traffic) are honoured in this mode; rubric, rubric_type, eval_kind, criterion_id, baseline_model, candidates[], assertions, other sample_filters and max_output_tokens are ignored. Screening additionally enforces a creation-time funds gate (402)."
|
|
964
|
+
},
|
|
965
|
+
{
|
|
966
|
+
"name": "name",
|
|
967
|
+
"type": "string",
|
|
968
|
+
"description": "Run name, 1..80 chars after trimming (required unless screening=true, where it is auto-generated).",
|
|
969
|
+
"required": true
|
|
970
|
+
},
|
|
971
|
+
{
|
|
972
|
+
"name": "rubric",
|
|
973
|
+
"type": "string",
|
|
974
|
+
"description": "Judge rubric, 10..2000 chars. Required for eval_kind=\"comparison\" (400 if shorter than 10 chars); ignored for criterion runs (the criterion's frozen judge prompt is the rubric)."
|
|
975
|
+
},
|
|
976
|
+
{
|
|
977
|
+
"name": "rubric_type",
|
|
978
|
+
"type": "string",
|
|
979
|
+
"description": "How the judge reads the rubric: \"direct\" (default) judges answers on the rubric alone; \"adherence\" also requires each sample to carry a reference answer (the logged reply), so the population must have text replies.",
|
|
980
|
+
"enum": [
|
|
981
|
+
"direct",
|
|
982
|
+
"adherence"
|
|
983
|
+
],
|
|
984
|
+
"default": "direct"
|
|
985
|
+
},
|
|
986
|
+
{
|
|
987
|
+
"name": "eval_kind",
|
|
988
|
+
"type": "string",
|
|
989
|
+
"description": "\"comparison\" (default): each candidate arm is judged pairwise against the baseline in both orderings → win rate with Wilson CI. \"criterion\": every model (baseline and candidates) is graded absolutely by a saved, calibrated criterion → observed and calibration-corrected pass rates; requires criterion_id.",
|
|
990
|
+
"enum": [
|
|
991
|
+
"comparison",
|
|
992
|
+
"criterion"
|
|
993
|
+
],
|
|
994
|
+
"default": "comparison"
|
|
995
|
+
},
|
|
996
|
+
{
|
|
997
|
+
"name": "criterion_id",
|
|
998
|
+
"type": "string",
|
|
999
|
+
"description": "Id of a workspace criterion (calibrated judge). Required when eval_kind=\"criterion\" (400 otherwise). The criterion's judge model and prompt override judge_model/rubric and are frozen into the run (criterion_snapshot). A trace-unit criterion requires baseline_model=\"__stored__\" and no candidates (it grades completed agent runs), and refuses if it was aligned on an older transcript instrument version. Population binding is enforced: sampling a tag/segment different from the criterion's calibrated population is refused (400); sampling with no population filter while the judge is scoped is allowed with a stored warning."
|
|
1000
|
+
},
|
|
1001
|
+
{
|
|
1002
|
+
"name": "baseline_model",
|
|
1003
|
+
"type": "string",
|
|
1004
|
+
"description": "The incumbent arm: a catalog model id this workspace is offered (validated, 400 \"Model '…' is not available.\"), or the sentinel \"__stored__\" to judge candidates against the incumbent's STORED logged answers (nothing is regenerated for the baseline; no savings figure is computed). \"__stored__\" is required for trace_replay and for the certified-switch (noninferiority) gate shape. Required unless screening=true (then forced to \"__stored__\").",
|
|
1005
|
+
"required": true
|
|
1006
|
+
},
|
|
1007
|
+
{
|
|
1008
|
+
"name": "candidate_models",
|
|
1009
|
+
"type": "array",
|
|
1010
|
+
"description": "Array of catalog model ids (max 6, no duplicates, none equal to baseline_model). Each candidate answers every sample and is judged, so cost is linear in this count. A comparison needs at least 1 (400 otherwise); a criterion run may have 0 (grade the baseline alone). Ignored when candidates[] is present (candidates[] replaces it).",
|
|
1011
|
+
"items": "string"
|
|
1012
|
+
},
|
|
1013
|
+
{
|
|
1014
|
+
"name": "candidates",
|
|
1015
|
+
"type": "array",
|
|
1016
|
+
"description": "Versioned arms — an alternative to candidate_models (when present, candidate_models is ignored and this list defines the arms; max 6 total). Each item: {model: string (required, catalog id that runs the arm), label?: string (arm key shown in the report; must match /^[A-Za-z0-9][A-Za-z0-9 _.:+\\-]{0,63}$/ and must NOT contain \"/\"), system?: string (replace the logged system prompt on every sampled prompt; \"\" strips it; max 20000 chars; omit to keep the logged one), tools?: array of tool-definition objects (OpenAI format; replaces the logged tool definitions; [] offers none; max 64; omit to keep), n?: integer 2..8 (best-of-N: sample N times and keep the judge-preferred answer via N−1 pairwise knockout verdicts on the run's own rubric/judge; the arm pays for all N generations plus the selection verdicts)}. A bare {model} is identical to listing the id in candidate_models. Any arm that sets system, tools or n MUST carry a label (400 \"candidates[]: an arm that overrides system or tools needs a label\"); the label becomes the arm key in candidate_models/arms/results, and the override is stored as armOverrides[label] = {model, system?, tools?, n?}. Two arms may share one model (e.g. old prompt vs new prompt); metering follows the model that actually ran.",
|
|
1017
|
+
"items": "string"
|
|
1018
|
+
},
|
|
1019
|
+
{
|
|
1020
|
+
"name": "sample_count",
|
|
1021
|
+
"type": "integer",
|
|
1022
|
+
"description": "Number of prompts to sample from the population, integer 5..500. Defaults to 20 when omitted (non-screening). The run is refused at creation (400) if the filtered population cannot supply at least 5 distinct prompts (with references when rubric_type=\"adherence\" or baseline is \"__stored__\"). Screening: clamped to 5..500, default min(40, population).",
|
|
1023
|
+
"default": 20
|
|
1024
|
+
},
|
|
1025
|
+
{
|
|
1026
|
+
"name": "max_output_tokens",
|
|
1027
|
+
"type": "integer",
|
|
1028
|
+
"description": "Per-answer generation output cap, integer 256..16384 (default 4096) — sized so thinking models can finish reasoning and answer; the runaway-spend guard. Silently ignored in screening mode.",
|
|
1029
|
+
"default": 4096
|
|
1030
|
+
},
|
|
1031
|
+
{
|
|
1032
|
+
"name": "judge_model",
|
|
1033
|
+
"type": "string",
|
|
1034
|
+
"description": "Catalog chat model used as the judge (validated; 400 if not offered). Precedence when omitted: the workspace's default judge, then the house default judge. Ignored for criterion runs (the criterion's judge is frozen). In screening mode an explicit judge is honoured even if it shares a family with a contestant (the report discloses judge_shares_family) instead of being swapped."
|
|
1035
|
+
},
|
|
1036
|
+
{
|
|
1037
|
+
"name": "assertions",
|
|
1038
|
+
"type": "array",
|
|
1039
|
+
"description": "Up to 10 deterministic output checks run against every generated answer at finalize (free, exact — these are what CI should gate on via min_assertion_pass_rate). Each: {type, value?}. Types: \"json_valid\" (no value), \"json_schema\" (value = JSON Schema string ≤4000 chars, must parse), \"regex_match\" (value = pattern ≤200 chars, must compile), \"contains\" / \"not_contains\" (value = substring, required), \"max_length\" / \"min_length\" (value = non-negative integer as string), \"completed\" (finish reason was not a length cut-off), \"tool_called\" (value = tool name ≤200 chars; matches the canonical \"[tool call] name(args)\" notation), \"no_tool_call\". Invalid configs are rejected 400 with the specific reason.",
|
|
1040
|
+
"enum": [
|
|
1041
|
+
"json_valid",
|
|
1042
|
+
"json_schema",
|
|
1043
|
+
"regex_match",
|
|
1044
|
+
"contains",
|
|
1045
|
+
"not_contains",
|
|
1046
|
+
"max_length",
|
|
1047
|
+
"min_length",
|
|
1048
|
+
"completed",
|
|
1049
|
+
"tool_called",
|
|
1050
|
+
"no_tool_call"
|
|
1051
|
+
],
|
|
1052
|
+
"items": "string"
|
|
1053
|
+
},
|
|
1054
|
+
{
|
|
1055
|
+
"name": "sample_filters",
|
|
1056
|
+
"type": "object",
|
|
1057
|
+
"description": "Which population prompts are sampled from (success-only logged requests by default). Keys: tag?: string (only requests logged with this tag); model?: string (only requests served by this model); segment?: string (an auto-detected traffic segment = the prompt FAMILY shown as `segment` on GET /v1/logs rows — one application surface's traffic, stable under interpolated dates/ids); dataset_id?: string (sample from a managed dataset — e.g. a holdout split — instead of live logs; must belong to this workspace, 404 otherwise; the run needs ≥5 usable rows); trace_replay?: boolean (replay bake-off: sample WHOLE completed agent runs from the last 7 days, one teacher-forced sample per step, max 12 steps per run; REQUIRES baseline_model=\"__stored__\" (400 otherwise) and ≥5 replayable steps). Empty-string values are treated as absent. Screening mode reads only dataset_id here."
|
|
1058
|
+
}
|
|
1059
|
+
],
|
|
1060
|
+
"responseSummary": "201 with the queued run in the same shape as GET /v1/evals/{id} (status PENDING, results null, progress_ratio 0, arms[] describing each candidate key). Poll GET /v1/evals/{id} until status is DONE|ERROR|CANCELLED, or gate a pipeline directly with GET /v1/evals/{id}/gate.",
|
|
1061
|
+
"notes": "MONEY: a run spends wallet credit (every generation for baseline + each arm, plus judge calls; best-of-N arms pay N×). Plain runs disclose cost and are gated lazily per tick by the runner; screening runs enforce a creation-time funds gate → 402 {error:{type:\"insufficient_quota\", code:\"insufficient_balance\"}}. The key's minting user must be workspace OWNER/ADMIN → otherwise 403. 400 on: invalid JSON, schema violations (name length, rubric length, sample_count range, >6 candidates, duplicates, candidate == baseline, bad assertion, bad arm label/override), unknown model, missing criterion_id, population-binding refusal, trace_replay without stored baseline, or too little population (\"Not enough logged traffic for this filter (need at least 5 distinct prompts…)\"). 404 for a dataset/criterion not in this workspace. Screening-specific: 422 {code:\"unprocessable\"} when there is nothing to screen (no/too little logged traffic, or nothing cheaper than the incumbent — the message carries the import hint), 400 for an unusable candidate_models list, 402 for funds. All errors are {error:{message, type, code}}. Input keys are snake_case; internally converted to camelCase (candidate_models→candidateModels, candidates[]→candidateModels keys + armOverrides map keyed by label, sample_filters.dataset_id→datasetId, trace_replay→traceReplay, max_output_tokens→genMaxOutputTokens).",
|
|
1062
|
+
"spends": true
|
|
1063
|
+
},
|
|
1064
|
+
{
|
|
1065
|
+
"name": "compare_evals",
|
|
1066
|
+
"method": "GET",
|
|
1067
|
+
"path": "/v1/evals/compare",
|
|
1068
|
+
"summary": "Compare two finished eval runs arm-by-arm (before vs after a prompt, tool or index change) and get each arm's delta with a 95% interval and a significance flag, instead of eyeballing two reports.",
|
|
1069
|
+
"scope": "read",
|
|
1070
|
+
"query": [
|
|
1071
|
+
{
|
|
1072
|
+
"name": "a",
|
|
1073
|
+
"type": "string",
|
|
1074
|
+
"description": "Eval run id of the BEFORE run (baseline of the comparison).",
|
|
1075
|
+
"required": true
|
|
1076
|
+
},
|
|
1077
|
+
{
|
|
1078
|
+
"name": "b",
|
|
1079
|
+
"type": "string",
|
|
1080
|
+
"description": "Eval run id of the AFTER run. Delta is reported as b − a.",
|
|
1081
|
+
"required": true
|
|
1082
|
+
}
|
|
1083
|
+
],
|
|
1084
|
+
"responseSummary": "{a:{id, name, created_at, judge_model, eval_kind}, b:{…same}, rows:[{arm (candidate key), metric (\"win_rate\" for comparison runs; \"observed_pass_rate\" and, when both runs carry one, \"corrected_pass_rate\" for criterion runs), a:{rate, n, ci:[lo,hi]}, b:{rate, n, ci}, delta (b.rate − a.rate), delta_ci:[lo,hi] (Newcombe 95%), significant (interval excludes zero)}], unmatched:{a:[arm keys only in a], b:[arm keys only in b]}}. Cache-Control: no-store.",
|
|
1085
|
+
"notes": "400 when a or b is missing. 404 when either run is not in this workspace. 412 {code:\"precondition_failed\"} when both runs are not DONE, when they are different eval kinds, or when they used different judge models (a delta between judges measures the judges, not your change — re-run one with the other's judge). Best used with identical sample_filters on both runs. Read-only, no spend."
|
|
1086
|
+
},
|
|
1087
|
+
{
|
|
1088
|
+
"name": "get_failure_clusters",
|
|
1089
|
+
"method": "GET",
|
|
1090
|
+
"path": "/v1/evals/failure_clusters",
|
|
1091
|
+
"summary": "See live production failures grouped into systemic causes per criterion (judge FAIL rationales plus pending scan suspects, clustered by a model) so a customer can find what to fix first rather than reading failures one by one.",
|
|
1092
|
+
"scope": "read",
|
|
1093
|
+
"query": [
|
|
1094
|
+
{
|
|
1095
|
+
"name": "window_days",
|
|
1096
|
+
"type": "integer",
|
|
1097
|
+
"description": "Look-back window in days, integer 1..90.",
|
|
1098
|
+
"default": 7
|
|
1099
|
+
},
|
|
1100
|
+
{
|
|
1101
|
+
"name": "force",
|
|
1102
|
+
"type": "boolean",
|
|
1103
|
+
"description": "Pass the literal \"true\" to bypass the per-workspace one-hour cache and re-cluster now.",
|
|
1104
|
+
"default": false
|
|
1105
|
+
}
|
|
1106
|
+
],
|
|
1107
|
+
"responseSummary": "{window_days, generated_at, cached (true when served from the hourly cache), criteria:[{criterion_id, criterion_name, failures (online FAILs + pending suspects, deduped), without_reason (failures with no stored rationale — counted, never clustered), clusters:[{name, count, share (of this criterion's clustered failures), request_ids, example (one representative rationale verbatim)}]}]}. Cache-Control: no-store.",
|
|
1108
|
+
"notes": "400 \"window_days must be an integer 1..90\" for an out-of-range value. MONEY: a fresh clustering (cache miss or force=true) makes one small metered model call per criterion that has ≥4 failure reasons (at most 40 reasons per criterion) — billed to the wallet like other assists; cached responses cost nothing. Criteria with fewer than 4 reasons are listed with no clusters."
|
|
1109
|
+
},
|
|
1110
|
+
{
|
|
1111
|
+
"name": "get_eval",
|
|
1112
|
+
"method": "GET",
|
|
1113
|
+
"path": "/v1/evals/{id}",
|
|
1114
|
+
"summary": "Fetch one eval run's status, progress and — once DONE — its full results (per-arm win rate with CI, W/T/L, latency, eval cost, savings, or corrected pass rates for criterion runs); poll this after creating a run.",
|
|
1115
|
+
"scope": "read",
|
|
1116
|
+
"pathParams": [
|
|
1117
|
+
{
|
|
1118
|
+
"name": "id",
|
|
1119
|
+
"type": "string",
|
|
1120
|
+
"description": "The eval run id returned by POST /v1/evals."
|
|
1121
|
+
}
|
|
1122
|
+
],
|
|
1123
|
+
"responseSummary": "The run object: {id, name, rubric, rubric_type, eval_kind, criterion_snapshot, baseline_model, candidate_models, arms:[{key, model, label, system, tools, n}], judge_model, sample_count, sample_filters:{model?, tag?, segment?, dataset_id?, trace_replay?, screening?}, status (PENDING|RUNNING|DONE|ERROR|CANCELLED), error (null, a failure reason, or \"Cancelled by <email>\"), results (see list_evals for the comparison / criterion / screening shapes), assertions, created_at, progress_ratio (0..1; completed inference units over total — feed a progress bar)}.",
|
|
1124
|
+
"notes": "404 {code:\"not_found\"} when the run is not in this key's workspace. results is null until DONE. A DONE screening's results.screening carries the similarity lens (match rate — never part of win/loss), quality-vs-cost placement per candidate and a recommendation (a \"keep\" is a first-class good outcome). Read-only, no spend."
|
|
1125
|
+
},
|
|
1126
|
+
{
|
|
1127
|
+
"name": "delete_eval",
|
|
1128
|
+
"method": "DELETE",
|
|
1129
|
+
"path": "/v1/evals/{id}",
|
|
1130
|
+
"summary": "Permanently delete a finished (DONE|ERROR|CANCELLED) eval run together with its samples, generated answers and judge verdicts — for cleaning up runs a customer no longer needs as evidence.",
|
|
1131
|
+
"scope": "evals:write",
|
|
1132
|
+
"pathParams": [
|
|
1133
|
+
{
|
|
1134
|
+
"name": "id",
|
|
1135
|
+
"type": "string",
|
|
1136
|
+
"description": "The eval run id."
|
|
1137
|
+
}
|
|
1138
|
+
],
|
|
1139
|
+
"responseSummary": "200 {id} of the deleted run.",
|
|
1140
|
+
"notes": "Requires an OWNER/ADMIN minting user (403 otherwise). A PENDING|RUNNING run is refused with 400 \"Cancel it first — a live run can't be deleted.\" — call POST /v1/evals/{id}/cancel first. 404 if not found in this workspace. Deletion is irreversible and cascades to samples/outputs/verdicts; the models, judges and criteria the run referenced are untouched. After deletion GET /v1/evals/{id}/gate and /evidence return 404. No spend."
|
|
1141
|
+
},
|
|
1142
|
+
{
|
|
1143
|
+
"name": "cancel_eval",
|
|
1144
|
+
"method": "POST",
|
|
1145
|
+
"path": "/v1/evals/{id}/cancel",
|
|
1146
|
+
"summary": "Stop a PENDING or RUNNING eval run so no further generations or judge calls are billed — use it when a run was misconfigured or is no longer needed.",
|
|
1147
|
+
"scope": "evals:write",
|
|
1148
|
+
"pathParams": [
|
|
1149
|
+
{
|
|
1150
|
+
"name": "id",
|
|
1151
|
+
"type": "string",
|
|
1152
|
+
"description": "The eval run id."
|
|
1153
|
+
}
|
|
1154
|
+
],
|
|
1155
|
+
"responseSummary": "200 with the run object (same shape as GET /v1/evals/{id}) with status \"CANCELLED\" and error set to \"Cancelled by <email of the key's minting user>\"; progress_ratio reflects work completed so far.",
|
|
1156
|
+
"notes": "Requires OWNER/ADMIN minting user (403) — same gate as create, because it controls wallet spend. A run already DONE|ERROR|CANCELLED is refused 400 (\"This run is already done — only a pending or running eval can be cancelled.\"); a terminal result is never overwritten. 404 if not in this workspace. Cancelled runs are inert: no future tick claims them, so nothing more is billed; a slice already in flight finishes its bounded batch (already-paid work), and in the rare race where that was the last slice the run may still land DONE with real results. No request body is read."
|
|
1157
|
+
},
|
|
1158
|
+
{
|
|
1159
|
+
"name": "get_eval_evidence",
|
|
1160
|
+
"method": "GET",
|
|
1161
|
+
"path": "/v1/evals/{id}/evidence",
|
|
1162
|
+
"summary": "Download the forwardable proof bundle for a DONE eval run — verdict, frozen judge calibration and certificate, per-sample verdict lineage, refusal ledger events and an audit hash-chain attestation, optionally with the embedded CI-gate decision — so a customer can hand a reviewer or auditor one signed JSON that links claim to instrument to data.",
|
|
1163
|
+
"scope": "read",
|
|
1164
|
+
"pathParams": [
|
|
1165
|
+
{
|
|
1166
|
+
"name": "id",
|
|
1167
|
+
"type": "string",
|
|
1168
|
+
"description": "The eval run id (must be DONE)."
|
|
1169
|
+
}
|
|
1170
|
+
],
|
|
1171
|
+
"query": [
|
|
1172
|
+
{
|
|
1173
|
+
"name": "with_content",
|
|
1174
|
+
"type": "boolean",
|
|
1175
|
+
"description": "Pass the literal string \"true\" to include sampled prompts and generated answers in samples.lineage. Honoured only when the workspace has request logging (content storage) enabled; otherwise lineage stays ids/verdicts only and content.reason explains why.",
|
|
1176
|
+
"default": false
|
|
1177
|
+
},
|
|
1178
|
+
{
|
|
1179
|
+
"name": "min_win_rate",
|
|
1180
|
+
"type": "number",
|
|
1181
|
+
"description": "0..1. Embed the same gate evaluation as GET /v1/evals/{id}/gate: every candidate's win-rate CI lower bound must clear this. Values outside 0..1 or non-numeric are ignored."
|
|
1182
|
+
},
|
|
1183
|
+
{
|
|
1184
|
+
"name": "min_pass_rate",
|
|
1185
|
+
"type": "number",
|
|
1186
|
+
"description": "0..1. Criterion runs: corrected pass-rate CI lower bound (observed CI when the judge is unvalidated) must clear this."
|
|
1187
|
+
},
|
|
1188
|
+
{
|
|
1189
|
+
"name": "min_assertion_pass_rate",
|
|
1190
|
+
"type": "number",
|
|
1191
|
+
"description": "0..1. Exact all-assertions pass rate must clear this."
|
|
1192
|
+
},
|
|
1193
|
+
{
|
|
1194
|
+
"name": "noninferiority_margin",
|
|
1195
|
+
"type": "number",
|
|
1196
|
+
"description": "0..1. The certified switch test (see get_eval_gate)."
|
|
1197
|
+
},
|
|
1198
|
+
{
|
|
1199
|
+
"name": "model",
|
|
1200
|
+
"type": "string",
|
|
1201
|
+
"description": "Restrict the embedded gate checks to one arm/model key."
|
|
1202
|
+
}
|
|
1203
|
+
],
|
|
1204
|
+
"responseSummary": "200 bundle (snake_cased): {signature|null, unsigned?:true (when no signing secret is configured), bundle_v:1, generated_at, run:{id, workspace_id, name, eval_kind, status:\"DONE\", created_at, sample_count, baseline_model, candidate_models, judge_model, rubric_type, sample_filters, assertions}, results (verdict verbatim), gate:{params, verdict}|null (only when at least one gate param was given), instrument:{judge_model, judge_prompt, criterion_snapshot, certificate|null, note|null}, samples:{count, note, lineage:[{sample id, every verdict with the ordering that measured it (\"ab\"/\"ba\" pairwise halves, \"abs\" absolute, \"sim\" screening similarity), prompt/answers only with with_content}]}, content:{included, reason}, refusals:{window_days, scope, count, complete, events:[{kind, subject, reason, created_at}]}, attestation:{ok, checked_rows, head_seq, problems:[{seq, kind, detail}], acknowledged, window:{since, until, from_seq}|null, chain_head:{seq, last_hash}|null, statement}}.",
|
|
1205
|
+
"notes": "404 for a run outside this key's workspace. 412 {code:\"precondition_failed\"} for any run that is not DONE (PENDING/RUNNING/ERROR/CANCELLED) — fail closed like the gate; poll until DONE. with_content silently degrades (never errors) when request logging is off. Read-only, no spend."
|
|
1206
|
+
},
|
|
1207
|
+
{
|
|
1208
|
+
"name": "get_eval_gate",
|
|
1209
|
+
"method": "GET",
|
|
1210
|
+
"path": "/v1/evals/{id}/gate",
|
|
1211
|
+
"summary": "Turn a finished eval run into a CI deploy decision with one call — 200 when every requested threshold passes, 412 otherwise — so a pipeline can `curl -f` it and block a bad model/prompt change.",
|
|
1212
|
+
"scope": "read",
|
|
1213
|
+
"pathParams": [
|
|
1214
|
+
{
|
|
1215
|
+
"name": "id",
|
|
1216
|
+
"type": "string",
|
|
1217
|
+
"description": "The eval run id."
|
|
1218
|
+
}
|
|
1219
|
+
],
|
|
1220
|
+
"query": [
|
|
1221
|
+
{
|
|
1222
|
+
"name": "min_win_rate",
|
|
1223
|
+
"type": "number",
|
|
1224
|
+
"description": "0..1. Comparison runs: every candidate arm's win-rate 95% CI LOWER bound must be ≥ this (never the point estimate). Fails closed when the judge returned no verdict on too many pairs (unreportable)."
|
|
1225
|
+
},
|
|
1226
|
+
{
|
|
1227
|
+
"name": "min_pass_rate",
|
|
1228
|
+
"type": "number",
|
|
1229
|
+
"description": "0..1. Criterion runs: every model's calibration-corrected pass-rate CI lower bound must be ≥ this; falls back to the observed CI when the judge is unvalidated (the check's note says so)."
|
|
1230
|
+
},
|
|
1231
|
+
{
|
|
1232
|
+
"name": "min_assertion_pass_rate",
|
|
1233
|
+
"type": "number",
|
|
1234
|
+
"description": "0..1. Every model's exact all-assertions pass rate must be ≥ this (deterministic count). Fails if the run has no assertions configured."
|
|
1235
|
+
},
|
|
1236
|
+
{
|
|
1237
|
+
"name": "noninferiority_margin",
|
|
1238
|
+
"type": "number",
|
|
1239
|
+
"description": "0..1. THE CERTIFIED SWITCH TEST: on a criterion run whose baseline is \"__stored__\" (the incumbent's logged answers) scored by a calibrated judge, each candidate's pass-rate CI floor must reach the incumbent's rate minus this margin (0.05 = provably within 5 points at worst). Requires the stored-baseline arm AND a calibrated judge (corrected rates) — no observed-rate fallback; fails otherwise with an explanatory note."
|
|
1240
|
+
},
|
|
1241
|
+
{
|
|
1242
|
+
"name": "model",
|
|
1243
|
+
"type": "string",
|
|
1244
|
+
"description": "Restrict the checks to one arm/model key (candidate key as listed in candidate_models)."
|
|
1245
|
+
}
|
|
1246
|
+
],
|
|
1247
|
+
"responseSummary": "{pass: boolean, status: run status, checks:[{check: \"win_rate\"|\"pass_rate\"|\"assertion_pass_rate\"|\"noninferiority\", model, required, actual (CI lower bound or exact rate, null when unavailable), pass, note?}], reason? (set when the gate could not evaluate: run not DONE, run ERROR, or no thresholds given)}. HTTP 200 only when pass is true; 412 whenever anything failed.",
|
|
1248
|
+
"notes": "412 (not 4xx-error shape — the verdict body itself) when: the run is not DONE (\"Run not complete yet — poll until status is DONE.\" — fail closed), the run is ERROR, no threshold param was passed, or any check fails. Query values must parse as numbers in 0..1; anything else is treated as absent. 404 {error:{…}} when the run is not in this workspace. Keys in this response are NOT re-cased (they are already snake/single-word). Read-only, no spend. Pair it with a `read`-only scoped key for CI."
|
|
1249
|
+
},
|
|
1250
|
+
{
|
|
1251
|
+
"name": "get_eval_samples",
|
|
1252
|
+
"method": "GET",
|
|
1253
|
+
"path": "/v1/evals/{id}/samples",
|
|
1254
|
+
"summary": "Inspect the test cases behind a run's score — each sampled prompt, the answer every arm produced (reasoning traces stripped, as the judge saw them) and the per-sample verdict — the audit trail that makes a win rate trustworthy.",
|
|
1255
|
+
"scope": "read",
|
|
1256
|
+
"pathParams": [
|
|
1257
|
+
{
|
|
1258
|
+
"name": "id",
|
|
1259
|
+
"type": "string",
|
|
1260
|
+
"description": "The eval run id."
|
|
1261
|
+
}
|
|
1262
|
+
],
|
|
1263
|
+
"responseSummary": "A bare JSON array (no list envelope), one item per sample in order: {prompt (messages rendered as \"ROLE: content\" lines, clipped to 2000 chars), baseline_answer (the baseline's fresh answer, or the stored logged reply when baseline is \"__stored__\"; empty string on criterion runs), candidates:[{model (arm key), answer (clipped to 2000 chars), outcome}]}. outcome is \"win\"|\"loss\"|\"tie\"|\"failed\" (judge gave no reading) on comparison runs and \"pass\"|\"fail\"|\"unparsed\" on criterion runs; criterion runs list the baseline among candidates.",
|
|
1264
|
+
"notes": "404 when the run is not in this workspace. Works on any status (partial data while RUNNING; empty array before sampling). Texts are clipped server-side at 2000 chars with a \"…[clipped]\" marker — use GET /v1/evals/{id}/evidence?with_content=true for full transcripts. Read-only, no spend."
|
|
1265
|
+
},
|
|
1266
|
+
{
|
|
1267
|
+
"name": "list_training_files",
|
|
1268
|
+
"method": "GET",
|
|
1269
|
+
"path": "/v1/fine_tuning/files",
|
|
1270
|
+
"summary": "List the training files this workspace has uploaded for fine-tuning, newest first, so a customer can find the file id to start a job with.",
|
|
1271
|
+
"scope": "read",
|
|
1272
|
+
"responseSummary": "A bare JSON array (no list envelope): [{id (local record id), provider_file_id (the opaque upstream file id — THIS is the value to pass as training.file_id / training_file_id when creating a job), filename, bytes, purpose (\"fine-tune\"), created_at}].",
|
|
1273
|
+
"notes": "Feature-flag gated: when the `fineTuning` flag is off every /v1/fine_tuning route returns 404 {error:\"Fine-tuning is not enabled\"} (plain string error, not the nested shape). Note: the fine-tuning routes authenticate the key directly and do NOT currently enforce management scopes (any valid key of the workspace works); requiredScopeFor would classify writes here as platform:write. Read-only, no spend."
|
|
1274
|
+
},
|
|
1275
|
+
{
|
|
1276
|
+
"name": "upload_training_file",
|
|
1277
|
+
"method": "POST",
|
|
1278
|
+
"path": "/v1/fine_tuning/files",
|
|
1279
|
+
"summary": "Upload a JSONL training (or validation) file for fine-tuning; the returned provider_file_id is what a job creation references.",
|
|
1280
|
+
"scope": "platform:write",
|
|
1281
|
+
"body": [
|
|
1282
|
+
{
|
|
1283
|
+
"name": "file",
|
|
1284
|
+
"type": "string",
|
|
1285
|
+
"description": "multipart/form-data field named \"file\" (NOT JSON). The file's own name is used as the filename (falls back to \"training.jsonl\" when empty). Empty files are refused (400 \"File is empty\").",
|
|
1286
|
+
"required": true
|
|
1287
|
+
}
|
|
1288
|
+
],
|
|
1289
|
+
"responseSummary": "201 {id, provider_file_id, filename, bytes, purpose:\"fine-tune\", created_at}. Use provider_file_id (not id) as training.file_id / training_file_id / validation_file_id in POST /v1/fine_tuning/jobs.",
|
|
1290
|
+
"notes": "Request must be multipart/form-data with a 'file' field — 400 \"Expected multipart/form-data with a 'file' field\" / \"Missing 'file' field\" otherwise. Requires OWNER/ADMIN minting user (403). Rate limited per workspace: 20 uploads per 60s → 429 with Retry-After. Feature-flag gated (404 when fineTuning is off). Storage is billable; the upload itself does not charge the wallet. Scopes are not enforced on this route today (see list_training_files)."
|
|
1291
|
+
},
|
|
1292
|
+
{
|
|
1293
|
+
"name": "list_fine_tuning_jobs",
|
|
1294
|
+
"method": "GET",
|
|
1295
|
+
"path": "/v1/fine_tuning/jobs",
|
|
1296
|
+
"summary": "List this workspace's fine-tuning jobs (newest first) with live status, progress, output model and price — for monitoring training from CI or a script.",
|
|
1297
|
+
"scope": "read",
|
|
1298
|
+
"responseSummary": "A bare JSON array: [{id, provider_job_id, name|null, base_model, method (\"supervised\"|\"spec-draft\"), status (VALIDATING_FILES|QUEUED|RUNNING|SUCCEEDED|FAILED|CANCELLED), fine_tuned_model|null, deployed_model_name|null (servable name after deployment), deploy_status|null (\"queued\"|\"staging\"|\"relaying\"|\"converting\"|\"provisioning\"|\"serving\"|\"failed\"), deploy_error|null, trained_tokens (string)|null, trained_steps|null, total_steps|null, rate_per_m_token_usd (customer price per 1M trained tokens), billed_cost_usd|null (set on completion), error|null, created_at}].",
|
|
1299
|
+
"notes": "Statuses are reconciled live against the training backend on each call (best effort; DB state served on backend error); a locally terminal status is never resurrected. Internal margin (markup) is stripped from the wire shape. Feature-flag gated (404 when fineTuning is off). Read-only, no spend. Scopes not enforced on this route today."
|
|
1300
|
+
},
|
|
1301
|
+
{
|
|
1302
|
+
"name": "create_fine_tuning_job",
|
|
1303
|
+
"method": "POST",
|
|
1304
|
+
"path": "/v1/fine_tuning/jobs",
|
|
1305
|
+
"summary": "Start a supervised (SFT/LoRA) or spec-draft fine-tune of a catalog base model on an uploaded file or a workspace dataset, with bounds-checked hyperparameters — the way a customer trains a custom model from their own data.",
|
|
1306
|
+
"scope": "platform:write",
|
|
1307
|
+
"body": [
|
|
1308
|
+
{
|
|
1309
|
+
"name": "base_model",
|
|
1310
|
+
"type": "string",
|
|
1311
|
+
"description": "Base model id from the fine-tunable catalog (1..300 chars). Supervised jobs accept only the curated fine-tunable list (400 \"This model isn't available for fine-tuning. Pick one from the list.\"); spec-draft jobs need a model in the spec-draft catalog. camelCase alias baseModel also accepted (camelCase wins if both present).",
|
|
1312
|
+
"required": true
|
|
1313
|
+
},
|
|
1314
|
+
{
|
|
1315
|
+
"name": "training",
|
|
1316
|
+
"type": "object",
|
|
1317
|
+
"description": "Training data source (required unless training_file_id is given). Either {kind:\"file\", file_id: string (a provider_file_id from /v1/fine_tuning/files; alias fileId)} or {kind:\"dataset\", provider_dataset_id: string (alias providerDatasetId; a workspace dataset's provider id), version?: string (≤200), mapping: <column mapping>}. mapping is one of: {type:\"text\", text:{type:\"column\", name}} | {type:\"prompts\", prompt:{type:\"column\", name}, completion:{type:\"column\", name}} | {type:\"messages\", messages:{type:\"column\", name}} | {type:\"pretokenized\", input_ids:{type:\"column\", name}, labels?:{type:\"column\", name}, attention_mask?:{type:\"column\", name}}. Datasets are converted to a training file after the wallet gate. The file/dataset MUST belong to this workspace (404 \"Training file not found\" / \"Dataset not found\" otherwise)."
|
|
1318
|
+
},
|
|
1319
|
+
{
|
|
1320
|
+
"name": "validation",
|
|
1321
|
+
"type": "object",
|
|
1322
|
+
"description": "Optional held-out/validation data source, same shape as training ({kind:\"file\", file_id} or {kind:\"dataset\", provider_dataset_id, version?, mapping}). Providing one is what makes a later bake-off (POST /v1/fine_tuning/jobs/{id}/bakeoff) possible."
|
|
1323
|
+
},
|
|
1324
|
+
{
|
|
1325
|
+
"name": "training_file_id",
|
|
1326
|
+
"type": "string",
|
|
1327
|
+
"description": "Legacy shortcut: a provider_file_id to train on (1..500 chars); equivalent to training:{kind:\"file\", file_id}. Ignored when training is present. Alias trainingFileId."
|
|
1328
|
+
},
|
|
1329
|
+
{
|
|
1330
|
+
"name": "validation_file_id",
|
|
1331
|
+
"type": "string",
|
|
1332
|
+
"description": "Legacy shortcut for validation:{kind:\"file\", file_id}. Alias validationFileId."
|
|
1333
|
+
},
|
|
1334
|
+
{
|
|
1335
|
+
"name": "name",
|
|
1336
|
+
"type": "string",
|
|
1337
|
+
"description": "Display name for the job (≤300 chars)."
|
|
1338
|
+
},
|
|
1339
|
+
{
|
|
1340
|
+
"name": "suffix",
|
|
1341
|
+
"type": "string",
|
|
1342
|
+
"description": "Suffix appended to the fine-tuned model name (≤120 chars)."
|
|
1343
|
+
},
|
|
1344
|
+
{
|
|
1345
|
+
"name": "seed",
|
|
1346
|
+
"type": "integer",
|
|
1347
|
+
"description": "Training seed, integer 0..2147483647."
|
|
1348
|
+
},
|
|
1349
|
+
{
|
|
1350
|
+
"name": "method",
|
|
1351
|
+
"type": "string",
|
|
1352
|
+
"description": "\"supervised\" (SFT / LoRA; default) or \"spec-draft\" (train a draft speculator for speculative decoding).",
|
|
1353
|
+
"enum": [
|
|
1354
|
+
"supervised",
|
|
1355
|
+
"spec-draft"
|
|
1356
|
+
],
|
|
1357
|
+
"default": "supervised"
|
|
1358
|
+
},
|
|
1359
|
+
{
|
|
1360
|
+
"name": "hyperparameters",
|
|
1361
|
+
"type": "object",
|
|
1362
|
+
"description": "Supervised hyperparameters, all optional and bounds-checked (400 naming the field otherwise): n_epochs (int 1..100), learning_rate (number >0 and ≤1), batch_size (int 1..1024), context_length (int 128..262144), warmup_ratio (0..1), weight_decay (0..1), packing (boolean), max_grad_norm (>0 and ≤1000), lora (boolean; some bases are full-parameter only → 400 \"… supports full-parameter fine-tuning only\"), lora_r (int 1..512), lora_alpha (int 1..1024), lora_dropout (0..1). Keys are snake_case only."
|
|
1363
|
+
},
|
|
1364
|
+
{
|
|
1365
|
+
"name": "spec_draft_hyperparameters",
|
|
1366
|
+
"type": "object",
|
|
1367
|
+
"description": "Spec-draft hyperparameters (used when method=\"spec-draft\"): the common fields n_epochs, learning_rate, batch_size, context_length, warmup_ratio, weight_decay, packing, max_grad_norm (same bounds as above) plus architecture (string ≤200), num_decoding_heads (int 1..16), loss (string ≤100). Alias specDraftHyperparameters."
|
|
1368
|
+
},
|
|
1369
|
+
{
|
|
1370
|
+
"name": "integrations",
|
|
1371
|
+
"type": "array",
|
|
1372
|
+
"description": "Up to 10 export integrations; ONLY these two types are accepted (anything else is 400): {type:\"wandb\", wandb:{project (1..200), api_key (1..500), name? (≤200), entity? (≤200), tags? (≤50 strings ≤100)}} or {type:\"hf\", hf:{output_repo_name (1..200), api_token (1..500)}}.",
|
|
1373
|
+
"items": "string"
|
|
1374
|
+
}
|
|
1375
|
+
],
|
|
1376
|
+
"responseSummary": "201 {id (job id for all other /v1/fine_tuning/jobs/{id} calls), provider_job_id}. Poll GET /v1/fine_tuning/jobs/{id} for status and fine_tuned_model.",
|
|
1377
|
+
"notes": "MONEY: the wallet must hold a prepay runway of (1,000,000 estimated trained tokens × the model's per-token rate incl. markup) or the call fails 402 \"Insufficient balance: starting a fine-tune requires at least $X of runway. Top up and try again.\"; the final charge is metered from real trained tokens on completion (billed_cost_usd). Requires OWNER/ADMIN minting user (403). Rate limited per workspace: 20 creates per 60s → 429 with Retry-After. 400 \"Invalid JSON body\" or \"Invalid body: <path> — <zod message>\" (e.g. missing training data: pass training or training_file_id). 400 when no price is configured for the model (\"No fine-tuning price is set for this model yet.\"). Top-level keys accept both snake_case and camelCase; nested hyperparameter/integration/mapping keys are snake_case only. Feature-flag gated (404 when fineTuning is off). Scopes not enforced on this route today.",
|
|
1378
|
+
"spends": true
|
|
1379
|
+
},
|
|
1380
|
+
{
|
|
1381
|
+
"name": "get_fine_tuning_job",
|
|
1382
|
+
"method": "GET",
|
|
1383
|
+
"path": "/v1/fine_tuning/jobs/{id}",
|
|
1384
|
+
"summary": "Get one fine-tuning job's live status, step progress, trained tokens, output model name, deployment state and billed cost — poll this after creating a job.",
|
|
1385
|
+
"scope": "read",
|
|
1386
|
+
"pathParams": [
|
|
1387
|
+
{
|
|
1388
|
+
"name": "id",
|
|
1389
|
+
"type": "string",
|
|
1390
|
+
"description": "The job id returned by POST /v1/fine_tuning/jobs (not the provider_job_id)."
|
|
1391
|
+
}
|
|
1392
|
+
],
|
|
1393
|
+
"responseSummary": "{id, provider_job_id, name, base_model, method, status (VALIDATING_FILES|QUEUED|RUNNING|SUCCEEDED|FAILED|CANCELLED), fine_tuned_model|null, deployed_model_name|null, deploy_status|null, deploy_error|null, trained_tokens (string)|null, trained_steps|null, total_steps|null, rate_per_m_token_usd, billed_cost_usd|null, error|null, created_at}.",
|
|
1394
|
+
"notes": "404 \"Job not found\" when the job is not in this workspace. Live progress (trained_steps/total_steps) is fetched from the training backend best-effort; DB state is served if that fails. A SUCCEEDED job is trainable-not-servable until deployed (deployed_model_name stays null). Feature-flag gated (404 when fineTuning is off). Read-only, no spend. Scopes not enforced on this route today."
|
|
1395
|
+
},
|
|
1396
|
+
{
|
|
1397
|
+
"name": "cancel_fine_tuning_job",
|
|
1398
|
+
"method": "DELETE",
|
|
1399
|
+
"path": "/v1/fine_tuning/jobs/{id}",
|
|
1400
|
+
"summary": "Cancel a queued or running fine-tuning job so no more training is metered — the customer's cancel is authoritative even if the backend lags.",
|
|
1401
|
+
"scope": "platform:write",
|
|
1402
|
+
"pathParams": [
|
|
1403
|
+
{
|
|
1404
|
+
"name": "id",
|
|
1405
|
+
"type": "string",
|
|
1406
|
+
"description": "The job id."
|
|
1407
|
+
}
|
|
1408
|
+
],
|
|
1409
|
+
"responseSummary": "200 {ok:true}. The job's status becomes CANCELLED (idempotent: cancelling an already-CANCELLED job also returns 200 {ok:true}).",
|
|
1410
|
+
"notes": "This is a CANCEL, not a delete — the job record remains listed. Requires OWNER/ADMIN minting user (403). 404 when not in this workspace. 400 \"This run already finished — there is nothing to cancel.\" for SUCCEEDED or FAILED jobs. The backend cancel is attempted but a backend error does not block the local cancel; metering treats local CANCELLED as final and finalizes at $0 further spend. Feature-flag gated (404 when fineTuning is off). Scopes not enforced on this route today."
|
|
1411
|
+
},
|
|
1412
|
+
{
|
|
1413
|
+
"name": "get_fine_tuning_bakeoff",
|
|
1414
|
+
"method": "GET",
|
|
1415
|
+
"path": "/v1/fine_tuning/jobs/{id}/bakeoff",
|
|
1416
|
+
"summary": "Read a fine-tune's bake-off state, verdict (improved / regressed / inconclusive with NLL, perplexity and optional judged pass rates) and ledger-true spend — to decide whether the tuned model is worth deploying.",
|
|
1417
|
+
"scope": "read",
|
|
1418
|
+
"pathParams": [
|
|
1419
|
+
{
|
|
1420
|
+
"name": "id",
|
|
1421
|
+
"type": "string",
|
|
1422
|
+
"description": "The fine-tuning job id."
|
|
1423
|
+
}
|
|
1424
|
+
],
|
|
1425
|
+
"responseSummary": "{status (\"none\"|\"queued\"|\"running\"|\"done\"|\"failed\"), error|null (customer-safe reason when failed), holdout_present (false = no validation split, so a comparison cannot be offered), available (platform compute configured), estimated_max_usd|null (the consent ceiling a start would hold; null when no GPU rate is configured), verdict|null: {verdict (\"improved\"|\"regressed\"|\"inconclusive\", sign-test backed), nll_base, nll_tuned, win_count, total, ppl_base, ppl_tuned, judged?: {criterion_id, criterion_name, base_pass_rate, tuned_pass_rate, scored}|null}, spent_usd}.",
|
|
1426
|
+
"notes": "Unlike other GETs this one requires an OWNER/ADMIN minting user (403 otherwise) because it reads spend. 404 \"Run not found\" when the job is not in this workspace. status \"none\" with holdout_present=false means the run can never be compared (no held-out split). Feature-flag gated (404 when fineTuning is off). Read-only, no spend."
|
|
1427
|
+
},
|
|
1428
|
+
{
|
|
1429
|
+
"name": "start_fine_tuning_bakeoff",
|
|
1430
|
+
"method": "POST",
|
|
1431
|
+
"path": "/v1/fine_tuning/jobs/{id}/bakeoff",
|
|
1432
|
+
"summary": "Start a held-out bake-off that proves a succeeded supervised fine-tune against its base model (teacher-forced NLL/perplexity wins, optionally a judged win-rate) on an ephemeral GPU box — quality proof without deploying the model.",
|
|
1433
|
+
"scope": "platform:write",
|
|
1434
|
+
"pathParams": [
|
|
1435
|
+
{
|
|
1436
|
+
"name": "id",
|
|
1437
|
+
"type": "string",
|
|
1438
|
+
"description": "The fine-tuning job id (must be method \"supervised\", status SUCCEEDED, and have a validation/held-out split)."
|
|
1439
|
+
}
|
|
1440
|
+
],
|
|
1441
|
+
"body": [
|
|
1442
|
+
{
|
|
1443
|
+
"name": "judge_criterion_id",
|
|
1444
|
+
"type": "string",
|
|
1445
|
+
"description": "Optional id of a calibrated (aligned, request-unit) workspace criterion; adds a judged pass-rate comparison (base vs tuned, up to 100 generated answers per side) next to the objective NLL signal. Must be a string if present (400 otherwise). The body may be empty."
|
|
1446
|
+
}
|
|
1447
|
+
],
|
|
1448
|
+
"responseSummary": "202 {trigger_run_id} — the comparison is queued; poll GET /v1/fine_tuning/jobs/{id}/bakeoff for status and verdict.",
|
|
1449
|
+
"notes": "MONEY: metered GPU-box minutes (plus judge calls) bill to the wallet under the bakeoff:<jobId>: ledger prefix; a wallet HOLD for the whole cost ceiling (max 3 hours × up to 2 GPUs at the reference GPU rate, with markup) is placed before starting — 402 (\"…Top up and try again.\") if the wallet cannot hold it; only metered minutes are actually billed and the hold is released at the end. Requires OWNER/ADMIN minting user (403). Rate limited per workspace: 20 starts per 60s → 429. 404 \"Run not found\". 400 for: a spec-draft job, a job not SUCCEEDED, no held-out split (\"A comparison needs a held-out split the model didn't train on — this run has none.\"), a comparison already queued/running, comparison compute or orchestration not configured on the platform, a base model whose parameter count can't be parsed from its name, or a base model over 75B parameters. Invalid JSON body → 400. Feature-flag gated (404 when fineTuning is off). Scopes not enforced on this route today.",
|
|
1450
|
+
"spends": true
|
|
1451
|
+
},
|
|
1452
|
+
{
|
|
1453
|
+
"name": "list_grpo_runs",
|
|
1454
|
+
"method": "GET",
|
|
1455
|
+
"path": "/v1/grpo/runs",
|
|
1456
|
+
"summary": "List the workspace's online-RL (GRPO) training runs with ledger-true spend and outcomes, plus how many self-improvement candidates are waiting in the queue — use it to monitor training and decide whether to start another run.",
|
|
1457
|
+
"scope": "read",
|
|
1458
|
+
"responseSummary": "JSON object: { candidates_waiting: integer, auto_provision_available: boolean, runs: [ { id, status (ACTIVE|STOPPED|COMPLETED|FAILED|OVERBUDGET), model, budget_usd, spent_usd (reward/judge spend), gpu_spent_usd, env_spent_usd, gpu_hour_budget: number|null, gpu_rate_usd_per_hour: number|null, created_at (ISO), outcome: null | { steps?, first_half_mean_reward?, second_half_mean_reward?, stopped_by_tripwire?, bakeoff?: { verdict, delta, delta_ci95: [lo, hi], prompts, k, mean_sim_fraction, mean_tool_steps } } } ] }. Newest first, at most 50 runs; outcome is only fetched for the 10 newest non-ACTIVE runs (older ones return outcome null).",
|
|
1459
|
+
"notes": "Feature-flag gated: the entire training API (fineTuning flag) returns 404 'Fine-tuning is not enabled' when the flag is off. The key's minting user must be workspace OWNER/ADMIN or the call is 403. Spend figures come from the billing ledger, never self-reported. No pagination parameters."
|
|
1460
|
+
},
|
|
1461
|
+
{
|
|
1462
|
+
"name": "start_grpo_run",
|
|
1463
|
+
"method": "POST",
|
|
1464
|
+
"path": "/v1/grpo/runs",
|
|
1465
|
+
"summary": "Start an online-RL (GRPO) training run that improves a fine-tunable base model against a calibrated judge as the reward, with hard reward and GPU-hour budgets — use it to turn logged traffic or the candidate queue into a trained adapter.",
|
|
1466
|
+
"scope": "platform:write",
|
|
1467
|
+
"body": [
|
|
1468
|
+
{
|
|
1469
|
+
"name": "reward",
|
|
1470
|
+
"type": "object",
|
|
1471
|
+
"description": "Reward spec (camelCase keys). Either { mode: \"single\", criterionId: string } or { mode: \"compositional\", criterionIds: string[] (min 1), assertions?: object[] }. Each assertion is { type: \"json_valid\"|\"json_schema\"|\"regex_match\"|\"contains\"|\"not_contains\"|\"max_length\"|\"min_length\"|\"completed\"|\"tool_called\"|\"no_tool_call\", value?: string } or an exec assertion { type: \"exec\", command: string, timeoutSec?: integer (1..120) } — exec assertions are only allowed when `environment` is set (agentic/trace-unit rewards). Every criterion must exist in the workspace and be calibrated; a non-agentic run requires request-unit criteria, an agentic run (with `environment`) requires trace-unit criteria.",
|
|
1472
|
+
"required": true
|
|
1473
|
+
},
|
|
1474
|
+
{
|
|
1475
|
+
"name": "model",
|
|
1476
|
+
"type": "string",
|
|
1477
|
+
"description": "The policy model to train. Must be on the platform's fine-tunable base-model list, otherwise 400.",
|
|
1478
|
+
"required": true
|
|
1479
|
+
},
|
|
1480
|
+
{
|
|
1481
|
+
"name": "promptTag",
|
|
1482
|
+
"type": "string",
|
|
1483
|
+
"description": "Draw training prompts only from logged requests carrying this tag. Omit to sample the whole workspace's successful logged traffic."
|
|
1484
|
+
},
|
|
1485
|
+
{
|
|
1486
|
+
"name": "promptCount",
|
|
1487
|
+
"type": "integer",
|
|
1488
|
+
"description": "Number of prompts to train on. Minimum 10 (400 below that); clamped to 1..10000 at scheduling. A ~20% holdout (min 3) is carved off on top, and the workspace must have promptCount+holdout matching prompts logged or the run is refused with the real counts.",
|
|
1489
|
+
"required": true
|
|
1490
|
+
},
|
|
1491
|
+
{
|
|
1492
|
+
"name": "groupSize",
|
|
1493
|
+
"type": "integer",
|
|
1494
|
+
"description": "Rollouts sampled per prompt. Default 8, clamped 2..16."
|
|
1495
|
+
},
|
|
1496
|
+
{
|
|
1497
|
+
"name": "maxSteps",
|
|
1498
|
+
"type": "integer",
|
|
1499
|
+
"description": "Training steps. Default 100, clamped 1..5000."
|
|
1500
|
+
},
|
|
1501
|
+
{
|
|
1502
|
+
"name": "rewardBudgetUsd",
|
|
1503
|
+
"type": "number",
|
|
1504
|
+
"description": "Hard cap (USD) on judge/reward spend. Must be > 0. Held on the wallet at start.",
|
|
1505
|
+
"required": true
|
|
1506
|
+
},
|
|
1507
|
+
{
|
|
1508
|
+
"name": "gpuHourBudget",
|
|
1509
|
+
"type": "number",
|
|
1510
|
+
"description": "Hard cap on GPU hours. Must be > 0. For platform-provisioned GPUs the hours x frozen marked-up rate are held on the wallet at start.",
|
|
1511
|
+
"required": true
|
|
1512
|
+
},
|
|
1513
|
+
{
|
|
1514
|
+
"name": "useCandidateQueue",
|
|
1515
|
+
"type": "boolean",
|
|
1516
|
+
"description": "Train on the workspace's GRPO candidate queue (the self-improvement give-up set) for the single reward criterion instead of a tag sample. Only effective with reward.mode=single."
|
|
1517
|
+
},
|
|
1518
|
+
{
|
|
1519
|
+
"name": "platformGpu",
|
|
1520
|
+
"type": "object",
|
|
1521
|
+
"description": "Bill a platform-provisioned GPU: { gpuType: string, region: string, gpuCount?: integer (clamped 1..8) }. Rate + markup are frozen at start. Omit for bring-your-own compute (no GPU billing). 400 if no price is set for that GPU/region."
|
|
1522
|
+
},
|
|
1523
|
+
{
|
|
1524
|
+
"name": "allowSideEffects",
|
|
1525
|
+
"type": "boolean",
|
|
1526
|
+
"description": "Agentic runs only: permit calls to tools not declared read-only. Default false."
|
|
1527
|
+
},
|
|
1528
|
+
{
|
|
1529
|
+
"name": "environment",
|
|
1530
|
+
"type": "object",
|
|
1531
|
+
"description": "Agentic mode — run episodes inside the tool environment: { proxy_base_url: string (snake_case, required), max_steps?: integer, simulate?: boolean }. Setting this forces useVllm=true and requires trace-unit reward criteria."
|
|
1532
|
+
},
|
|
1533
|
+
{
|
|
1534
|
+
"name": "useVllm",
|
|
1535
|
+
"type": "boolean",
|
|
1536
|
+
"description": "Colocated vLLM rollouts (much faster steps). Forced true when `environment` is set."
|
|
1537
|
+
},
|
|
1538
|
+
{
|
|
1539
|
+
"name": "vllmGpuMemoryUtilization",
|
|
1540
|
+
"type": "number",
|
|
1541
|
+
"description": "Clamped 0.05..0.9."
|
|
1542
|
+
},
|
|
1543
|
+
{
|
|
1544
|
+
"name": "qlora",
|
|
1545
|
+
"type": "boolean",
|
|
1546
|
+
"description": "QLoRA 4-bit training. Default true."
|
|
1547
|
+
},
|
|
1548
|
+
{
|
|
1549
|
+
"name": "paramsB",
|
|
1550
|
+
"type": "number",
|
|
1551
|
+
"description": "Parameter count (billions) override for models whose name doesn't carry it. Must be > 0; capped at 1000."
|
|
1552
|
+
},
|
|
1553
|
+
{
|
|
1554
|
+
"name": "autoProvision",
|
|
1555
|
+
"type": "boolean",
|
|
1556
|
+
"description": "Let the platform provision an auto-sized GPU box. 400 if provisioning isn't configured; holds a conservative GPU commitment (highest active rate x 8 GPUs x gpuHourBudget) on the wallet."
|
|
1557
|
+
},
|
|
1558
|
+
{
|
|
1559
|
+
"name": "curriculumMixRatio",
|
|
1560
|
+
"type": "number",
|
|
1561
|
+
"description": "Candidate-queue runs: fraction of the training slice drawn from regular successful traffic. Clamped 0..0.9."
|
|
1562
|
+
},
|
|
1563
|
+
{
|
|
1564
|
+
"name": "holdoutCount",
|
|
1565
|
+
"type": "number",
|
|
1566
|
+
"description": "Holdout size override (rounded, clamped 1..2000). Use 50+ for a real bake-off claim. Default max(3, ceil(promptCount*0.2))."
|
|
1567
|
+
},
|
|
1568
|
+
{
|
|
1569
|
+
"name": "maxCompletionTokens",
|
|
1570
|
+
"type": "integer",
|
|
1571
|
+
"description": "Per-rollout generated-token budget, clamped 256..32768. Defaults: 8192 agentic, 1024 single-turn."
|
|
1572
|
+
},
|
|
1573
|
+
{
|
|
1574
|
+
"name": "tasksInline",
|
|
1575
|
+
"type": "array",
|
|
1576
|
+
"description": "Up to 10000 task objects for agentic runs: { goal: any[] (min 1), image?: string (<=500 chars), recorded?: any[], verifier?: [ { command: string (1..4000 chars), timeout_sec?: integer 1..120 } ] }.",
|
|
1577
|
+
"items": "object"
|
|
1578
|
+
},
|
|
1579
|
+
{
|
|
1580
|
+
"name": "autoAdopt",
|
|
1581
|
+
"type": "object",
|
|
1582
|
+
"description": "Opt-in auto-deploy on an 'improved' bake-off verdict: { aliasName: string (1..120, must already exist in the workspace or the start is refused), canaryPercent?: integer 1..50, gpuType: string, region: string, templateFlavor?: string }. Deploys to a dedicated endpoint, canaries on the alias, and the online gate earns the promote. Inconclusive/regressed rounds never deploy."
|
|
1583
|
+
}
|
|
1584
|
+
],
|
|
1585
|
+
"responseSummary": "201 with { trigger_run_id: string } — the orchestration handle for the run (the GrpoRun id shows up in GET /v1/grpo/runs once registered).",
|
|
1586
|
+
"notes": "SPENDS MONEY: the whole commitment (rewardBudgetUsd + GPU hours at the frozen marked-up rate, or a conservative ceiling for autoProvision) is atomically HELD on the wallet at start; 402 when the wallet can't hold it; 400 'No wallet for this workspace' when there is no payment method. Body keys are camelCase only (except environment.proxy_base_url / max_steps and verifier[].timeout_sec, which are snake_case); unknown keys pass through. Zod validation failure returns 400 { error: 'Invalid body: <path> — <message>' } (flat error shape). Rate limited to 20 starts/min per workspace (429). Feature-flag gated (fineTuning flag off → 404). Requires OWNER/ADMIN (403). Other 400 refusals: unaligned/wrong-unit/drift-flagged judge, iterated-RL round gate (fresh grades needed on a self-trained policy), model not fine-tunable, promptCount < 10, not enough logged prompts or queued candidates, autoAdopt alias missing, no GPU price set, orchestration not configured.",
|
|
1587
|
+
"spends": true
|
|
1588
|
+
},
|
|
1589
|
+
{
|
|
1590
|
+
"name": "get_grpo_run",
|
|
1591
|
+
"method": "GET",
|
|
1592
|
+
"path": "/v1/grpo/runs/{id}",
|
|
1593
|
+
"summary": "Fetch one online-RL (GRPO) run's status, ledger-true spend, and training outcome (reward trend, bake-off verdict) — use it to poll a run you started.",
|
|
1594
|
+
"scope": "read",
|
|
1595
|
+
"pathParams": [
|
|
1596
|
+
{
|
|
1597
|
+
"name": "id",
|
|
1598
|
+
"type": "string",
|
|
1599
|
+
"description": "The GRPO run id (from the runs list). Scoped to the workspace: a foreign or unknown id is 404."
|
|
1600
|
+
}
|
|
1601
|
+
],
|
|
1602
|
+
"responseSummary": "JSON object: { id, status, model, budget_usd, spent_usd, gpu_spent_usd, env_spent_usd, gpu_hour_budget, gpu_rate_usd_per_hour, created_at, outcome: null | { steps, first_half_mean_reward, second_half_mean_reward, stopped_by_tripwire, bakeoff?: { verdict, delta, delta_ci95, prompts, k, mean_sim_fraction, mean_tool_steps } } }. Outcome is always attempted for this single run (null while ACTIVE or when no artifact exists).",
|
|
1603
|
+
"notes": "Feature-flag gated (fineTuning flag off → 404). OWNER/ADMIN key required (403). 404 'Run not found' for foreign ids."
|
|
1604
|
+
},
|
|
1605
|
+
{
|
|
1606
|
+
"name": "stop_grpo_run",
|
|
1607
|
+
"method": "POST",
|
|
1608
|
+
"path": "/v1/grpo/runs/{id}/stop",
|
|
1609
|
+
"summary": "Stop an ACTIVE online-RL (GRPO) run and release its wallet hold immediately — use it to cut a run short when spend or results aren't what you expected.",
|
|
1610
|
+
"scope": "platform:write",
|
|
1611
|
+
"pathParams": [
|
|
1612
|
+
{
|
|
1613
|
+
"name": "id",
|
|
1614
|
+
"type": "string",
|
|
1615
|
+
"description": "The GRPO run id. Must be ACTIVE and belong to the workspace."
|
|
1616
|
+
}
|
|
1617
|
+
],
|
|
1618
|
+
"responseSummary": "200 { ok: true }. The run flips to STOPPED; the reward server refuses further scoring and the orchestrator exits on its next sweep.",
|
|
1619
|
+
"notes": "Idempotency: a run that is not ACTIVE (already stopped/completed) or not in the workspace returns 404 'Run not found or not active'. Money: the commitment hold is released right away; already-metered spend stays billed. Feature-flag gated (fineTuning flag off → 404). OWNER/ADMIN key required (403). No body is read."
|
|
1620
|
+
},
|
|
1621
|
+
{
|
|
1622
|
+
"name": "get_grpo_run_weights",
|
|
1623
|
+
"method": "GET",
|
|
1624
|
+
"path": "/v1/grpo/runs/{id}/weights",
|
|
1625
|
+
"summary": "Get short-lived presigned download links for a finished online-RL (GRPO) run's trained adapter files so you can self-host the weights — use it after a run completes (or stops with a partial checkpoint).",
|
|
1626
|
+
"scope": "read",
|
|
1627
|
+
"pathParams": [
|
|
1628
|
+
{
|
|
1629
|
+
"name": "id",
|
|
1630
|
+
"type": "string",
|
|
1631
|
+
"description": "The GRPO run id. Must be in a terminal status (COMPLETED, STOPPED, or FAILED)."
|
|
1632
|
+
}
|
|
1633
|
+
],
|
|
1634
|
+
"responseSummary": "JSON object: { run_id, status, partial: boolean (true for STOPPED/FAILED — files are a partial checkpoint, not the finished adapter), files: [ { name (e.g. adapter_model.safetensors), size_bytes: integer|null, url (presigned GET, valid 15 minutes), expires_at (ISO) } ], empty_reason?: string (present when files is empty), storage_unavailable?: true (weight storage not configured — try later) }. Files sorted safetensors first, then adapter files, then config/tokenizer.",
|
|
1635
|
+
"notes": "400 while the run is ACTIVE ('Weights are available once the run finishes.') and for OVERBUDGET runs ('This run has no trained adapter to download.'). 404 for foreign/unknown runs. Links expire after 15 minutes — re-call to refresh. Every call is audit-logged as a weight export. Feature-flag gated (fineTuning flag off → 404). OWNER/ADMIN key required (403)."
|
|
1636
|
+
},
|
|
1637
|
+
{
|
|
1638
|
+
"name": "list_label_sets",
|
|
1639
|
+
"method": "GET",
|
|
1640
|
+
"path": "/v1/label_sets",
|
|
1641
|
+
"summary": "List the workspace's golden sets — named collections of human-graded requests with their measured label quality (inter-rater kappa) — use it to see which sets exist, which are frozen, and which judges calibrate on them.",
|
|
1642
|
+
"scope": "read",
|
|
1643
|
+
"responseSummary": "JSON array (bare array, newest first) of { id, name, description, size: integer, membership_hash: string|null, frozen_at: ISO|null, kappa: number|null, agreement: number|null, kappa_n: integer|null, rater_count: integer|null, attached_to: [ { id, name } ] (criteria calibrating on this set), created_at }.",
|
|
1644
|
+
"notes": "Returns a bare JSON array, not a { object: 'list' } envelope. kappa is null until the set is frozen, and stays null after freezing when no blind re-grades by a second rater exist inside the set."
|
|
1645
|
+
},
|
|
1646
|
+
{
|
|
1647
|
+
"name": "create_label_set",
|
|
1648
|
+
"method": "POST",
|
|
1649
|
+
"path": "/v1/label_sets",
|
|
1650
|
+
"summary": "Create a golden set from human-graded requests — either explicit request ids or the newest N grades — as the first step toward a frozen, kappa-measured calibration set for a judge.",
|
|
1651
|
+
"scope": "evals:write",
|
|
1652
|
+
"body": [
|
|
1653
|
+
{
|
|
1654
|
+
"name": "name",
|
|
1655
|
+
"type": "string",
|
|
1656
|
+
"description": "Set name, trimmed, 1..80 chars (400 otherwise).",
|
|
1657
|
+
"required": true
|
|
1658
|
+
},
|
|
1659
|
+
{
|
|
1660
|
+
"name": "description",
|
|
1661
|
+
"type": "string",
|
|
1662
|
+
"description": "Optional description; trimmed and truncated to 500 chars."
|
|
1663
|
+
},
|
|
1664
|
+
{
|
|
1665
|
+
"name": "request_ids",
|
|
1666
|
+
"type": "array",
|
|
1667
|
+
"description": "Explicit members. Every id must carry a human (non-verifier) grade in this workspace, otherwise 400 naming how many are missing. Deduplicated. Takes precedence over `latest` when non-empty.",
|
|
1668
|
+
"items": "string"
|
|
1669
|
+
},
|
|
1670
|
+
{
|
|
1671
|
+
"name": "latest",
|
|
1672
|
+
"type": "integer",
|
|
1673
|
+
"description": "When request_ids is absent/empty: take the newest N human grades. Default 200; clamped to 20..5000."
|
|
1674
|
+
}
|
|
1675
|
+
],
|
|
1676
|
+
"responseSummary": "201 with the set object: { id, name, description, size, membership_hash: null, frozen_at: null, kappa: null, agreement: null, kappa_n: null, rater_count: null, attached_to: [], created_at }. The set is NOT frozen yet.",
|
|
1677
|
+
"notes": "A set needs at least 20 distinct graded requests (400 'A golden set needs at least 20 graded requests (have N)') and at most 5000. Verifier-sourced labels never count as members. Freeze the set (POST /v1/label_sets/{id}/freeze) before attaching it to a criterion."
|
|
1678
|
+
},
|
|
1679
|
+
{
|
|
1680
|
+
"name": "attach_label_set",
|
|
1681
|
+
"method": "POST",
|
|
1682
|
+
"path": "/v1/label_sets/{id}/attach",
|
|
1683
|
+
"summary": "Attach a frozen golden set to a judge criterion so its calibration runs on that set and its certificate carries the set's kappa — or detach it.",
|
|
1684
|
+
"scope": "evals:write",
|
|
1685
|
+
"pathParams": [
|
|
1686
|
+
{
|
|
1687
|
+
"name": "id",
|
|
1688
|
+
"type": "string",
|
|
1689
|
+
"description": "The golden set id. Must be frozen to attach; ignored when detach=true."
|
|
1690
|
+
}
|
|
1691
|
+
],
|
|
1692
|
+
"body": [
|
|
1693
|
+
{
|
|
1694
|
+
"name": "criterion_id",
|
|
1695
|
+
"type": "string",
|
|
1696
|
+
"description": "The judge criterion to attach the set to. Non-empty string required (400 'criterion_id is required'); 404 'Criterion not found' if not in the workspace.",
|
|
1697
|
+
"required": true
|
|
1698
|
+
},
|
|
1699
|
+
{
|
|
1700
|
+
"name": "detach",
|
|
1701
|
+
"type": "boolean",
|
|
1702
|
+
"description": "When exactly true, clears the criterion's golden set (sets label_set_id to null) instead of attaching {id}."
|
|
1703
|
+
}
|
|
1704
|
+
],
|
|
1705
|
+
"responseSummary": "200 { criterion_id: string, label_set_id: string|null } — the criterion's new golden-set binding.",
|
|
1706
|
+
"notes": "400 'Freeze the golden set first — an unfrozen set can change under the calibration.' when the set has no frozen_at. 404 'Golden set not found' for foreign/unknown set ids (when attaching). A criterion holds at most one golden set; attaching replaces the previous one."
|
|
1707
|
+
},
|
|
1708
|
+
{
|
|
1709
|
+
"name": "freeze_label_set",
|
|
1710
|
+
"method": "POST",
|
|
1711
|
+
"path": "/v1/label_sets/{id}/freeze",
|
|
1712
|
+
"summary": "Freeze a golden set: seal its membership with a hash and measure inter-rater agreement (kappa) on its members — required before a judge can calibrate on it.",
|
|
1713
|
+
"scope": "evals:write",
|
|
1714
|
+
"pathParams": [
|
|
1715
|
+
{
|
|
1716
|
+
"name": "id",
|
|
1717
|
+
"type": "string",
|
|
1718
|
+
"description": "The golden set id (workspace-scoped; 404 'Golden set not found' otherwise)."
|
|
1719
|
+
}
|
|
1720
|
+
],
|
|
1721
|
+
"responseSummary": "200 with the updated set object: { id, name, description, size, membership_hash (sha256 of sorted member ids), frozen_at (ISO), kappa: number|null, agreement: number|null, kappa_n: integer|null, rater_count: integer, attached_to: [ { id, name } ], created_at }.",
|
|
1722
|
+
"notes": "No body. Re-freezing an already-frozen set re-measures kappa/agreement/rater_count but keeps the original frozen_at and membership. kappa is null (and reported as null, never as good) when fewer than two raters' blind re-grades exist inside the set. 400 if inter-rater stats are unavailable."
|
|
1723
|
+
},
|
|
1724
|
+
{
|
|
1725
|
+
"name": "list_labels",
|
|
1726
|
+
"method": "GET",
|
|
1727
|
+
"path": "/v1/labels",
|
|
1728
|
+
"summary": "List the workspace's human pass/fail grades (ground-truth labels) on logged requests, newest first — use it to audit or export the verdicts every judge is measured against.",
|
|
1729
|
+
"scope": "read",
|
|
1730
|
+
"query": [
|
|
1731
|
+
{
|
|
1732
|
+
"name": "verdict",
|
|
1733
|
+
"type": "string",
|
|
1734
|
+
"description": "Filter to one verdict. Any other value is ignored (no filter).",
|
|
1735
|
+
"enum": [
|
|
1736
|
+
"pass",
|
|
1737
|
+
"fail"
|
|
1738
|
+
]
|
|
1739
|
+
},
|
|
1740
|
+
{
|
|
1741
|
+
"name": "limit",
|
|
1742
|
+
"type": "integer",
|
|
1743
|
+
"description": "Page size. Default 50, clamped 1..200. Non-numeric or 0 falls back to the default.",
|
|
1744
|
+
"default": 50
|
|
1745
|
+
},
|
|
1746
|
+
{
|
|
1747
|
+
"name": "offset",
|
|
1748
|
+
"type": "integer",
|
|
1749
|
+
"description": "Rows to skip (offset pagination). Default 0.",
|
|
1750
|
+
"default": 0
|
|
1751
|
+
}
|
|
1752
|
+
],
|
|
1753
|
+
"responseSummary": "JSON { object: 'list', total: integer (matching rows across all pages), data: [ { id, request_id, verdict ('pass'|'fail'), critique: string|null, source ('human'|'assist_accepted'|'verifier'), scope ('request'|'trace'), created_at (ISO) } ] }.",
|
|
1754
|
+
"notes": "Offset pagination: page through with offset += limit until offset >= total. Rows include labels of every source (human, assist_accepted, verifier)."
|
|
1755
|
+
},
|
|
1756
|
+
{
|
|
1757
|
+
"name": "create_label",
|
|
1758
|
+
"method": "POST",
|
|
1759
|
+
"path": "/v1/labels",
|
|
1760
|
+
"summary": "Record a human or downstream-system pass/fail verdict on a logged request (or on the whole agent run it belongs to) — this is the ground truth judge calibration, corrected pass rates, and training rewards are measured against.",
|
|
1761
|
+
"scope": "evals:write",
|
|
1762
|
+
"body": [
|
|
1763
|
+
{
|
|
1764
|
+
"name": "request_id",
|
|
1765
|
+
"type": "string",
|
|
1766
|
+
"description": "The gateway request id being graded (1..128 chars after trim). For scope=trace, send the run's FINAL-step request id. Missing/empty → 400.",
|
|
1767
|
+
"required": true
|
|
1768
|
+
},
|
|
1769
|
+
{
|
|
1770
|
+
"name": "verdict",
|
|
1771
|
+
"type": "string",
|
|
1772
|
+
"description": "The grade. WARNING: if omitted the route defaults to \"pass\" — always send it explicitly.",
|
|
1773
|
+
"required": true,
|
|
1774
|
+
"enum": [
|
|
1775
|
+
"pass",
|
|
1776
|
+
"fail"
|
|
1777
|
+
]
|
|
1778
|
+
},
|
|
1779
|
+
{
|
|
1780
|
+
"name": "critique",
|
|
1781
|
+
"type": "string",
|
|
1782
|
+
"description": "Why (max 2000 chars). Strongly encouraged on fails — becomes judge few-shot material and failure-taxonomy text. null allowed."
|
|
1783
|
+
},
|
|
1784
|
+
{
|
|
1785
|
+
"name": "scope",
|
|
1786
|
+
"type": "string",
|
|
1787
|
+
"description": "\"request\" (default) grades this one exchange; \"trace\" grades the whole agent run the request belongs to. Trace-unit judges calibrate only against trace-scoped labels. Unknown values are rejected with 400, never silently dropped.",
|
|
1788
|
+
"enum": [
|
|
1789
|
+
"request",
|
|
1790
|
+
"trace"
|
|
1791
|
+
]
|
|
1792
|
+
}
|
|
1793
|
+
],
|
|
1794
|
+
"responseSummary": "201 with the stored label: { id, request_id, verdict, critique, source ('human'), scope, fail_causes: string[], created_at }.",
|
|
1795
|
+
"notes": "Upsert by request_id: re-labeling the same request replaces the verdict/critique/scope (newest judgment wins); a 'pass' clears any prior failure attributions. Requires the key's minting user to be workspace OWNER/ADMIN (403) — labels define quality. Side effects: settles pending judge suspicions on the trace, fulfils pending recalibration-slice requests, and flags affected judges' calibrations for revision. Validation errors (e.g. bad scope, over-long critique) return 400 with the schema message."
|
|
1796
|
+
},
|
|
1797
|
+
{
|
|
1798
|
+
"name": "list_logs",
|
|
1799
|
+
"method": "GET",
|
|
1800
|
+
"path": "/v1/logs",
|
|
1801
|
+
"summary": "Browse the workspace's logged chat exchanges (request messages + assistant reply, secret-scrubbed) with filters for model, tag, auto-detected traffic segment, finish reason, cache hit and time range — use it to inspect real traffic before grading, building datasets, or running evals.",
|
|
1802
|
+
"scope": "read",
|
|
1803
|
+
"query": [
|
|
1804
|
+
{
|
|
1805
|
+
"name": "model",
|
|
1806
|
+
"type": "string",
|
|
1807
|
+
"description": "Exact model name filter."
|
|
1808
|
+
},
|
|
1809
|
+
{
|
|
1810
|
+
"name": "tag",
|
|
1811
|
+
"type": "string",
|
|
1812
|
+
"description": "Exact request tag filter."
|
|
1813
|
+
},
|
|
1814
|
+
{
|
|
1815
|
+
"name": "segment",
|
|
1816
|
+
"type": "string",
|
|
1817
|
+
"description": "Auto-detected traffic segment: the FAMILY of system prompts sharing one template (value of a row's `segment` field; the reserved value \"none\" is the no-system-prompt segment). Exact match."
|
|
1818
|
+
},
|
|
1819
|
+
{
|
|
1820
|
+
"name": "finish_reason",
|
|
1821
|
+
"type": "string",
|
|
1822
|
+
"description": "Exact finish-reason filter (e.g. stop, length, tool_calls)."
|
|
1823
|
+
},
|
|
1824
|
+
{
|
|
1825
|
+
"name": "cache_hit",
|
|
1826
|
+
"type": "boolean",
|
|
1827
|
+
"description": "\"true\" or \"false\" — filter to cached / uncached responses. Any other value = no filter.",
|
|
1828
|
+
"enum": [
|
|
1829
|
+
"true",
|
|
1830
|
+
"false"
|
|
1831
|
+
]
|
|
1832
|
+
},
|
|
1833
|
+
{
|
|
1834
|
+
"name": "start",
|
|
1835
|
+
"type": "integer",
|
|
1836
|
+
"description": "Inclusive lower bound, Unix seconds (positive integer; other values ignored)."
|
|
1837
|
+
},
|
|
1838
|
+
{
|
|
1839
|
+
"name": "end",
|
|
1840
|
+
"type": "integer",
|
|
1841
|
+
"description": "Upper bound, Unix seconds (positive integer; other values ignored)."
|
|
1842
|
+
},
|
|
1843
|
+
{
|
|
1844
|
+
"name": "limit",
|
|
1845
|
+
"type": "integer",
|
|
1846
|
+
"description": "Page size, clamped 1..100. Default 25.",
|
|
1847
|
+
"default": 25
|
|
1848
|
+
},
|
|
1849
|
+
{
|
|
1850
|
+
"name": "offset",
|
|
1851
|
+
"type": "integer",
|
|
1852
|
+
"description": "Rows to skip (offset pagination). Default 0.",
|
|
1853
|
+
"default": 0
|
|
1854
|
+
}
|
|
1855
|
+
],
|
|
1856
|
+
"responseSummary": "JSON { object: 'list', total: integer, limit, offset, data: [ { request_id, created_at (Unix seconds), model, tag, segment (prompt family id), segment_exact (exact system-prompt hash), trace_id, finish_reason, streamed: boolean, cache_hit: boolean, fallback_from: string|null, prompt_tokens, completion_tokens, messages: parsed JSON array of request messages (null if unparseable), response: parsed assistant message object (null if unparseable) } ] }. Newest first, successful (non-aborted) exchanges only.",
|
|
1857
|
+
"notes": "Request logging is opt-in per workspace: returns 409 { error: { message, type: 'invalid_request_error', code: 'logging_disabled' } } when it is off — an empty list would otherwise read as 'no traffic'. Offset pagination: repeat with offset += limit until offset >= total. Pass a row's `segment` back as ?segment= or into an eval's sample_filters.segment to slice by that application surface."
|
|
1858
|
+
},
|
|
1859
|
+
{
|
|
1860
|
+
"name": "export_logs",
|
|
1861
|
+
"method": "GET",
|
|
1862
|
+
"path": "/v1/logs/export",
|
|
1863
|
+
"summary": "Export the filtered logged exchanges as JSONL in chat format — one {\"messages\":[...]} line per exchange with the assistant reply appended — ready to pipe into your own training or eval tooling.",
|
|
1864
|
+
"scope": "read",
|
|
1865
|
+
"query": [
|
|
1866
|
+
{
|
|
1867
|
+
"name": "model",
|
|
1868
|
+
"type": "string",
|
|
1869
|
+
"description": "Exact model name filter."
|
|
1870
|
+
},
|
|
1871
|
+
{
|
|
1872
|
+
"name": "tag",
|
|
1873
|
+
"type": "string",
|
|
1874
|
+
"description": "Exact request tag filter."
|
|
1875
|
+
},
|
|
1876
|
+
{
|
|
1877
|
+
"name": "segment",
|
|
1878
|
+
"type": "string",
|
|
1879
|
+
"description": "Auto-detected traffic segment (prompt family) — exact match, same values as GET /v1/logs rows' `segment`."
|
|
1880
|
+
},
|
|
1881
|
+
{
|
|
1882
|
+
"name": "finish_reason",
|
|
1883
|
+
"type": "string",
|
|
1884
|
+
"description": "Exact finish-reason filter."
|
|
1885
|
+
},
|
|
1886
|
+
{
|
|
1887
|
+
"name": "cache_hit",
|
|
1888
|
+
"type": "boolean",
|
|
1889
|
+
"description": "\"true\" or \"false\".",
|
|
1890
|
+
"enum": [
|
|
1891
|
+
"true",
|
|
1892
|
+
"false"
|
|
1893
|
+
]
|
|
1894
|
+
},
|
|
1895
|
+
{
|
|
1896
|
+
"name": "start",
|
|
1897
|
+
"type": "integer",
|
|
1898
|
+
"description": "Inclusive lower bound, Unix seconds."
|
|
1899
|
+
},
|
|
1900
|
+
{
|
|
1901
|
+
"name": "end",
|
|
1902
|
+
"type": "integer",
|
|
1903
|
+
"description": "Upper bound, Unix seconds."
|
|
1904
|
+
}
|
|
1905
|
+
],
|
|
1906
|
+
"responseSummary": "200 with Content-Type application/jsonl; charset=utf-8. Body: newline-terminated lines, each {\"messages\": [ ...request messages, assistantReplyMessage ]}, newest first. Response headers: X-Omnia-Export-Count (lines written) and X-Omnia-Export-Capped ('true' when the 10,000-row cap was hit — narrow the filter, e.g. a time range, to get the rest).",
|
|
1907
|
+
"notes": "No limit/offset — the export is capped at 10,000 rows; use X-Omnia-Export-Capped to detect truncation. Rows whose stored JSON doesn't parse are skipped, never fail the export. 409 { error: string } (flat shape) when request logging is disabled for the workspace. Only successful (non-aborted) exchanges are exported.",
|
|
1908
|
+
"raw": true
|
|
1909
|
+
},
|
|
1910
|
+
{
|
|
1911
|
+
"name": "list_model_versions",
|
|
1912
|
+
"method": "GET",
|
|
1913
|
+
"path": "/v1/model_versions",
|
|
1914
|
+
"summary": "List the workspace's model-version chain — one immutable record per completed training round, pinning the judge, curriculum and holdout its verdict depended on — use it to review improvement history and pick a version to adopt or roll back to.",
|
|
1915
|
+
"scope": "read",
|
|
1916
|
+
"query": [
|
|
1917
|
+
{
|
|
1918
|
+
"name": "base_model",
|
|
1919
|
+
"type": "string",
|
|
1920
|
+
"description": "Filter to one base model's lineage (exact match)."
|
|
1921
|
+
}
|
|
1922
|
+
],
|
|
1923
|
+
"responseSummary": "JSON { object: 'list', data: [ { id, parent_id: string|null, base_model, artifact_ref, served_model: string|null (null = not deployed/servable yet), source_run_id, source_kind ('grpo'|'finetune'), verdict: any (bake-off verdict JSON), judge_criterion_id, curriculum_hash, holdout_hash, comparable_to_parent: boolean (true only when parent's holdout hash matches — otherwise treat the delta as a discontinuity), adopted_at: ISO|null, created_at: ISO } ] }. Newest first, at most 200.",
|
|
1924
|
+
"notes": "NOT feature-flag gated (deliberately readable even when training is paused, so the audit trail stays visible). The key's minting user must be workspace OWNER/ADMIN (403). No pagination beyond the 200 cap."
|
|
1925
|
+
},
|
|
1926
|
+
{
|
|
1927
|
+
"name": "get_model_version",
|
|
1928
|
+
"method": "GET",
|
|
1929
|
+
"path": "/v1/model_versions/{id}",
|
|
1930
|
+
"summary": "Fetch one model version's record (lineage, artifact, served model name, verdict, pinned hashes, adoption time) — use it to inspect a specific training round before adopting it.",
|
|
1931
|
+
"scope": "read",
|
|
1932
|
+
"pathParams": [
|
|
1933
|
+
{
|
|
1934
|
+
"name": "id",
|
|
1935
|
+
"type": "string",
|
|
1936
|
+
"description": "The model version id. Workspace-scoped: a foreign or unknown id is 404 'Version not found in this workspace.'"
|
|
1937
|
+
}
|
|
1938
|
+
],
|
|
1939
|
+
"responseSummary": "JSON object { id, parent_id, base_model, artifact_ref, served_model, source_run_id, source_kind, verdict, judge_criterion_id, curriculum_hash, holdout_hash, comparable_to_parent, adopted_at, created_at } — same shape as the list rows.",
|
|
1940
|
+
"notes": "Not feature-flag gated. OWNER/ADMIN key required (403)."
|
|
1941
|
+
},
|
|
1942
|
+
{
|
|
1943
|
+
"name": "adopt_model_version",
|
|
1944
|
+
"method": "POST",
|
|
1945
|
+
"path": "/v1/model_versions/{id}/adopt",
|
|
1946
|
+
"summary": "Point a model alias at a deployed model version — adoption and rollback are the same audited repoint on different rows of the chain — use it to promote a trained round into production or roll back to an earlier one.",
|
|
1947
|
+
"scope": "platform:write",
|
|
1948
|
+
"pathParams": [
|
|
1949
|
+
{
|
|
1950
|
+
"name": "id",
|
|
1951
|
+
"type": "string",
|
|
1952
|
+
"description": "The model version id to route traffic to. Must be in the workspace and have a served_model (deployed)."
|
|
1953
|
+
}
|
|
1954
|
+
],
|
|
1955
|
+
"body": [
|
|
1956
|
+
{
|
|
1957
|
+
"name": "aliasName",
|
|
1958
|
+
"type": "string",
|
|
1959
|
+
"description": "camelCase only. Name of an existing alias in the workspace (404 'Alias \"<name>\" not found.' otherwise). Non-empty string required (400).",
|
|
1960
|
+
"required": true
|
|
1961
|
+
}
|
|
1962
|
+
],
|
|
1963
|
+
"responseSummary": "200 { served_model: string } — the model name the alias now resolves to.",
|
|
1964
|
+
"notes": "MOVES PRODUCTION TRAFFIC: the alias's target is replaced and any live canary split on it is cleared (canary_model null, canary_percent 0). 400 'This version is not deployed yet — deploy its weights before routing traffic to it.' when served_model is null. 404 for a foreign version id or unknown alias. Feature-flag gated (fineTuning flag off → 404) unlike the GET routes. OWNER/ADMIN key required (403). Zod failure returns 400 { error: 'Invalid body: aliasName — ...' } (flat shape). Adoption stamps adopted_at on first adoption only; it is audit-logged."
|
|
1965
|
+
},
|
|
1966
|
+
{
|
|
1967
|
+
"name": "list_raft_rounds",
|
|
1968
|
+
"method": "GET",
|
|
1969
|
+
"path": "/v1/raft/rounds",
|
|
1970
|
+
"summary": "List the workspace's self-improvement (rejection-sampling fine-tuning) rounds, newest first, with winners/give-ups, budget and ledger-true spend — read-only observability for rounds started from the dashboard.",
|
|
1971
|
+
"scope": "read",
|
|
1972
|
+
"responseSummary": "JSON { rounds: [ { id, status, criterion_name, policy_model, prompt_count, candidates_per_prompt, winners_count, giveups_count, produced_job_id: string|null (the fine-tuning job a COMPLETED round produced), skip_reason: string|null (SKIPPED rounds), budget_usd: number|null, spent_usd: number, created_at: ISO, completed_at: ISO|null } ] }. At most 200 rounds.",
|
|
1973
|
+
"notes": "List-only: there is no public start endpoint for rounds. Feature-flag gated (fineTuning flag off → 404). OWNER/ADMIN key required (403). Returns { rounds: [] } (not an error) when the round history table hasn't been provisioned yet. spent_usd is 0 for rounds created before per-round budgets existed."
|
|
1974
|
+
},
|
|
1975
|
+
{
|
|
1976
|
+
"name": "get_judge_settings",
|
|
1977
|
+
"method": "GET",
|
|
1978
|
+
"path": "/v1/settings/judge",
|
|
1979
|
+
"summary": "Read the workspace's default judge model (used for eval runs that don't name their own judge) and the platform's house default.",
|
|
1980
|
+
"scope": "read",
|
|
1981
|
+
"responseSummary": "JSON { default_judge_model: string|null (null = house default / auto), house_default: string }. Sent with Cache-Control: no-store.",
|
|
1982
|
+
"notes": "Precedence at run time: a run's own judge_model > this workspace default > house_default. Screening still swaps a default that would judge its own sibling model."
|
|
1983
|
+
},
|
|
1984
|
+
{
|
|
1985
|
+
"name": "set_judge_settings",
|
|
1986
|
+
"method": "PUT",
|
|
1987
|
+
"path": "/v1/settings/judge",
|
|
1988
|
+
"summary": "Set (or clear) the workspace's default judge model for eval runs that don't specify one — must be a chat model from the platform catalog offered to this workspace.",
|
|
1989
|
+
"scope": "platform:write",
|
|
1990
|
+
"body": [
|
|
1991
|
+
{
|
|
1992
|
+
"name": "default_judge_model",
|
|
1993
|
+
"type": "string",
|
|
1994
|
+
"description": "Model id (e.g. \"openai/gpt-4.1\") to use as the default judge, or null / omitted / empty string to revert to the house default. Any non-string, non-null value → 400 'default_judge_model must be a string or null'. Must be a chat (non-embedding) model this workspace is offered, otherwise 400 '\"<model>\" isn't a chat model this workspace is offered.'"
|
|
1995
|
+
}
|
|
1996
|
+
],
|
|
1997
|
+
"responseSummary": "200 { default_judge_model: string|null, house_default: string } — the settings after the update.",
|
|
1998
|
+
"notes": "Judges run on the platform's key and are metered to the wallet, so only platform-catalog models qualify (a workspace's own provider keys are for inference, not judging). Needs platform:write (it is a workspace setting)."
|
|
1999
|
+
},
|
|
2000
|
+
{
|
|
2001
|
+
"name": "get_setup_status",
|
|
2002
|
+
"method": "GET",
|
|
2003
|
+
"path": "/v1/setup/status",
|
|
2004
|
+
"summary": "Answer 'where am I and what should I do next?' in one call — workspace identity, logging state, traffic and grade counts, judge calibration progress, and the single dependency-ordered next step; also the cheapest way to check that an API key is live and which workspace it belongs to.",
|
|
2005
|
+
"scope": "read",
|
|
2006
|
+
"responseSummary": "JSON (camelCase keys — this route does NOT snake-case): { workspace: { slug, name }, logging: { enabled: boolean, retentionDays: integer }, traffic: { loggedConversations: integer|null (null = log store unreachable, NOT zero traffic) }, grades: { total, neededToCalibrate (grades still short of the 30 required) }, judges: { total, calibrated, trustworthy, failGradesNeeded: integer|null }, next: { action: 'enable_logging'|'send_traffic'|'grade'|'create_judge'|'calibrate'|'grade_failures'|'recalibrate'|'compare', detail: string, href: string (dashboard path) } }. With request header Accept: text/plain the same data is returned as flat snake_case key=value lines (e.g. grades_total=12, next_action=grade), one per line.",
|
|
2007
|
+
"notes": "Never cached (Cache-Control: no-store). A 200 proves the key is valid; 401 otherwise. Field casing differs from every other /v1 route (camelCase in JSON, snake_case only in the text/plain form)."
|
|
2008
|
+
},
|
|
2009
|
+
{
|
|
2010
|
+
"name": "get_trace",
|
|
2011
|
+
"method": "GET",
|
|
2012
|
+
"path": "/v1/traces/{traceId}",
|
|
2013
|
+
"summary": "Fetch every logged step of one agent run or conversation (grouped by the X-Omnia-Trace-Id you sent), oldest-first in execution order and including aborted partials — use it for error analysis of a multi-step run.",
|
|
2014
|
+
"scope": "read",
|
|
2015
|
+
"pathParams": [
|
|
2016
|
+
{
|
|
2017
|
+
"name": "traceId",
|
|
2018
|
+
"type": "string",
|
|
2019
|
+
"description": "The trace id sent as X-Omnia-Trace-Id on the gateway requests. 404 'Trace not found' when no logged step carries it."
|
|
2020
|
+
}
|
|
2021
|
+
],
|
|
2022
|
+
"responseSummary": "JSON { object: 'list', trace_id, data: [ { request_id, created_at (Unix seconds), model, alias: string|null, tag, status ('SUCCESS'|'ABORTED'), finish_reason, streamed, cache_hit, fallback_from, prompt_tokens, completion_tokens, messages: parsed request messages (null if unparseable), response: parsed assistant message (null if unparseable) } ] } ordered oldest first.",
|
|
2023
|
+
"notes": "Requires request logging to be enabled — 409 { error: string } (flat shape) otherwise. Unlike /v1/logs this includes ABORTED partial rows (a run that died at step 4 is the finding). No pagination or filters."
|
|
2024
|
+
},
|
|
2025
|
+
{
|
|
2026
|
+
"name": "verify_document",
|
|
2027
|
+
"method": "POST",
|
|
2028
|
+
"path": "/v1/verify",
|
|
2029
|
+
"summary": "Verify that a downloaded certificate or evidence bundle was issued by the platform and has not been altered, by re-deriving its HMAC signature — use it when a third party hands you a document and you need to trust its numbers.",
|
|
2030
|
+
"scope": "read",
|
|
2031
|
+
"body": [
|
|
2032
|
+
{
|
|
2033
|
+
"name": "document",
|
|
2034
|
+
"type": "object",
|
|
2035
|
+
"description": "The full signed JSON document exactly as downloaded (a certificate or evidence bundle carrying signature: { alg: 'HS256', key_id, value }). The `document` key must be present (400 'Body must be { document: <signed JSON> }' otherwise); its value may be any JSON.",
|
|
2036
|
+
"required": true
|
|
2037
|
+
}
|
|
2038
|
+
],
|
|
2039
|
+
"responseSummary": "Always 200 for a well-formed body: { ok: true, key_id: string } when the bytes are ours and unaltered; otherwise { ok: false, reason: 'unsigned' (no signature field) | 'malformed' (not an object or signature shape wrong) | 'unknown_key' (signed by a key this platform doesn't hold, e.g. after rotation) | 'mismatch' (any field was edited) | 'no_secret' (verification not configured on the platform) }.",
|
|
2040
|
+
"notes": "Read scope suffices (POST that writes nothing); nothing is stored. Verification canonicalises the document (keys sorted recursively, undefined dropped) before hashing, so key order does not matter but any value change does. Sent with Cache-Control: no-store."
|
|
2041
|
+
}
|
|
2042
|
+
];
|