@withgateway/sdk 0.3.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +3005 -1401
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2996 -1392
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +128 -37
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +126 -35
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-mJhG3vZU.d.mts → index-Di9nNqOY.d.mts} +26 -3
- package/dist/{index-DUHmS74p.d.ts → index-LA-PMRJT.d.ts} +26 -3
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1078 -260
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1040 -251
- package/dist/index.mjs.map +1 -1
- package/dist/replay/index.d.mts +1 -1
- package/dist/replay/index.d.ts +1 -1
- package/dist/replay/index.js.map +1 -1
- package/dist/replay/index.mjs.map +1 -1
- package/dist/traces/index.js +125 -34
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +123 -32
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +1 -1
- package/dist/tracing/index.d.ts +1 -1
- package/dist/tracing/index.js +7 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +7 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +819 -438
- package/dist/worlds/index.d.ts +819 -438
- package/dist/worlds/index.js +912 -236
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +908 -232
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +4 -6
- package/runtime/gatewaysdk/environment/_materialize.py +5 -25
- package/runtime/gatewaysdk/environment/_world.py +82 -66
- package/runtime/gatewaysdk/environment/backing.py +20 -24
- package/runtime/gatewaysdk/environment/core.py +1 -16
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
- package/runtime/gatewaysdk/environment/runtime.py +45 -2
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +198 -474
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +321 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +276 -18
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +207 -19
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +4 -1
- package/runtime/gatewaysdk/environment/schema/conform.py +38 -35
- package/runtime/gatewaysdk/environment/schema/connector.py +94 -152
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +2 -1
- package/runtime/gatewaysdk/environment/schema/connectors/github/api/handlers.py +217 -58
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +10 -19
- package/runtime/gatewaysdk/environment/schema/connectors/github/scenarios/default/seed/github_webhooks.json +10 -2
- package/runtime/gatewaysdk/environment/schema/connectors/github/tests/webhooks.json +47 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +21 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/api/handlers.py +185 -18
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/connector.toml +29 -14
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +43 -12
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scenarios/default/seed/jamf_webhooks.json +2 -2
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +4 -17
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/tests/webhooks.json +107 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/vendor-openapi.json +1 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/api/handlers.py +85 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/connector.toml +19 -31
- package/runtime/gatewaysdk/environment/schema/connectors/linear/provenance.json +18 -8
- package/runtime/gatewaysdk/environment/schema/connectors/linear/scenarios/default/seed/webhooks.json +19 -3
- package/runtime/gatewaysdk/environment/schema/connectors/linear/tests/test_webhooks.py +71 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +14 -13
- package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
- package/runtime/gatewaysdk/environment/schema/database/guard.py +45 -37
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
- package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +52 -5
- package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +438 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +32 -116
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +20 -5
- package/runtime/gatewaysdk/environment/schema/platform.py +29 -234
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +136 -176
- package/runtime/gatewaysdk/environment/schema/scaffold.py +2 -1
- package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +50 -28
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +20 -22
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +25 -25
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +150 -30
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
- package/runtime/gatewaysdk/environment/schema/tools_world.py +103 -203
- package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
- package/runtime/gatewaysdk/environment/schema/verbs.py +38 -36
- package/runtime/gatewaysdk/environment/schema/webhooks.py +687 -0
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
- package/runtime/gatewaysdk/environment/schema/world_host.py +970 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +83 -11
- package/runtime/gatewaysdk/environment/world.py +1 -1
- package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/skills/connector-schema-authoring/SKILL.md +50 -28
- package/skills/connector-schema-authoring/connector-worlds.md +282 -18
- package/skills/connector-schema-authoring/world-schema.md +7 -6
- package/skills/world-data-ingestion/ROWS.md +12 -8
- package/skills/world-data-ingestion/SKILL.md +20 -22
- package/skills/worlds-getting-started/SKILL.md +25 -25
- package/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
# Journal checks: verifiers over the requests a session's world received
|
|
2
|
+
|
|
3
|
+
A World Host session records every request its world receives, through every door: the api origin,
|
|
4
|
+
the page's `api_path` (ui) and a declared tool's call. Before a grade the platform waits until the
|
|
5
|
+
host has stored every request it holds for the session (the flush barrier), reads them back and
|
|
6
|
+
hands the grader those requests with whether they are all of them. Three ways to check them, one
|
|
7
|
+
implementation underneath: a declarative check, a `verify.py`, a SQL verifier.
|
|
8
|
+
|
|
9
|
+
Only the World Host records requests. A stored task whose checks read them is refused at validation
|
|
10
|
+
and at dispatch on any other engine, a run config included (`checks on the session's journal
|
|
11
|
+
(gateway_requests, calls, a python grader that reads it) need the World Host`). A local `worlds serve`
|
|
12
|
+
session reads `calls` from its own call log; there and in `benchmarks dry-run` the rest fails closed
|
|
13
|
+
as `not recorded`, whatever the grader accepts.
|
|
14
|
+
|
|
15
|
+
## What a grader reads
|
|
16
|
+
|
|
17
|
+
`gateway_requests`, one row per request, oldest first:
|
|
18
|
+
|
|
19
|
+
```json
|
|
20
|
+
{ "seq": 2, "id": "0a1b…", "at": "2026-10-05T12:00:01.000Z", "durationMs": 12.5,
|
|
21
|
+
"via": "api", "operation": "create_record", "method": "POST", "path": "/records",
|
|
22
|
+
"query": { "dry_run": "false" }, "rows": 1, "found": true,
|
|
23
|
+
"request": { "headers": { "content-type": "application/json" }, "body": { "name": "Acme" },
|
|
24
|
+
"bodyBytes": 16, "truncated": false, "contentType": "application/json" },
|
|
25
|
+
"response": { "status": 201, "body": { "id": "rec_1" }, "bodyBytes": 15, "truncated": false,
|
|
26
|
+
"contentType": "application/json" } }
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
- `via` is `api`, `ui` or `tool`; a tool call has no `method` or `path`.
|
|
30
|
+
- A JSON body reads as JSON, any other as text. Bodies are what the ledger stored: redacted when
|
|
31
|
+
captured, the first 64 KB (`truncated: true` past that). A request past the grade's budget (2,000
|
|
32
|
+
bodies, 8 MiB) carries `withheld: ["request.headers", "request.body", "response.body"]`.
|
|
33
|
+
- `gateway_journal` is the summary: `{ "complete": true, "reason": null, "requests": 4, "withheld": 0 }`.
|
|
34
|
+
`complete` is false when the host shed bodies or lost requests, did not answer the barrier, the
|
|
35
|
+
platform refused some, the session was reopened from a checkpoint or recovered after its worker or
|
|
36
|
+
host stopped, fewer requests remain than a past grade read, or the session holds more than 20,000;
|
|
37
|
+
`reason` says which. `withheld` counts the requests whose bodies the grade's budget left out.
|
|
38
|
+
- What a session's world was asked, to write a check against: `gateway worlds session requests
|
|
39
|
+
<SESSION_ID> --json` (MCP `list_world_requests`), or the `asked:` lines `task try` prints.
|
|
40
|
+
|
|
41
|
+
## 1. Declarative checks (`assertions`)
|
|
42
|
+
|
|
43
|
+
```json
|
|
44
|
+
{ "kind": "assertions", "checks": [
|
|
45
|
+
{ "id": "created", "entity": "gateway_requests",
|
|
46
|
+
"where": { "operation": "create_record", "response.status": 201 } },
|
|
47
|
+
{ "id": "no_deletes", "entity": "gateway_requests", "where": { "method": "DELETE" }, "assert": "absent" },
|
|
48
|
+
{ "id": "one_create", "entity": "gateway_requests", "where": { "operation": "create_record" },
|
|
49
|
+
"assert": "count", "count": { "eq": 1 } },
|
|
50
|
+
{ "id": "looked_first", "entity": "gateway_requests", "where": { "operation": "get_record" },
|
|
51
|
+
"before": { "operation": "update_record" } },
|
|
52
|
+
{ "id": "named_acme", "entity": "gateway_requests", "where": { "operation": "create_record" },
|
|
53
|
+
"assert": "all", "field": "request.body.name", "value": "Acme" }
|
|
54
|
+
] }
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
- `where`, `field`, `before` and `after` name `seq`, `id`, `at`, `durationMs`, `via`, `operation`,
|
|
58
|
+
`method`, `path`, `rows`, `found`, `request.body`, `request.bodyBytes`, `request.truncated`,
|
|
59
|
+
`request.contentType`, `response.status`, `response.body`, `response.bodyBytes`,
|
|
60
|
+
`response.truncated`, `response.contentType`, or a path under `query.`, `request.headers.`,
|
|
61
|
+
`request.body.`, `response.body.`. Anything else is refused with the field it meant:
|
|
62
|
+
`gateway_requests has no field response.stauts; did you mean response.status?`.
|
|
63
|
+
- `before: {…}` keeps the requests before the first one it matches (all of them when none does);
|
|
64
|
+
`after: {…}` keeps the requests after the first one it matches (none when none does). They go on
|
|
65
|
+
`gateway_requests` and `calls` checks only.
|
|
66
|
+
- `entity: "calls"` on a World Host session is the call log's view of the same requests: every
|
|
67
|
+
door, tool calls included, with no bound (`operation`, `args`, `query`, `body`, `selectors`,
|
|
68
|
+
`via`, `status`, `rows`, `found`). A request no route took and a tool call the door refused stay
|
|
69
|
+
in `gateway_requests` only.
|
|
70
|
+
|
|
71
|
+
## 2. A `verify.py` that reads the journal
|
|
72
|
+
|
|
73
|
+
```json
|
|
74
|
+
{ "kind": "python", "readsJournal": true, "source": "<verify.py below>" }
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
import json, sys
|
|
79
|
+
|
|
80
|
+
state = json.load(sys.stdin)
|
|
81
|
+
creates = [r for r in state["gateway_requests"]
|
|
82
|
+
if r["operation"] == "create_record" and r["response"]["status"] == 201]
|
|
83
|
+
print(json.dumps({"reward": 1 if len(creates) == 1 else 0, "one_create": len(creates) == 1}))
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Only a grader that declares `readsJournal: true` gets `gateway_requests` and `gateway_journal` on
|
|
87
|
+
stdin, beside the rows, `calls` and `report`. On an incomplete journal, or one whose bodies were
|
|
88
|
+
withheld, it scores 0 with the reason unless it also declares `acceptsIncompleteJournal: true`; then
|
|
89
|
+
it runs and reads `state["gateway_journal"]` itself (a body cut at 64 KB says `truncated: true`).
|
|
90
|
+
With no journal recorded at all it scores 0 either way.
|
|
91
|
+
|
|
92
|
+
## 3. A SQL verifier (`verifiers/<name>.sql`, a `[[tasks]]` world)
|
|
93
|
+
|
|
94
|
+
```sql
|
|
95
|
+
SELECT CASE WHEN COUNT(*) = 0 THEN 1.0 ELSE 0.0 END
|
|
96
|
+
FROM gateway_requests WHERE method = 'DELETE';
|
|
97
|
+
```
|
|
98
|
+
|
|
99
|
+
```sql
|
|
100
|
+
SELECT CASE WHEN COUNT(*) = 1 THEN 1.0 ELSE 0.0 END
|
|
101
|
+
FROM gateway_requests
|
|
102
|
+
WHERE operation = 'create_record' AND status = 201
|
|
103
|
+
AND json_extract(request_body, '$.name') = 'Acme';
|
|
104
|
+
```
|
|
105
|
+
|
|
106
|
+
`gateway_requests` columns: `seq, id, at, duration_ms, via, operation, method, path, query, status,
|
|
107
|
+
rows, found, request_headers, request_body, request_body_bytes, request_truncated,
|
|
108
|
+
request_content_type, response_body, response_body_bytes, response_truncated,
|
|
109
|
+
response_content_type, withheld` (query, headers and bodies are the stored text: read them with
|
|
110
|
+
`json_extract`). `gateway_journal(complete, reason, requests, withheld)`. A verifier that reads
|
|
111
|
+
either on an incomplete journal, or reads a body column while a body was withheld or cut at 64 KB,
|
|
112
|
+
scores 0, and the grade names why under `reasons`, unless `gateway-env.toml` lets it (a read counts
|
|
113
|
+
however the statement reaches the table, joins included):
|
|
114
|
+
|
|
115
|
+
```toml
|
|
116
|
+
[verifiers.created_at_least_once]
|
|
117
|
+
accepts_incomplete_journal = true
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
Let only a check that a missing request can only make fail accept an incomplete journal: `COUNT(*) >
|
|
121
|
+
0`, not `= 0` or `= 1`, which pass on a journal that lost a request they count. No flag grades a
|
|
122
|
+
journal never recorded, nor a linked world's verifier that reads one: the session's journal is the
|
|
123
|
+
root world's.
|
|
124
|
+
|
|
125
|
+
An entity may not be named `gateway_…` (the compiler refuses it), so the names never collide.
|
|
126
|
+
|
|
127
|
+
## Fail closed
|
|
128
|
+
|
|
129
|
+
A check the stored requests do not decide fails and says why:
|
|
130
|
+
|
|
131
|
+
```
|
|
132
|
+
FAIL no_deletes — journal incomplete (the World Host lost 2 requests): the 4 requests it holds do not decide this check
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
A check they do decide still grades: `created` passes on an incomplete journal that holds the
|
|
136
|
+
request. Every World Host grade but a rubric's takes the barrier and records how complete the
|
|
137
|
+
journal was, so a later `try` or `regrade` of the session knows. A `calls` check on `args`, `body`
|
|
138
|
+
or `selectors` has the bodies fetched for it; one on `operation` or `status` alone does not need
|
|
139
|
+
them.
|
|
140
|
+
|
|
141
|
+
## Try, save, regrade
|
|
142
|
+
|
|
143
|
+
```bash
|
|
144
|
+
gateway worlds task try task.json --session <SESSION_ID> # a spec not saved yet; files nothing
|
|
145
|
+
gateway worlds task try <TASK_ID> --session <SESSION_ID> --version 3 --json
|
|
146
|
+
gateway worlds task update <TASK_ID> --spec task.json # save: the next version
|
|
147
|
+
gateway worlds task validate <TASK_ID>
|
|
148
|
+
gateway worlds task regrade <TASK_ID> # the task's latest run, whole, by the current version
|
|
149
|
+
gateway worlds task regrade <TASK_ID> --run <EVALUATION_ID> --version 3
|
|
150
|
+
gateway worlds task regrade <TASK_ID> --all-runs --limit 20
|
|
151
|
+
gateway worlds task regrade <TASK_ID> --session <SESSION_ID> # one rollout; its run then shows the versions it mixes
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
```
|
|
155
|
+
try the spec on session ses_01: graded live
|
|
156
|
+
journal: 4 requests, incomplete (the World Host lost 2 requests)
|
|
157
|
+
PASS created — matched #2 api POST create_record 201 (+1 more)
|
|
158
|
+
FAIL no_deletes — journal incomplete (the World Host lost 2 requests): the 4 requests it holds do not decide this check
|
|
159
|
+
reward 0.5
|
|
160
|
+
asked:
|
|
161
|
+
1x api GET list_records 200 /records
|
|
162
|
+
2x api POST create_record 201 /records
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
`try` grades a live session in place, and a closed one on an ephemeral copy restored from the
|
|
166
|
+
checkpoint its last grade took, over the requests that grade read. A session no grade checkpointed
|
|
167
|
+
refuses checks over rows; checks over `gateway_requests` and `calls` still run. A check on an entity
|
|
168
|
+
the session's world does not define is refused, as an open refuses it.
|
|
169
|
+
|
|
170
|
+
`regrade` grades each rollout's session the way `try` does and replaces the rollout's grade in
|
|
171
|
+
place: the grade it held moves to `info.grades`, `info.verifierVersion` names the version that
|
|
172
|
+
graded it, and the run's metrics and label (`verifier v3`, or `verifiers v2, v3` for a mixed run)
|
|
173
|
+
follow. A release gate reads the new rewards on its next read. Exit 1 when nothing was regraded, a
|
|
174
|
+
rollout kept its grade (already graded by that version, so a rerun adds no history; no checkpoint
|
|
175
|
+
for a check over rows) or a rollout was regraded to no score (`--json` holds its reasons). A run or
|
|
176
|
+
session that is not the task's is refused. A task over several worlds is not regraded, and `try` on
|
|
177
|
+
one of its sessions grades that world's own grader, not the cross-world verifiers.
|
|
178
|
+
|
|
179
|
+
| CLI | MCP | REST | SDK |
|
|
180
|
+
| --- | --- | --- | --- |
|
|
181
|
+
| `worlds task try` | `try_world_task({ sessionId, taskId \| spec, version? })` (MEMBER and above: it runs your verify.py) | `POST /api/public/world-tasks/try` | `new TaskGrades(opts).try(sessionId, target)` / `TaskGrades().try_task(session_id, task_id=… \| spec=…)` |
|
|
182
|
+
| `worlds task regrade` | `regrade_world_task({ taskId, version?, runs?, allRuns?, sessions?, limit? })` | `POST /api/public/world-tasks/{taskId}/regrade` | `.regrade(taskId, scope)` / `.regrade(task_id, …)` |
|