@withgateway/sdk 0.3.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +3005 -1401
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2996 -1392
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +128 -37
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +126 -35
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-mJhG3vZU.d.mts → index-Di9nNqOY.d.mts} +26 -3
- package/dist/{index-DUHmS74p.d.ts → index-LA-PMRJT.d.ts} +26 -3
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1078 -260
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1040 -251
- package/dist/index.mjs.map +1 -1
- package/dist/replay/index.d.mts +1 -1
- package/dist/replay/index.d.ts +1 -1
- package/dist/replay/index.js.map +1 -1
- package/dist/replay/index.mjs.map +1 -1
- package/dist/traces/index.js +125 -34
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +123 -32
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +1 -1
- package/dist/tracing/index.d.ts +1 -1
- package/dist/tracing/index.js +7 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +7 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +819 -438
- package/dist/worlds/index.d.ts +819 -438
- package/dist/worlds/index.js +912 -236
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +908 -232
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +4 -6
- package/runtime/gatewaysdk/environment/_materialize.py +5 -25
- package/runtime/gatewaysdk/environment/_world.py +82 -66
- package/runtime/gatewaysdk/environment/backing.py +20 -24
- package/runtime/gatewaysdk/environment/core.py +1 -16
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
- package/runtime/gatewaysdk/environment/runtime.py +45 -2
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +198 -474
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +321 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +276 -18
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +207 -19
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +4 -1
- package/runtime/gatewaysdk/environment/schema/conform.py +38 -35
- package/runtime/gatewaysdk/environment/schema/connector.py +94 -152
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +2 -1
- package/runtime/gatewaysdk/environment/schema/connectors/github/api/handlers.py +217 -58
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +10 -19
- package/runtime/gatewaysdk/environment/schema/connectors/github/scenarios/default/seed/github_webhooks.json +10 -2
- package/runtime/gatewaysdk/environment/schema/connectors/github/tests/webhooks.json +47 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +21 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/api/handlers.py +185 -18
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/connector.toml +29 -14
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +43 -12
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scenarios/default/seed/jamf_webhooks.json +2 -2
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +4 -17
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/tests/webhooks.json +107 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/vendor-openapi.json +1 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/api/handlers.py +85 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/connector.toml +19 -31
- package/runtime/gatewaysdk/environment/schema/connectors/linear/provenance.json +18 -8
- package/runtime/gatewaysdk/environment/schema/connectors/linear/scenarios/default/seed/webhooks.json +19 -3
- package/runtime/gatewaysdk/environment/schema/connectors/linear/tests/test_webhooks.py +71 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +14 -13
- package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
- package/runtime/gatewaysdk/environment/schema/database/guard.py +45 -37
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
- package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +52 -5
- package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +438 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +32 -116
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +20 -5
- package/runtime/gatewaysdk/environment/schema/platform.py +29 -234
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +136 -176
- package/runtime/gatewaysdk/environment/schema/scaffold.py +2 -1
- package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +50 -28
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +20 -22
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +25 -25
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +150 -30
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
- package/runtime/gatewaysdk/environment/schema/tools_world.py +103 -203
- package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
- package/runtime/gatewaysdk/environment/schema/verbs.py +38 -36
- package/runtime/gatewaysdk/environment/schema/webhooks.py +687 -0
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
- package/runtime/gatewaysdk/environment/schema/world_host.py +970 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +83 -11
- package/runtime/gatewaysdk/environment/world.py +1 -1
- package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/skills/connector-schema-authoring/SKILL.md +50 -28
- package/skills/connector-schema-authoring/connector-worlds.md +282 -18
- package/skills/connector-schema-authoring/world-schema.md +7 -6
- package/skills/world-data-ingestion/ROWS.md +12 -8
- package/skills/world-data-ingestion/SKILL.md +20 -22
- package/skills/worlds-getting-started/SKILL.md +25 -25
- package/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
|
@@ -75,8 +75,8 @@ registration, a platform builder, live ingestion, tools, or a registry framework
|
|
|
75
75
|
| `gateway worlds schema import PATH --from <kind> --spec <url\|path> [--scope scope.toml]` | not implemented; author the contract and provenance manually |
|
|
76
76
|
| `gateway worlds schema parity PATH --surface <tools-list.json> \| --connection <id>` (advisory report) | not implemented |
|
|
77
77
|
| `gateway worlds playground <connector>` | not implemented |
|
|
78
|
-
| Platform upload of a JSON Schema, OpenAPI, or DDL as a new connector | not implemented as a UI upload; the platform agent
|
|
79
|
-
| Platform agent record ingestion
|
|
78
|
+
| Platform upload of a JSON Schema, OpenAPI, or DDL as a new connector | not implemented as a UI upload; the platform agent authors the tree in its sandbox and runs `gateway worlds create <slug> --from ./dir` through `run_gateway_cli`, the same door as the CLI (2026-10-10) |
|
|
79
|
+
| Platform agent record ingestion | the CLI's own `data import <slug>`, `pull`, `push` and `describe` through `run_gateway_cli`; see `documentation/features/agent.md` |
|
|
80
80
|
| SDK/public and platform publication | validates assembled files and actual stored archive bytes through the trusted compiler/materializer, including READY/dedup/ref paths |
|
|
81
81
|
| Connectors Slack world creation | durable structured capture → shared Python mapper → immutable publication; explicit synthetic mode |
|
|
82
82
|
| Worker schema tests/workbench | shared validation and isolated E2B author execution; missing isolation is non-green |
|
|
@@ -143,7 +143,7 @@ deliverable can finish with synthetic fixtures and explicitly skipped live valid
|
|
|
143
143
|
Tool handlers (`tools/*.py`) reach the world's rows through `from gateway_env import store`. It has
|
|
144
144
|
exactly these methods; `gateway worlds schema check .` refuses any other `store.<name>` at file:line
|
|
145
145
|
with this list, and a hosted call into such a handler answers 400 `{error: "handler", detail, handler}`
|
|
146
|
-
(an `AttributeError`/`TypeError`/`NameError` is the author's bug; anything else a handler raises is a
|
|
146
|
+
(an `AttributeError`/`TypeError`/`NameError` is the author's bug; `raise Refusal(404, body)` (imported from `gateway_env` at module top) answers that 4xx and body; anything else a handler raises is a
|
|
147
147
|
500 `{error: "engine", detail, handler}`; never a bare exception string). A route's handler answers the
|
|
148
148
|
same way on every door, and an answer JSON cannot carry (a `datetime`, a set, NaN) is its 400; a route
|
|
149
149
|
answering bytes names its `Content-Type` (a vendor's `404 text/plain`), and the bytes go out as sent.
|
|
@@ -194,10 +194,11 @@ No `upsert`: `get`, then `insert` or `update`. Read-only SQL: `gateway_env.db.on
|
|
|
194
194
|
## Runtime authoring boundary
|
|
195
195
|
|
|
196
196
|
When adding a bridge command to `schema/platform.py`, mirror it as a typed wrapper in
|
|
197
|
-
`worldSchemaRuntime.ts` (`compileWorldFiles` / `
|
|
198
|
-
in, `{entries,
|
|
199
|
-
`tests/environment/test_schema_platform.py`, and let
|
|
200
|
-
the subprocess
|
|
197
|
+
`worldSchemaRuntime.ts` (`compileWorldFiles` / `importWorldData` are the pattern: file map
|
|
198
|
+
in, `{entries, ...}` out, errors as `SchemaBridgeError`), cover it in
|
|
199
|
+
`tests/environment/test_schema_platform.py`, and let a service call the wrapper, never
|
|
200
|
+
the subprocess; the public `/api/public/worlds/runtime` route reaches only the commands in
|
|
201
|
+
`WORLD_RUNTIME_COMMANDS`. The bridge needs Python ≥ 3.12 (`tomllib`, and the store's sqlite `setconfig`); set `GATEWAY_SCHEMA_PYTHON`
|
|
201
202
|
when the default interpreter is older.
|
|
202
203
|
|
|
203
204
|
Use `worldSchemaRuntime.ts` for platform validation and `schema.platform` for
|
|
@@ -273,8 +274,12 @@ These source checks do not imply live-provider validation or executed fidelity s
|
|
|
273
274
|
|
|
274
275
|
A world can describe the vendor it mirrors in `connector.toml` at its root: base URL,
|
|
275
276
|
auth scheme (`header`, `bearer`, `basic`, `query`, `oauth2`, `none`) with the secret named by an
|
|
276
|
-
environment variable (
|
|
277
|
-
|
|
277
|
+
environment variable (a `bearer`'s `prefixes` lists the `Authorization` words the vendor takes,
|
|
278
|
+
`["Bearer", "token"]` for GitHub, the first carried; a `basic` key is the Basic user with an empty
|
|
279
|
+
password, or with `credential = "password"` the password under any user, as Atlassian and
|
|
280
|
+
ServiceNow take it), the vendor's own login route (`[auth.login]`; an oauth2 `token_url` path may
|
|
281
|
+
name a segment, `/ccx/oauth2/{tenant}/token`) and presigned download routes (`[auth.downloads]`),
|
|
282
|
+
capture pacing, a redaction policy (`hash` or `drop` fields at any
|
|
278
283
|
depth), and the operations the agent actually calls (method, path with `{args}`, query
|
|
279
284
|
params, and the entity plus results path their rows land in). The same file is meant to
|
|
280
285
|
drive the platform's custom source later, so describe once.
|
|
@@ -316,6 +321,10 @@ Three refinements the first three vendors forced:
|
|
|
316
321
|
as well as a path argument; a parameter the caller leaves out does not narrow, and one
|
|
317
322
|
sent more than once (`status[]=a&status[]=b`) matches rows carrying any of its values.
|
|
318
323
|
Handlers read every value from `ctx.query_all`; the `query` argument keeps the first.
|
|
324
|
+
- **Uploads.** A multipart form reaches a handler as its fields (a file part reads
|
|
325
|
+
`{"filename", "content_type", "size"}`) with the bytes in `ctx.files[name].content`; a raw
|
|
326
|
+
body (`application/octet-stream`, `text/csv`) reads `body` as `{}` with the bytes in
|
|
327
|
+
`ctx.raw`. A declared route refuses an upload as the vendor's 400.
|
|
319
328
|
|
|
320
329
|
Four more the Brex retrospective forced (each cost a day of one-field-per-run fixing):
|
|
321
330
|
|
|
@@ -349,21 +358,17 @@ redacted pages back into `captures/`.
|
|
|
349
358
|
|
|
350
359
|
### From the platform agent (no checkout)
|
|
351
360
|
|
|
352
|
-
The console agent
|
|
353
|
-
`
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
(
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
refused before anything is queued. `run_gateway_cli` runs the TypeScript `gateway worlds …` CLI
|
|
364
|
-
(the same flags as this page) staged into the sandbox, local-only (paths under `/workspace`,
|
|
365
|
-
no `--host`/`--remote`/`workspace:<slug>`, platform env stripped, pinned to its local runtime),
|
|
366
|
-
so publishing always goes through the tools above. To parallelise several connectors, `spawn_subagent` with a brief
|
|
361
|
+
The console agent runs the CLI on this page through `run_gateway_cli` in its sandbox: write
|
|
362
|
+
`connector.toml` + `schema/` under `./acme` → `connector validate ./acme`, `connector conform ./acme --openapi ./spec.json`,
|
|
363
|
+
`schema check ./acme`, `test ./acme` → `create acme --from ./acme` (billed: the session's
|
|
364
|
+
`provisionWorlds` or `confirm: true`) → `start_connector_capture({ containerId, op, args, params })` →
|
|
365
|
+
`get_connector_capture({ …, downloadTo })` → `connector ingest ./acme` → `push ./acme`, then
|
|
366
|
+
`edit close acme`. `connector publish ./acme` makes it a workspace connector (`hub:write` and
|
|
367
|
+
`allowConnectorWrites`, else `confirm: true`). A `./dir` runs in the sandbox; a slug reaches the
|
|
368
|
+
platform as the user through a key minted for that one call, valid only on the routes that verb calls and
|
|
369
|
+
deleted when the call ends; `--host` and key flags are refused. `list_world_secrets` shows the
|
|
370
|
+
env var names the world holds, never values; a `connector.toml` with an inline credential is
|
|
371
|
+
refused before anything is queued. To parallelise several connectors, `spawn_subagent` with a brief
|
|
367
372
|
naming only the tools that child needs (`allowSubagents` on the session).
|
|
368
373
|
|
|
369
374
|
### From the CLI sandbox (no checkout, no Python)
|
|
@@ -399,23 +404,40 @@ them:
|
|
|
399
404
|
- `unauthorized` / `not_found` / `invalid` are JSON strings replayed on 401 / 404 / 400, so
|
|
400
405
|
the vendor's own error bodies come back; `""` (or `'null'`) means no body at all. A route
|
|
401
406
|
whose missing row the vendor answers its own way sets `not_found` and `not_found_status`
|
|
402
|
-
under `[operations.api]`, falling back to `[api]`.
|
|
407
|
+
under `[operations.api]`, falling back to `[api]`. A path the world does not serve answers
|
|
408
|
+
`not_found`; a path it serves, called with another method, answers `wrong_method` (405 with
|
|
409
|
+
`Allow`, no body unless declared; `wrong_method_status` for a vendor that says 404). A write
|
|
410
|
+
the contract refuses answers `invalid` exactly as declared; the refusal's rule goes to the
|
|
411
|
+
host's log, never the answer.
|
|
412
|
+
- `link_header = true` sends RFC 8288 `Link: <next>; rel="next"` (and `rel="prev"`) on list
|
|
413
|
+
answers, each URL the request's own (GitHub's paging); `cors = true` answers a browser's
|
|
414
|
+
preflight before any credential and names any origin (exposing `Link`) on every answer, for a vendor whose
|
|
415
|
+
API serves pages (`worlds serve` answers only this machine's pages). HEAD answers as GET
|
|
416
|
+
without a body on every world.
|
|
403
417
|
- `[operations.api] action` makes a route a `create`, `update`, `delete` or `handler`; handlers live in the world's `api/handlers.py` and carry computed results and workflow state (`ctx.ladder` on the clock). A world is read _and_ write: an agent that creates a search and gets 501 is not testing anything. Full reference: `documentation/features/connector-worlds.md`.
|
|
404
418
|
- `[operations.api] filter = { email = "attributes.email" }` maps a path argument to the
|
|
405
419
|
row field it selects on, matched case-insensitively at any depth. An operation without
|
|
406
420
|
an `entity` answers 501 `not_modelled` rather than inventing a response.
|
|
421
|
+
- `[auth.login]` mirrors a vendor that trades its key for a token (`path`, `credential = { json |
|
|
422
|
+
form | header = "<field>" }`, `required`, `reply = '{"data": {"token": "{token}"}}'`);
|
|
423
|
+
later calls carry the token the way `scheme` says, and only tokens minted for that session count.
|
|
424
|
+
`[auth.downloads] routes = ["download_x"]` lets those GET routes answer a link a handler signs with
|
|
425
|
+
`ctx.signed_url(path)` without the key. Tokens and links live as long as the session; set
|
|
426
|
+
`ttl_seconds` (login, downloads, oauth2) only when the eval needs expiry. `mode = "open"` under
|
|
427
|
+
`[auth]` opens sessions with no credential at all (a capability URL; `--auth` on the session's
|
|
428
|
+
open overrides; `scheme = "open"` is refused). Full grammar: connector-worlds.md, "The door".
|
|
407
429
|
- `[ui] static = "ui/static"` (+ `entry`, `api_path`, `generated`) is the page over the
|
|
408
430
|
routes: generated by `compile` from the contract unless `generated = false`, served by the
|
|
409
431
|
host on the session's ui origin (`?token=` once, then a cookie; `api_path` rewritten onto
|
|
410
432
|
the routes) and locally on its own port (and at `/__ui/`). A page holds no key; the host presents the world's
|
|
411
|
-
credential for it, the way it does for a
|
|
433
|
+
credential for it, the way it does for a caller holding the session token in `X-World-Session`.
|
|
412
434
|
|
|
413
435
|
`gateway worlds serve PATH --port 8080 [--api-key K]` runs it locally; the runner's admin
|
|
414
436
|
endpoints (`/__world/seed`, `/__world/reset`, `/__world/state`) sit beside the vendor routes
|
|
415
437
|
and answer only the run's `X-World-Admin-Token` (`gateway worlds test` hands a world's pytest
|
|
416
438
|
files theirs as `WORLD_ADMIN_TOKEN`; by hand, use `/__session/*`); `/healthz` answers everyone,
|
|
417
|
-
and a World Host session's doors answer none of the
|
|
439
|
+
and a World Host session's api and tool doors answer none of them (there `/healthz` is the world's own path; its ui origin answers `/healthz` under `api_path` to the twin's own page). `gateway bench push PATH` publishes the world: for a world carrying
|
|
418
440
|
`connector.toml` it packages the api-kind bundle the platform hosts on the way (root
|
|
419
441
|
legacy-shaped on purpose, since the api runner refuses a schema bundle there; `server/`
|
|
420
442
|
needs nothing installed). Open a session with `--surface api` and hand the client that
|
|
421
|
-
URL; the vendor's auth
|
|
443
|
+
URL; the vendor's auth applies as `[auth]` declares it, with the session's own api token as the credential.
|
|
@@ -264,17 +264,18 @@ schema type, the same rule in the TypeScript CLI (`worldData/cast.ts`) and the P
|
|
|
264
264
|
|
|
265
265
|
| Field type | Cell | Value |
|
|
266
266
|
| --- | --- | --- |
|
|
267
|
-
| `integer` | `42`, `-7` | the number; `4.5` or `abc`
|
|
267
|
+
| `integer` | `42`, `-7` | the number; `4.5` or `abc` stays text, refused by the contract |
|
|
268
268
|
| `number` | `12.5`, `1e3`, `3` | the number (always a float) |
|
|
269
|
-
| `boolean` | `true`/`false`/`1`/`0`, any case | the boolean; `yes`
|
|
269
|
+
| `boolean` | `true`/`false`/`1`/`0`, any case | the boolean; `yes` stays text, refused by the contract |
|
|
270
270
|
| `array` / `object` | JSON text of that shape (quote it in the CSV: `"[""a"",""b""]"`) | the parsed value |
|
|
271
271
|
| `string` | anything | the text as given |
|
|
272
272
|
| `integer \| string` (union) | each member tried in order, `string` last | the first that fits |
|
|
273
|
-
| any, empty cell |
|
|
273
|
+
| any, empty cell | (nothing) | null when the type admits null; `""` for a required string; left out otherwise (a required one is then refused by the contract) |
|
|
274
274
|
|
|
275
|
-
A cell that fits none
|
|
276
|
-
|
|
277
|
-
|
|
275
|
+
A cell that fits none stays its text and the contract refuses the row as it refuses a JSON row
|
|
276
|
+
— `taxi_trip {"id":3}.fare: Expected number`, with the usual pointer — so a file reports every
|
|
277
|
+
bad row at once (`--report` writes them all) and nothing is written. A column the schema does
|
|
278
|
+
not name stays text and the gate refuses it as an unknown field. A CSV
|
|
278
279
|
on the drive (`drive:<path>`) is still refused: import it from the machine that holds it, or
|
|
279
280
|
through `worlds sandbox import` when it sits in a sandbox.
|
|
280
281
|
|
|
@@ -311,8 +312,11 @@ the package version keeps every snapshot (local, hosted, pinned), with no re-imp
|
|
|
311
312
|
that only adds to the domain (a new entity, a new field that is not required, another
|
|
312
313
|
`domain_version`) carries the rows: compile migrates the snapshots the tree holds and keeps the
|
|
313
314
|
pin with the stored domain's `lock` and `entities`, `sandbox check` says the commit migrates the
|
|
314
|
-
hosted snapshot, and `sandbox commit` migrates it
|
|
315
|
-
|
|
315
|
+
hosted snapshot, and `sandbox commit` migrates it as a job (a `migration` batch: the CLI follows it,
|
|
316
|
+
MCP answers `{batchId}` for `get_data_batch`) and re-pins the tree (a CLI push of such a change is
|
|
317
|
+
refused, naming the sandbox). Importing exactly the batch behind the served snapshot again (the
|
|
318
|
+
same chunks in the same order, with `redact` off) is a no-op: the batch lands on the same version
|
|
319
|
+
and names the one it repeats (`unchangedFrom`). Any other change to an entity, a field (its description
|
|
316
320
|
or title too), a key or a relationship refuses it, naming each change:
|
|
317
321
|
|
|
318
322
|
```text
|
|
@@ -53,10 +53,9 @@ A CSV/TSV is decided by its name and read as a stream (bounded memory at any siz
|
|
|
53
53
|
`--delimiter <char>` splits the cells (one character, `\t` for a tab; default `,`, tab for a
|
|
54
54
|
`.tsv`). The header row names the fields and **every cell is cast by the entity's schema
|
|
55
55
|
type**: `integer`/`number` parse as numbers, `boolean` takes true/false/1/0, `array`/`object`
|
|
56
|
-
parse as JSON, `string` stays text; an empty cell is null when the field admits null, left out
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
pinned by `tests/conformance/csv_cast.json`. Without `--entity` a CSV is refused.
|
|
56
|
+
parse as JSON, `string` stays text; an empty cell is null when the field admits null, `""` for a required string, left out otherwise. A cell that does not fit stays text and the contract refuses its row as it refuses a JSON row:
|
|
57
|
+
every bad row named by entity, key and field (`taxi_trip {"id":3}.fare: Expected number`), all counted, each in `--report`.
|
|
58
|
+
Same rule in both CLIs, pinned by `tests/conformance/csv_cast.json`. Without `--entity` a CSV is refused.
|
|
60
59
|
|
|
61
60
|
Rules:
|
|
62
61
|
|
|
@@ -110,7 +109,7 @@ gateway worlds data query <WORLD> users --where email=jo@acme.test --resolve #
|
|
|
110
109
|
gateway worlds data sql <WORLD> 'SELECT id, name FROM users LIMIT 5' # read-only SQL, any engine
|
|
111
110
|
```
|
|
112
111
|
|
|
113
|
-
`data sql` runs ONE read-only statement (joins, `GROUP BY`, counts) over a schema world's entity tables,
|
|
112
|
+
`data sql` runs ONE read-only statement (joins, `GROUP BY`, counts) over a schema world's entity tables (and `json_each` / `json_tree` over their JSON),
|
|
114
113
|
whatever its database engine: a write, `ATTACH`, `PRAGMA` or a second statement is refused; at most 1000 rows
|
|
115
114
|
(`--limit`). MCP twin: `sql_world_data({ slug, sql, limit? })`. A db/schema.sql world is a SQLite file: `sqlite3`.
|
|
116
115
|
|
|
@@ -156,10 +155,10 @@ JSONL chunks of `--chunk-bytes` (64 MiB default) or `--chunk-rows N`, uploads th
|
|
|
156
155
|
`--parallel 4` at a time straight to the bucket, and completes the batch; the platform
|
|
157
156
|
validates every row against the contract, applies the batch onto the world's data snapshot
|
|
158
157
|
and, with `--publish now` (default), cuts a data-only version (no image rebuild) on your open edit session
|
|
159
|
-
(`editSessionId`; only your credential reads it
|
|
158
|
+
(`editSessionId`; only your credential's unpinned reads see it, and `slug@main`, runs and tasks keep the sealed `main`, until `worlds edit close <slug>` or 60 idle minutes seal it). `--wait`
|
|
160
159
|
(default) polls to the end and prints the batch: `{batchId, state, rows, refusalCount,
|
|
161
160
|
refusals[:200], entityCounts, versionId?, batchDigest?}`. `--no-wait` prints `{batchId,
|
|
162
|
-
state}` and `gateway worlds data status <batchId
|
|
161
|
+
state}` and `gateway worlds data status <batchId> [--report <file>]` reads it later. Exit codes: 0 applied or
|
|
163
162
|
published, 2 refused (the report is written when `--report` is given: JSONL of refusal
|
|
164
163
|
records, at most 10,000 then `{"truncated": true, "total": N}`), 1 error.
|
|
165
164
|
|
|
@@ -221,13 +220,11 @@ policy (`drop`, `hash` as `sha256:<hex>` of `<slug>:<value>`, `preserve` as a sa
|
|
|
221
220
|
same-character-class hash for identifiers, `round`) unless `--no-redact`. Output is tagged
|
|
222
221
|
JSONL; the summary `{"read", "rows", "unknownFields", "redacted"}` is on stderr.
|
|
223
222
|
|
|
224
|
-
Platform agent (console): the same
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
slugs); land the result with `ingest_to_world`. Capture pages are not rows: `get_connector_capture(downloadTo)` →
|
|
230
|
-
`run_gateway_cli connector ingest` → `ingest_to_world`.
|
|
223
|
+
Platform agent (console): the same CLI, through `run_gateway_cli` in its sandbox. Build the rows
|
|
224
|
+
file with code, then `run_gateway_cli({ command: "data import", args: ["<slug>", "./rows.jsonl", "--dry-run"] })`
|
|
225
|
+
and again without `--dry-run`; a slug reaches the platform as the user, a `./dir` stays in the
|
|
226
|
+
sandbox. Capture pages are not rows: `get_connector_capture(downloadTo)` →
|
|
227
|
+
`run_gateway_cli connector ingest ./<world>` → `push ./<world>`.
|
|
231
228
|
|
|
232
229
|
### 2c. Turn tool calls into rows: pull, transform, ingest
|
|
233
230
|
|
|
@@ -285,7 +282,7 @@ what nests, what a write's `args` become, which call wins — in code you read t
|
|
|
285
282
|
write. The worked CRM example (nested contacts/deals exploded out of an account read, a
|
|
286
283
|
deal's nested account lifted, activity rows from `crm_log_activity` args keyed by the
|
|
287
284
|
observation id, a deletion turned into an event row, latest call wins) is on
|
|
288
|
-
|
|
285
|
+
https://docs.surfacearea.ai/worlds/data in both languages; the fixtures live in
|
|
289
286
|
`gatewaysdk/sdk/tests/environment/fixtures/crm/` (`records.jsonl`, `transform.py`,
|
|
290
287
|
`transform.ts`, `ingest.toml`, `rows.json`).
|
|
291
288
|
|
|
@@ -363,13 +360,13 @@ operations route by name; `captureToolCalls(tools, sink)` records the calls). Py
|
|
|
363
360
|
redact="off", dry_run=False)` and `ingest_tool_calls`; `capture_tool_calls`. Bridge:
|
|
364
361
|
`ingest-calls {files, records, map?, redact?, dry_run?, mode?}` → the report + `projected`
|
|
365
362
|
rows (+ `files` when the tree changed); transforms never run in the bridge. No MCP twin yet;
|
|
366
|
-
the console agent
|
|
363
|
+
the console agent runs the same `data ingest` through `run_gateway_cli`.
|
|
367
364
|
|
|
368
365
|
### 2d. Rows generated or downloaded inside a sandbox
|
|
369
366
|
|
|
370
367
|
```bash
|
|
371
368
|
gateway worlds sandbox exec <id> 'curl -sSL https://example.test/trips.csv -o data/trips.csv' # or a script that writes rows
|
|
372
|
-
gateway worlds sandbox import <id> data/trips.csv --entity taxi_trip [--delimiter ...] [--mode append|replace] [--dry-run]
|
|
369
|
+
gateway worlds sandbox import <id> data/trips.csv --entity taxi_trip [--delimiter ...] [--mode append|replace] [--dry-run] [--report refusals.jsonl]
|
|
373
370
|
gateway worlds sandbox commit <id> -m "tools" # the SOURCE tree only; rows are not in a commit
|
|
374
371
|
```
|
|
375
372
|
|
|
@@ -378,8 +375,9 @@ sandbox (`POST /api/public/benchmark-containers/sandboxes/{id}/import`; the sand
|
|
|
378
375
|
chunks it to JSONL in a scratch dir the platform reads and removes) and streams it through the
|
|
379
376
|
same ledger as `data import <slug>`: contract, refusals with pointers, `--mode`, `--merge`,
|
|
380
377
|
`--redact`, `--per-chunk`, `-m`. Prints `{versionId, entityCounts, rowCount}`, or `{batchId,
|
|
381
|
-
state}` to poll with `data status`.
|
|
382
|
-
`
|
|
378
|
+
state}` to poll with `data status`. Refused rows exit 2: `{batchId, state: refused, refusalCount, refusals}` (all counted,
|
|
379
|
+
the first 200 inline); `--report` writes every refusal as JSONL, a `--per-chunk` import's and a polled batch's too. MCP:
|
|
380
|
+
`import_world_sandbox_data({ sandboxId, path, entity?, mode? })`. A new-world sandbox commits once first; `commit` never carries rows.
|
|
383
381
|
|
|
384
382
|
### 3. Verify on a live session
|
|
385
383
|
|
|
@@ -390,7 +388,7 @@ gateway worlds session open <slug> -t <scenario> --surface api # prints s
|
|
|
390
388
|
gateway worlds session seed <SESSION_ID> rows.json # kind:"seed" call, waits for the result
|
|
391
389
|
gateway worlds session seed <SESSION_ID> rows.json --redact refuse # same, refusing plaintext in a redacted field (apply hashes it first)
|
|
392
390
|
gateway worlds session seed <SESSION_ID> rows.json --mode replace # swap the named entities
|
|
393
|
-
curl -H "
|
|
391
|
+
curl -H "X-World-Session: $API_TOKEN" "$API_URL/<route>" # read it back (api.token from session open)
|
|
394
392
|
```
|
|
395
393
|
|
|
396
394
|
Prints `{sessionId, mode, result}`. MCP: `seed_world_session({ sessionId, rows, mode? })` waits
|
|
@@ -457,7 +455,7 @@ Sanitize before importing: hash or drop personal fields the world's `[redact]` p
|
|
|
457
455
|
| refused, kind `api` (or `harbor`), naming the file to edit | the world is a generated bundle or a task suite | edit the named source, `bench push` again |
|
|
458
456
|
| `version` missing from the result | a local directory was given | push with `gateway bench push <dir>` when ready |
|
|
459
457
|
| `... is over what a world's source tree carries as data (4 MiB / 100000 rows ...)`; `data/initial.json would be N MiB with these rows, over the 4 MiB publication reads a source file whole`; `exceed the inline cap ... without the bundled runtime` | a directory import above the tree cap; or no `python3`, so the rows travel whole (5,000 / 4 MiB) | `worlds sandbox import` from a sandbox, `data import <slug>` with keys (never keys into the sandbox); install python3 |
|
|
460
|
-
|
|
|
458
|
+
| `<entity> {"<key>":…}.<field>: Expected number` (integer, boolean, array, object) on a CSV row | a cell the field's schema type does not fit: it stays text and the contract refuses its row | fix the cell or the header; `--report` lists every row; nothing was written |
|
|
461
459
|
| exit 2, `state: refused` | the platform refused the batch; `--report` has every record | fix the rows named, `--resume` is not for this: run the import again |
|
|
462
460
|
| `nothing to resume` / `only an open or uploading batch resumes` | no incomplete manifest, or the batch already ran | run the import without `--resume` |
|
|
463
461
|
| `409 base_mismatch` from `/__world/seed` | the overlay named another snapshot digest than the pod is on | the runner posts the digest of the version's snapshot; a mismatch is a version mismatch, never reseed silently |
|
|
@@ -497,4 +495,4 @@ handler or route edit keeps the imported data while an entity, field or key edit
|
|
|
497
495
|
- `connector-schema-authoring`: the data model the rows are checked against.
|
|
498
496
|
- `documentation/features/world-data-ingestion.md`: what exists, the contract, code pointers.
|
|
499
497
|
- `documentation/features/connector-worlds.md`: captures, ingest, serving the vendor's API.
|
|
500
|
-
- Consumer page:
|
|
498
|
+
- Consumer page: https://docs.surfacearea.ai/worlds/data.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: worlds-getting-started
|
|
3
|
-
description: Starter guide for building worlds and environments on the platform, end to end, for an agent (Claude Code, Cursor, Codex). Use when starting a world from scratch, building a world or environment, mocking a vendor, importing a Swagger or OpenAPI spec into a world (schema init --from-openapi), choosing between a shipped template, a workspace connector, an OpenAPI document, a vendor API capture or a custom contract, provisioning worlds for tenants in one call (worlds create), publishing with bench push, opening a session, running and grading, or iterating with pull, fork, diff and edit sessions (worlds edit status/close/rollback: version control without PRs). Every CLI step names its MCP twin (list_connectors, publish_connector, create_world, init_world_from_connector, env_write_file, env_commit, import_world_data, run_evaluation and the rest) and where the result shows in the UI.
|
|
3
|
+
description: Starter guide for building worlds and environments on the platform, end to end, for an agent (Claude Code, Cursor, Codex). Use when starting a world from scratch, building a world or environment, mocking a vendor, importing a Swagger or OpenAPI spec into a world (schema init --from-openapi), choosing between a shipped template, a workspace connector, an OpenAPI document, a vendor API capture or a custom contract, provisioning worlds for tenants in one call (worlds create), publishing with bench push, opening a session, reading every request a world received (worlds requests), checks over those requests (journal checks on gateway_requests, calls, verify.py, SQL verifiers; task try before you save, task regrade of past runs), running and grading, or iterating with pull, fork, diff and edit sessions (worlds edit status/close/rollback: version control without PRs). Every CLI step names its MCP twin (list_connectors, publish_connector, create_world, init_world_from_connector, env_write_file, env_commit, import_world_data, run_evaluation and the rest) and where the result shows in the UI.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# Worlds: getting started
|
|
@@ -17,7 +17,7 @@ The shortest correct path from nothing to a hosted world with data, a session, a
|
|
|
17
17
|
CLI first; the MCP twin sits under each step for an agent that talks to the platform over MCP instead of a shell.
|
|
18
18
|
Deeper references are linked by path, not repeated.
|
|
19
19
|
|
|
20
|
-
Vocabulary (from
|
|
20
|
+
Vocabulary (from https://docs.surfacearea.ai/glossary): **world** (the sealed replica), **scenario** (a task in
|
|
21
21
|
it), **rollout** (one attempt), **run** (a filed set of rollouts), **eval** (the grade), **gate** (the bar).
|
|
22
22
|
Identifiers keep the old words: `environment`, `container`, `task set`, `bench`.
|
|
23
23
|
|
|
@@ -155,7 +155,7 @@ environment; land them with `env_commit({ containerId, changeReason })` (one ver
|
|
|
155
155
|
Tool handlers (`tools/*.py`) reach the world's rows through `from gateway_env import store`. It has
|
|
156
156
|
exactly these methods; `gateway worlds schema check .` refuses any other `store.<name>` at file:line
|
|
157
157
|
with this list, and a hosted call into such a handler answers 400 `{error: "handler", detail, handler}`
|
|
158
|
-
(an `AttributeError`/`TypeError`/`NameError` is the author's bug; anything else a handler raises is a
|
|
158
|
+
(an `AttributeError`/`TypeError`/`NameError` is the author's bug; `raise Refusal(404, body)` (imported from `gateway_env` at module top) answers that 4xx and body; anything else a handler raises is a
|
|
159
159
|
500 `{error: "engine", detail, handler}`; never a bare exception string).
|
|
160
160
|
|
|
161
161
|
- `get(entity: str, key: dict) -> dict | None` — one row by primary key, or `None`.
|
|
@@ -192,7 +192,7 @@ gateway worlds describe ./x # entities, keys, re
|
|
|
192
192
|
gateway worlds data show ./x # kind + entity counts
|
|
193
193
|
gateway worlds data query ./x users --where email=jo@acme.test --resolve # the rows you mean; plaintext into a hashed field
|
|
194
194
|
gateway worlds data import ./x rows.json # rows file: {entity: [rows]} | JSONL | .csv/.tsv --entity (cells cast by the schema; --delimiter)
|
|
195
|
-
gateway files fetch https://example.test/rows.csv --to imports/rows.csv --header "Authorization: Bearer {{secret:API_KEY}}" # the platform fetches; the secret
|
|
195
|
+
gateway files fetch https://example.test/rows.csv --to imports/rows.csv --header "Authorization: Bearer {{secret:API_KEY}}" # the platform fetches; the secret goes only to origins it lists (gateway secret origins API_KEY https://example.test)
|
|
196
196
|
gateway worlds connector capture ./x --op <name> --arg k=v # from the real vendor, redacted
|
|
197
197
|
gateway worlds connector ingest ./x # captures -> data/initial.json; `--dry-run --report` checks only, nothing written
|
|
198
198
|
gateway worlds data calls pull calls.jsonl --from traces --session <id> # an agent's real calls, raw; prints records per tool + result shapes
|
|
@@ -220,8 +220,7 @@ gateway worlds connector publish ./x --slug x -m "first cut" # the template as
|
|
|
220
220
|
|
|
221
221
|
`bench push` makes the version sessions and runs address (`slug@version`). `connector publish` saves the contract
|
|
222
222
|
(never `data/`, `captures/`, `db/`) so anyone in the org can `schema init --connector workspace:x`. The world's
|
|
223
|
-
`tests/`, `tasks.json`, `verifiers/` and `openapi/` travel with the connector into every world initialized from it
|
|
224
|
-
(they used to be left behind, so the new world had no suite to run). Both, for a world you expect to rebuild.
|
|
223
|
+
`tests/`, `tasks.json`, `verifiers/` and `openapi/` travel with the connector into every world initialized from it. Both, for a world you expect to rebuild.
|
|
225
224
|
|
|
226
225
|
MCP: `publish_connector({ slug, files, message? })` is the template write path. `init_world_from_connector({ slug, name })`
|
|
227
226
|
makes the world under a PAT session; under an API key it returns the initialized files plus the publish steps
|
|
@@ -319,9 +318,10 @@ gateway worlds session open x --task-file task.json --surface api # the sessio
|
|
|
319
318
|
gateway worlds session status <SESSION_ID> # poll until ready
|
|
320
319
|
gateway worlds session seed <SESSION_ID> rows.json # rows into this session only
|
|
321
320
|
gateway worlds session state <SESSION_ID> # every entity's rows, the dump a grader reads
|
|
322
|
-
gateway worlds session reset <SESSION_ID> # back to the snapshot (+ the task's seed)
|
|
321
|
+
gateway worlds session reset <SESSION_ID> # back to the snapshot (+ the task's seed); `session restart <SESSION_ID> [--version <pin>]` does it on what its pin (x@ref; unpinned = newest you may read) points to NOW, same id, URLs + token, running or closed; a grade then reads only the requests after its session.restart ledger entry
|
|
323
322
|
gateway worlds session grade <SESSION_ID> [--timeout S] # {reward, rewards, raw}; exploratory → {reward: null, reason: "no grader"}
|
|
324
|
-
gateway worlds session export <SESSION_ID> --out calls.jsonl [--requests] # the platform's ledger; --requests
|
|
323
|
+
gateway worlds session export <SESSION_ID> --out calls.jsonl [--requests] # the platform's ledger; --requests adds the session's entries from the world's request ledger (kind "request": via, status, sizes; bodies by result.requestId), live or closed
|
|
324
|
+
gateway worlds requests x [--session <SESSION_ID>] [--via api|ui|tool|webhook] [--status 4xx] [--since 2026-10-05] [--json] # every request the world received (every session, version and door), newest first, kept for its life; reading records nothing; `requests show x <REQUEST_ID> [--out f]` adds headers + bodies, `session requests <SESSION_ID>` lists one session's (exit 3: its engine records no requests, only World Host sessions do)
|
|
325
325
|
gateway worlds data ingest x calls.jsonl --transform shape.py # the calls back in as rows, shaped by your code or --map ingest.toml (world-data-ingestion §2c)
|
|
326
326
|
gateway worlds session close <SESSION_ID> # a closed session reopens by id, same URLs + token, from its last checkpoint: gateway worlds session reopen <SESSION_ID>
|
|
327
327
|
```
|
|
@@ -344,7 +344,7 @@ verifiers) is refused `world_kind_retired`; one on an api bundle opens and lists
|
|
|
344
344
|
`instruction` is what the agent is told; `seed` (`{mode?, rows}`, the live-seed contract) is laid on the clone and re-applied by `reset`;
|
|
345
345
|
`grader` is `assertions` (`entity`, `where`, `exists|absent|count|all|any`, `field op value`; each check's `id` becomes a `rewards` key;
|
|
346
346
|
`entity: "calls"` reads the call log — `operation`, `args`, `selectors`, `status`, `rows`, `found`, and `via`: `ui` from the page's `api_path`,
|
|
347
|
-
`api` from the api origin, `tool` for a declared tool's call; a credential-named or secret-shaped field reads `[redacted]`; `rows`/`found` are null where the answer declares no rows, a request too large to keep names what it lacks in `withheld`, and a check grades whenever the calls it can read decide it, ungraded only where those could change the result), `{kind: "python", source, timeoutSeconds?}` (a `verify.py` body; it reads the state as JSON on stdin — `{"<entity>": {"<primary key>": row}, "calls": [...], "report": "<the agent's report>"}`, a verifier's entities as `<alias>.<entity>` — and prints ONE flat JSON object as the last line of stdout, `reward` plus a number from 0 to 1 per check — `true`/`false` read as 1/0, `null` or a string as ungraded, no `reward` leaves the task ungraded, and a nested object or array (a `rewards` map included), a number outside 0..1 or NaN/Infinity fails the grade by key and value; it must exit 0 within `timeoutSeconds`, default and most 30, or the grade fails with the reason) or `{kind: "rubric", rubric, criteria?, model?}`
|
|
347
|
+
`api` from the api origin, `tool` for a declared tool's call; a credential-named or secret-shaped field reads `[redacted]`; `rows`/`found` are null where the answer declares no rows, a request too large to keep names what it lacks in `withheld`, and a check grades whenever the calls it can read decide it, ungraded only where those could change the result; on a World Host session `calls` is read from the session's journal, every door and no bound, and `entity: "gateway_requests"` matches every request its world received — `operation`, `method`, `path`, `query.<k>`, `response.status`, `request.body.<path>`, `response.body.<path>`, with `before`/`after` to order them — failing closed, with the reason, where a journal missing requests cannot decide a check: `references/journal-checks.md`), `{kind: "python", source, timeoutSeconds?, readsJournal?, acceptsIncompleteJournal?}` (a `verify.py` body; it reads the state as JSON on stdin — `{"<entity>": {"<primary key>": row}, "calls": [...], "report": "<the agent's report>"}`, a verifier's entities as `<alias>.<entity>` — and prints ONE flat JSON object as the last line of stdout, `reward` plus a number from 0 to 1 per check — `true`/`false` read as 1/0, `null` or a string as ungraded, no `reward` leaves the task ungraded, and a nested object or array (a `rewards` map included), a number outside 0..1 or NaN/Infinity fails the grade by key and value; it must exit 0 within `timeoutSeconds`, default and most 30, or the grade fails with the reason) or `{kind: "rubric", rubric, criteria?, model?}`
|
|
348
348
|
(an LLM judge over end state + call log, graded on the platform: `grade` answers pending until
|
|
349
349
|
the worker finalizes it); `tools: [names]` narrows the operations the session may call to
|
|
350
350
|
exactly those (a name the world does not serve is a 400); `metadata` rides the session. At most one of
|
|
@@ -366,21 +366,20 @@ gateway worlds task update <TASK_ID> --spec task2.json # a changed spec is
|
|
|
366
366
|
gateway worlds session open x --task-id <TASK_ID> [--task-version 1] --surface api
|
|
367
367
|
gateway worlds session task save <SESSION_ID> --name acme-q3 # promote a session's inline task into a stored one
|
|
368
368
|
gateway worlds task delete <TASK_ID>
|
|
369
|
+
gateway worlds task try task.json --session <SESSION_ID> # a spec (or <TASK_ID>) graded on a stored World Host session, each check with the requests it matched; saves nothing
|
|
370
|
+
gateway worlds task regrade <TASK_ID> [--run <EVAL_ID>|--all-runs|--session <ID>] [--version N] # past runs graded again in place, no agent; default the latest run
|
|
369
371
|
```
|
|
370
372
|
|
|
371
|
-
SDK: `createTask / listTasks / getTask / updateTask / validateTask / deleteTask`,
|
|
372
|
-
`
|
|
373
|
-
`
|
|
374
|
-
(Python). MCP: `create_world_task`, `update_world_task`, `list_world_tasks`, `get_world_task`,
|
|
375
|
-
`validate_world_task`, `delete_world_task`, `save_world_session_task`. The world's Scenarios
|
|
376
|
-
tab lists the tasks pinned to it under a `world` kind chip.
|
|
373
|
+
SDK: `createTask / listTasks / getTask / updateTask / validateTask / deleteTask`, `openSession(slug, { task: { id, version? } })`, `session.saveTask(name)`, `new TaskGrades().try / regrade` (TS);
|
|
374
|
+
`create_task` … `delete_task`, `open_session(slug, {"id": …})`, `session.save_task(name)`, `TaskGrades().try_task / regrade` (Python). MCP: `create_world_task`, `update_world_task`, `list_world_tasks`, `get_world_task`,
|
|
375
|
+
`validate_world_task`, `delete_world_task`, `save_world_session_task`, `try_world_task`, `regrade_world_task`. The world's Scenarios tab lists the tasks pinned to it under a `world` kind chip.
|
|
377
376
|
|
|
378
377
|
**Stored tasks as a hosted eval.** A run config dispatches EITHER the bundle's scenarios
|
|
379
378
|
(`--tasks`) OR stored tasks (`--world-tasks`), never both:
|
|
380
379
|
|
|
381
380
|
```bash
|
|
382
381
|
gateway bench run-configs create --name nightly --models openai/gpt-5-mini --world-tasks <TASK_ID>,<OTHER_ID>@2
|
|
383
|
-
gateway bench dispatch <RUN_CONFIG_ID> # each stored task is a sample; rubric tasks are refused at dispatch
|
|
382
|
+
gateway bench dispatch <RUN_CONFIG_ID> # each stored task is a sample; rubric tasks and tasks whose checks read calls or requests are refused at dispatch
|
|
384
383
|
```
|
|
385
384
|
|
|
386
385
|
`<id>` runs the task's current version at dispatch time, `<id>@N` that version forever. SDK:
|
|
@@ -395,10 +394,10 @@ answers 503. That is a cold start, not a fault, until about twenty minutes have
|
|
|
395
394
|
`status` print `idleExpiresAt` (null once pinned) + `pinned`; `status` warns under 5 minutes. A call
|
|
396
395
|
body over 4mb is refused client-side: `{status: "error", code: "payload_too_large"}`, exit 2.
|
|
397
396
|
|
|
398
|
-
The world's API
|
|
399
|
-
the
|
|
397
|
+
The world's API mirrors the vendor's auth as `[auth]` declares it: the session's api token after an `Authorization` word the vendor takes, in its header, as its Basic user or password, its query key, a token the world's `[auth.login]` minted, or an
|
|
398
|
+
`[auth.downloads]` link; your own tooling sends it in `X-World-Session` (the SDKs' `apiHeaders`) or as a bearer, on every world. No auth for an eval: `session open --auth open` (or `mode = "open"` under `[auth]`) prints `api.auth: open` and a capability URL that needs no credential. `worlds serve` takes the `--api-key` pin (default `conform`) the vendor's ways alone (connector-worlds.md, "The door").
|
|
400
399
|
|
|
401
|
-
MCP: `seed_world_session`, `export_world_session`, `save_world_session_task`; open/call/close are CLI or REST only.
|
|
400
|
+
MCP: `seed_world_session`, `export_world_session`, `save_world_session_task`, `list_world_requests`, `get_world_request`; open/call/close are CLI or REST only.
|
|
402
401
|
Find what you opened: `gateway worlds session list` (every state; the live ones are `--state RUNNING` and `--state IDLE`) / `sandbox list`; `session pin <id>` stops the reaper.
|
|
403
402
|
|
|
404
403
|
## Test the world (not the agent)
|
|
@@ -433,7 +432,7 @@ gateway worlds call http://localhost:8080 slack.list_channels --args '{...}' --c
|
|
|
433
432
|
`GET /__session/{task,tools,state}`, `POST /__session/{call,grade,reset,seed,close}`; `--task` and
|
|
434
433
|
`--scenario` belong to `serve`. A directory target (`worlds call ./x <tool>`) opens and closes its
|
|
435
434
|
own session per call. A HOSTED session is not a served url (the World Host serves no `/__session/*`;
|
|
436
|
-
its `api.url` takes the session's
|
|
435
|
+
its `api.url` takes the session's token): act on it by id, `gateway worlds call <sessionId> <tool> --state --grade --close`.
|
|
437
436
|
|
|
438
437
|
## 7. Run and grade
|
|
439
438
|
|
|
@@ -463,18 +462,18 @@ MCP: `run_evaluation({ models | runConfigId, versionId?, numExamples?, rolloutsP
|
|
|
463
462
|
```bash
|
|
464
463
|
gateway bench pull x # reconstruct the head version's tree; `bench status ./x` = branch, base, local edits
|
|
465
464
|
gateway bench fork x --into x-experiment # a new container sharing content; `bench diff <base> <head>` per-file, `bench log x` versions
|
|
466
|
-
gateway worlds edit status x # your open EDIT SESSION: every push / sandbox commit / data import after the first version lands there; only
|
|
465
|
+
gateway worlds edit status x # your open EDIT SESSION: every push / sandbox commit / data import after the first version lands there; only your unpinned reads see it (no @ref: describe x, session open x, pull x, test x); x@main, version pins, runs and tasks get SEALED versions only
|
|
467
466
|
gateway worlds edit close x -m "why" # seal it: main moves to the session's tip for everyone; each version stays in `log`, `edit rollback` undoes the whole session (60 idle minutes seal it too). `edit list x` = history, `edit discard x` drops it unseen
|
|
468
467
|
gateway worlds edit rollback x <sessionId> # undo a sealed session (source + data together) as a new sealed session; --force when later sessions would go too
|
|
469
|
-
gateway worlds task edit close <task> -m "why" # a change across a TASK's worlds: seal them all in ONE step or none (`task edit status <task>` first: your unsealed worlds,
|
|
470
|
-
gateway worlds sandbox open x
|
|
468
|
+
gateway worlds task edit close <task> -m "why" # a change across a TASK's worlds: seal them all in ONE step or none (`task edit status <task>` first: your unsealed worlds, which every `task up` and run skips until sealed; `task edit rollback <task> --group <groupId>` undoes the group)
|
|
469
|
+
gateway worlds sandbox open x # no local checkout (no @ref: your open edit's tip, else main; x@main is the sealed main): write|exec|check|commit|close <id> edit the tree server-side; `bench sandbox` is the same set
|
|
471
470
|
gateway worlds sandbox exec <id> 'gateway worlds schema entity add . tickets --fields id:string,title:string -k id && gateway worlds schema compile . && gateway worlds data import . rows.json && gateway worlds schema check . && gateway worlds test .'
|
|
472
471
|
gateway worlds sandbox open --new y --from custom # a world that does not exist yet (no base); its first `commit` creates it — references/new-world-sandbox.md
|
|
473
472
|
gateway worlds sandbox import <id> data/trips.csv --entity trip # rows a sandbox made or curl'ed → the PLATFORM world, read by the platform; `commit` carries the source tree only
|
|
474
473
|
```
|
|
475
474
|
|
|
476
475
|
The sandbox carries the platform's `gateway` CLI and `sqlite3` on PATH: `exec` runs `worlds
|
|
477
|
-
schema|data|test|serve|call …` on the working copy with nothing on your machine; on a world whose data is a hosted snapshot, a tool, handler or
|
|
476
|
+
schema|data|test|serve|call …` on the working copy with nothing on your machine; on a world whose data is a hosted snapshot, `data show|sql`, `test` and `call` read its rows (laid beside the tree before each `exec`), a tool, handler, route, new entity or optional field compiles and keeps that data, any other domain edit is refused (`references/gotchas.md`). `open` prints
|
|
478
477
|
`cli: {version, digest}`, or `cli: null` plus the reason under `warnings`. Remote verbs (`push`, `run`,
|
|
479
478
|
`session open`) stop inside the sandbox by design: it holds no platform credential, and never write one
|
|
480
479
|
into its tree — `check`/`commit` hold back scratch, credential-shaped files (`.env`, `.gateway-env`, `*.env`,
|
|
@@ -489,6 +488,7 @@ MCP: `list_world_edits({ slug })`, `close_world_edit`, `discard_world_edit`, `ro
|
|
|
489
488
|
- `references/ui-map.md`: every route, per thing (connector page, world tabs, Worlds › Map, sessions, tests, stored tasks).
|
|
490
489
|
- `references/cli-mcp-map.md`: every CLI step with its MCP twin, the steps that have none, and how the platform (console) agent reaches the same surface (`run_gateway_cli`, `spawn_subagent`).
|
|
491
490
|
- `references/gotchas.md`: offline vs platform scaffolding, `bench push` vs `worlds create`, the World Host as the only engine, sharing, seeds, `max_tokens`, secret preflight, `secret set` upsert.
|
|
491
|
+
- `references/journal-checks.md`: checks over the requests a session's world received (declarative, `verify.py`, SQL), fail-closed reasons, `task try` and `task regrade`.
|
|
492
492
|
|
|
493
493
|
## Related
|
|
494
494
|
|
|
@@ -497,4 +497,4 @@ MCP: `list_world_edits({ slug })`, `close_world_edit`, `discard_world_edit`, `ro
|
|
|
497
497
|
- `documentation/features/connector-worlds.md` (`connector.toml`, capture, conform, serving), `workspace-connectors.md`
|
|
498
498
|
(the connector object, sharing, MCP surface), `world-schema.md` (the `gateway-world/1` contract and store API);
|
|
499
499
|
the first and last ship with the npm package (`gateway worlds skill connector-schema-authoring --install .`).
|
|
500
|
-
- Consumer pages: `
|
|
500
|
+
- Consumer pages (docs.surfacearea.ai): `worlds/getting-started`, `worlds/sessions`, `worlds/data`, `worlds/custom-connections`, `worlds/slack`.
|