@withgateway/sdk 0.2.6 → 0.2.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/README.md +24 -8
  2. package/dist/cli.js +15468 -11263
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +15233 -11028
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +291 -40
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +284 -37
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/hooks/index.d.mts +1 -1
  13. package/dist/hooks/index.d.ts +1 -1
  14. package/dist/hooks/index.js +203 -47
  15. package/dist/hooks/index.js.map +1 -1
  16. package/dist/hooks/index.mjs +198 -42
  17. package/dist/hooks/index.mjs.map +1 -1
  18. package/dist/{index-BaKu5ZSU.d.mts → index-C3UzD0G3.d.mts} +75 -1
  19. package/dist/{index-BaKu5ZSU.d.ts → index-C3UzD0G3.d.ts} +75 -1
  20. package/dist/{index-CLkMr6VG.d.ts → index-D-_4Ibcn.d.ts} +36 -4
  21. package/dist/{index-CdbmO4aX.d.mts → index-FfK4Nump.d.mts} +36 -4
  22. package/dist/index.d.mts +3 -3
  23. package/dist/index.d.ts +3 -3
  24. package/dist/index.js +2149 -615
  25. package/dist/index.js.map +1 -1
  26. package/dist/index.mjs +2053 -566
  27. package/dist/index.mjs.map +1 -1
  28. package/dist/traces/index.js +268 -31
  29. package/dist/traces/index.js.map +1 -1
  30. package/dist/traces/index.mjs +262 -28
  31. package/dist/traces/index.mjs.map +1 -1
  32. package/dist/tracing/index.js +19 -11
  33. package/dist/tracing/index.js.map +1 -1
  34. package/dist/tracing/index.mjs +19 -11
  35. package/dist/tracing/index.mjs.map +1 -1
  36. package/dist/worlds/index.d.mts +886 -117
  37. package/dist/worlds/index.d.ts +886 -117
  38. package/dist/worlds/index.js +1770 -452
  39. package/dist/worlds/index.js.map +1 -1
  40. package/dist/worlds/index.mjs +1752 -434
  41. package/dist/worlds/index.mjs.map +1 -1
  42. package/package.json +14 -10
  43. package/runtime/gatewaysdk/environment/_toml.py +3 -10
  44. package/runtime/gatewaysdk/environment/_world.py +95 -28
  45. package/runtime/gatewaysdk/environment/runtime.py +10 -6
  46. package/runtime/gatewaysdk/environment/schema/__main__.py +35 -0
  47. package/runtime/gatewaysdk/environment/schema/api.py +19 -23
  48. package/runtime/gatewaysdk/environment/schema/batch.py +68 -14
  49. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +4 -2
  50. package/runtime/gatewaysdk/environment/schema/cast.py +264 -0
  51. package/runtime/gatewaysdk/environment/schema/compiler.py +35 -1
  52. package/runtime/gatewaysdk/environment/schema/conform.py +28 -10
  53. package/runtime/gatewaysdk/environment/schema/connector.py +59 -14
  54. package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +38 -1
  55. package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.toml +3 -1
  56. package/runtime/gatewaysdk/environment/schema/credentials.py +20 -0
  57. package/runtime/gatewaysdk/environment/schema/csv_rows.py +117 -0
  58. package/runtime/gatewaysdk/environment/schema/cycles.py +121 -0
  59. package/runtime/gatewaysdk/environment/schema/data.py +21 -5
  60. package/runtime/gatewaysdk/environment/schema/grading.py +95 -13
  61. package/runtime/gatewaysdk/environment/schema/handler_errors.py +44 -0
  62. package/runtime/gatewaysdk/environment/schema/handler_scan.py +175 -0
  63. package/runtime/gatewaysdk/environment/schema/host.py +68 -115
  64. package/runtime/gatewaysdk/environment/schema/host_requests.py +78 -0
  65. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +12 -0
  66. package/runtime/gatewaysdk/environment/schema/ingest.py +5 -18
  67. package/runtime/gatewaysdk/environment/schema/local.py +7 -10
  68. package/runtime/gatewaysdk/environment/schema/openapi_document.py +322 -0
  69. package/runtime/gatewaysdk/environment/schema/openapi_import.py +382 -0
  70. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +441 -0
  71. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +629 -0
  72. package/runtime/gatewaysdk/environment/schema/openapi_source.py +210 -0
  73. package/runtime/gatewaysdk/environment/schema/platform.py +110 -299
  74. package/runtime/gatewaysdk/environment/schema/platform_import.py +182 -0
  75. package/runtime/gatewaysdk/environment/schema/platform_query.py +95 -0
  76. package/runtime/gatewaysdk/environment/schema/route_bindings.py +176 -0
  77. package/runtime/gatewaysdk/environment/schema/runtime.py +56 -7
  78. package/runtime/gatewaysdk/environment/schema/scaffold.py +64 -5
  79. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +70 -22
  80. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +54 -4
  81. package/runtime/gatewaysdk/environment/schema/skills/hooks-getting-started/SKILL.md +27 -2
  82. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +62 -0
  83. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +91 -54
  84. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +146 -146
  85. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +11 -7
  86. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +30 -0
  87. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/new-world-sandbox.md +43 -0
  88. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +90 -0
  89. package/runtime/gatewaysdk/environment/schema/store.py +15 -6
  90. package/runtime/gatewaysdk/environment/schema/store_contract.py +119 -0
  91. package/runtime/gatewaysdk/environment/schema/tasks.py +121 -32
  92. package/runtime/gatewaysdk/environment/schema/tools_world.py +32 -10
  93. package/runtime/gatewaysdk/environment/schema/tree_import.py +228 -0
  94. package/runtime/gatewaysdk/environment/schema/ui.py +18 -2
  95. package/runtime/gatewaysdk/environment/schema/validation.py +35 -0
  96. package/runtime/gatewaysdk/environment/schema/verbs.py +8 -2
  97. package/runtime/gatewaysdk/environment/world.py +42 -3
  98. package/runtime/gatewaysdk/environment/world_session_contract.py +57 -0
  99. package/skills/benchmarks-getting-started/SKILL.md +70 -22
  100. package/skills/connector-schema-authoring/SKILL.md +54 -4
  101. package/skills/connector-schema-authoring/connector-worlds.md +21 -7
  102. package/skills/connector-schema-authoring/world-schema.md +144 -83
  103. package/skills/hooks-getting-started/SKILL.md +27 -2
  104. package/skills/world-data-ingestion/ROWS.md +62 -0
  105. package/skills/world-data-ingestion/SKILL.md +91 -54
  106. package/skills/worlds-getting-started/SKILL.md +146 -146
  107. package/skills/worlds-getting-started/references/cli-mcp-map.md +11 -7
  108. package/skills/worlds-getting-started/references/gotchas.md +30 -0
  109. package/skills/worlds-getting-started/references/new-world-sandbox.md +43 -0
  110. package/skills/worlds-getting-started/references/openapi-import.md +90 -0
@@ -1,6 +1,6 @@
1
1
  ---
2
2
  name: benchmarks-getting-started
3
- description: Build, validate, dry-run, push, run and read a benchmark on the platform, for an agent (Claude Code, Cursor, Codex). A benchmark is either a world with tasks (tools + data, a schema when the vendor has one; a prompt and the SQL verifiers that grade the world's end state) or a container benchmark (a harbor task tree graded by its own tests/test.sh in a container, or a verifiers package graded by Python reward functions), run k times per task per model. Use when building a benchmark, an eval suite, a coding benchmark, a task set or a leaderboard for an agent; when you have a harbor task tree, task.toml files, a verifiers package, a Taskset, or tests that grade an agent; when choosing between a world benchmark and a container (benchmark runner) benchmark; when you need a dry run before pushing; when running k rollouts per task per model or reading per-task results. Every CLI step names its MCP twin (create_world, env_write_file, env_commit, create_environment_version, run_evaluation, get_task_performance, list_rollouts, get_rollout_samples) and where the result shows in the UI.
3
+ description: Build, validate, dry-run, push, run and read a benchmark on the platform, for an agent (Claude Code, Cursor, Codex). A benchmark is either a world with tasks (tools + data, a schema when the vendor has one; a prompt and the SQL verifiers that grade the world's end state) or a container benchmark (a harbor task tree graded by its own tests/test.sh in a container, or a verifiers package graded by Python reward functions), run k times per task per model. Use when building a benchmark, an eval suite, a coding benchmark, a task set or a leaderboard for an agent; when you have a harbor task tree, task.toml files, a verifiers package, a Taskset, or tests that grade an agent; when choosing between a world benchmark and a container (benchmark runner) benchmark; when you need a dry run before pushing (locally in Docker, or hosted on the benchmark runner with `--hosted` when the machine has no Docker); when running k rollouts per task per model or reading per-task results. Every CLI step names its MCP twin (create_world, env_write_file, env_commit, create_environment_version, run_evaluation, get_task_performance, list_rollouts, get_rollout_samples) and where the result shows in the UI.
4
4
  ---
5
5
 
6
6
  # Benchmarks: getting started
@@ -41,20 +41,23 @@ Segmentation, so one substrate never crashes into the other's infrastructure:
41
41
  | Storage | a world version (`POST /api/public/worlds`; later versions through the same hub as containers) | a benchmark-container version (`bench push`: resolve, versions, finalize) |
42
42
  | Engine | World Host (a hosted session per rollout; the mock-tools runner for the legacy bundle) | benchmark runner on AWS Batch: `harbor run` per task, or verifiers v1 `run_eval` in-process |
43
43
  | Results rows | evaluations, samples, `GET /environments/{id}/performance` | the same rows and endpoint |
44
+ | Dry run | local only: the task's session in the bundled runtime (`benchmarks dry-run <dir>`) | local (Docker, `benchmarks dry-run <dir>`) **or hosted** (`--hosted`: the runner, the oracle, the real image rule, `POST /environments/{id}/dry-run`, harbor only; a verifiers package or a world is refused with a 400 naming the local command); a hosted dry run is `kind: dry-run`, never evidence |
44
45
  | Refused when a tree crosses over | `worlds create` answers `world_kind_retired` for a harbor tree or a verifiers package, before any write; the hub's versions door answers `benchmark_substrate_mismatch` (409) for a container tree pushed under a world's slug | the hub's versions door answers `benchmark_substrate_mismatch` for a world tree pushed under a container's slug; the runner's harbor engine refuses a schema bundle (`require_legacy_bundle`) |
45
46
  | Read before writing | `benchmarks push` reads the slug's substrate (`GET /environments/{id}` → `substrate`) and refuses a cross-substrate push with exit 2 and no request; `--kind` that disagrees with the tree is refused before that | the same pre-check; a container benchmark that IS a container benchmark opens on the runner — `session open` / `task create` do not refuse it (only the legacy tools bundle is retired on serve) |
46
47
 
47
48
  ## 0. Authenticate
48
49
 
49
50
  ```bash
50
- npm install -g @withgateway/sdk # the gateway CLI (TypeScript; python3 >= 3.11 runs the local runtime)
51
+ npm install -g @withgateway/sdk # the gateway CLI (TypeScript; python3 >= 3.12 runs the local runtime)
51
52
  gateway auth login --host https://<your-gateway> --public-key pk-lf-... --api-key sk-lf-...
52
53
  ```
53
54
 
54
- `init`, `validate`, `dry-run` and the local world commands run without credentials; `push`, `run` and
55
- `results` need them. `dry-run` on a harbor tree executes in Docker when Docker is reachable and prints
56
- the exact plan when it is not; on a world it opens the task's session in the bundled Python runtime
57
- (python3 >= 3.11) and prints the plan when there is none.
55
+ `init`, `validate`, `dry-run` and the local world commands run without credentials; `push`, `run`,
56
+ `results` and `dry-run --hosted` need them. `dry-run` on a harbor tree executes in Docker when Docker
57
+ is reachable and prints the exact plan when it is not; `dry-run --hosted` runs the same one task on
58
+ the benchmark runner (harbor's oracle, no model), so a machine with nothing but the npm CLI and project
59
+ keys can author, dry-run, push and run a container benchmark; on a world it opens the task's session
60
+ in the bundled Python runtime (python3 >= 3.12) and prints the plan when there is none.
58
61
 
59
62
  ## A. A harbor (container) benchmark: init → validate → dry-run → push → run → results
60
63
 
@@ -177,6 +180,50 @@ docker-compose file requires EXA_API_KEY … export EXA_API_KEY here for the dry
177
180
  it from … gateway bench secrets set --env code-bench --key EXA_API_KEY`, exit 0, nothing spawned.
178
181
  Values never appear in the printed argv.
179
182
 
183
+ **Hosted (`--hosted`): the runner, the real image rule, no Docker here.** A Dockerfile-first local
184
+ dry-run passes a task whose `task.toml` names `docker_image`, because harbor never builds that
185
+ Dockerfile on the runner (A1); the hosted dry-run catches it in about two minutes, not in a
186
+ 15-minute model run.
187
+
188
+ ```bash
189
+ gateway benchmarks dry-run code-bench --task hello --hosted # a pushed slug (or slug@ref)
190
+ gateway benchmarks dry-run ./code-bench --task hello --hosted # a dir that tracks a slug (.gateway/.env-metadata.json)
191
+ gateway benchmarks dry-run ./code-bench --task hello --hosted --push # the dir differs from the pushed version: push first, then dry-run that version
192
+ gateway environment dry-run code-bench --task hello # the same run under the environment group (the Python CLI's spelling)
193
+ ```
194
+
195
+ ONE Batch job on the benchmark runner (`POST /api/public/environments/{id}/dry-run` with
196
+ `{ task, versionId? }`): harbor kind only, `--agent oracle` (the task's `solution/solve.sh` in place
197
+ of the agent), the ONE named task (the runner prunes the working copy to it, the same
198
+ `GATEWAY_TASK_NAMES` filter sharded runs use), one rollout, one example — capped on the server, a body
199
+ that asks for more is refused (400) — the image chosen by harbor's rule and the `bench secrets` compose
200
+ requirements enforced exactly like a real run (missing → refused before dispatch with the
201
+ `gateway bench secrets set …` command). Expected closing lines:
202
+
203
+ ```
204
+ image: build environment/Dockerfile # or `image: ghcr.io/acme/tool:1.2 (prebuilt: docker_image)`
205
+ ran: harbor run -p …/tasks -a oracle -e docker -o …/jobs -k 1 -n 1 -y -q -l 1 # no -m: the oracle calls no model; the working copy holds only `hello`
206
+ reward: 1 {"reward":1,"file_exists":1,"content_ok":1} · exit 0
207
+ where: hosted (code-bench 0.1.0 · 3f9a2c1d · task hello · agent oracle, no model · not evidence)
208
+ run: <run_id> · COMPLETED
209
+ page: https://<your-gateway>/project/<id>/world/runs/<evaluation_id>
210
+ status: gateway environment run-status <run_id>
211
+ ```
212
+
213
+ Exit 0 only when the oracle scored 1.0; lower → exit 2 naming it; nothing graded → exit 2
214
+ `reward: null (ungraded)`; an image pull/build failure → exit 2 with the runner's error tail (bounded,
215
+ no host paths, no secret values). `--wait` (default, `--wait-timeout 1200`) and `--no-wait` (return
216
+ after dispatch with the run id and the `run-status` command) mirror `benchmarks run`; `--json` prints
217
+ dispatch + verdict. A dir with uncommitted changes vs the pushed version is refused (exit 2, nothing
218
+ dispatched) unless `--push`, which goes through the ONE `benchmarks push` path. A world or a verifiers
219
+ package is refused with a 400 that names the local alternative (`benchmarks dry-run <dir>`).
220
+
221
+ **Not evidence.** The run is `kind: dry-run` (`evidence: false`, `agent: oracle`): the run page and
222
+ `environment run-status` show `dryRun`, the rollouts list labels the evaluation `dryRun: true`,
223
+ `benchmarks results` / `get_task_performance` exclude it, and the Batch job is named `bench-dryrun-…`
224
+ with `GATEWAY_DRY_RUN=oracle` in its environment, so an operator can tell it apart in the queue. MCP twin:
225
+ `dry_run_environment` (`containerId`, `task`, `versionId?`).
226
+
180
227
  ### A4. Push
181
228
 
182
229
  ```bash
@@ -211,15 +258,16 @@ MCP: `create_environment_version` (or `env_write_file` + `env_commit`).
211
258
  ```bash
212
259
  gateway benchmarks run code-bench --model openrouter/openai/gpt-5-mini --model anthropic/claude-sonnet-4-6 -k 3
213
260
  gateway benchmarks run code-bench -k 5 --no-wait # models from benchmark.json; return after dispatch
214
- gateway environment run-status <run_id> # one run's verdict; settled: false = still running
261
+ gateway environment run-status <run_id> # verdict; while running: `0/15 rollouts, 95s since dispatch`
262
+ gateway bench run <run_id> --cancel # CANCELLED here, runner job terminated on Batch (counts what Batch accepted)
215
263
  ```
216
264
 
217
265
  One runner job per model (`POST /api/public/environments/{id}/run`): it downloads the version, runs
218
266
  `harbor run -p tasks -a mini-swe-agent -m <model> -e docker -k <rollouts>` — the agent in each task's
219
267
  container, then `tests/test.sh` — and harvests every trial into an evaluation. Models are
220
268
  litellm-style, provider-prefixed (`openrouter/vendor/model`, `anthropic/…`); the project's LLM
221
- connection supplies the key. `--num-examples N` caps the tasks. The first run on a fresh version
222
- builds images and can take fifteen minutes.
269
+ connection supplies the key. `--num-examples N` caps the tasks; `-k` is 1–64 (the platform refuses
270
+ more before any row or job exists, whatever the client sent). The first run builds images: ~15 min.
223
271
 
224
272
  MCP: `run_evaluation({ models, versionId?, numExamples?, rolloutsPerExample? })`; `list_rollouts`.
225
273
 
@@ -227,7 +275,7 @@ MCP: `run_evaluation({ models, versionId?, numExamples?, rolloutsPerExample? })`
227
275
 
228
276
  ```bash
229
277
  gateway benchmarks results code-bench # per task: mean reward, pass rate, samples, no score
230
- gateway benchmarks results code-bench --runs --json
278
+ gateway benchmarks results code-bench --runs --json # also --trend | --all | --version 1.2.0
231
279
  gateway environment samples <container_id> <evaluation_id>
232
280
  ```
233
281
 
@@ -236,12 +284,12 @@ Expected:
236
284
  ```
237
285
  task mean reward pass rate samples no score
238
286
  hello 1.000 100% 3 0
239
- sort_csv 0.333 33% 3 0
240
- 6 samples, 0 without a score
287
+ 3 samples, 0 without a score
241
288
  ```
242
289
 
243
- `GET /api/public/environments/{id}/performance`; null scores are their own count. Sub-rewards from
244
- `reward.json` are on each sample. MCP: `get_task_performance`, `get_rollout_samples`.
290
+ `GET /api/public/environments/{id}/performance`; null scores are their own count; `reward.json` sub-rewards
291
+ are on each sample. MCP: `get_task_performance`, `get_rollout_samples`. Default is the newest version with
292
+ results; `--json` carries the table's selection (`view`, `version`, `tasks`), never every version's rows.
245
293
 
246
294
  | Thing | Where in the UI |
247
295
  | --- | --- |
@@ -361,7 +409,7 @@ reward is for the untouched world; an agent's rollout is what `benchmarks run` g
361
409
 
362
410
  A verifier that cannot run — bad SQL, a missing entity — prints `FAIL verifier 'x': …` with SQLite's
363
411
  words and exits 1; that is what the dry run is for. A reward of 1.0 here means the verifier passes
364
- before any agent acts: rewrite it. Without the bundled runtime (no python3 >= 3.11) the four steps
412
+ before any agent acts: rewrite it. Without the bundled runtime (no python3 >= 3.12) the four steps
365
413
  print and the run is marked not executed, exit 0.
366
414
 
367
415
  A hand-driven rollout, the way an agent's session goes:
@@ -419,8 +467,10 @@ Every run records the version it ran against; `results` shows the per-version tr
419
467
  | `benchmarks init` | none (local); `env_write_file` + `env_commit` on an existing environment |
420
468
  | `benchmarks validate` | world: `validate_connector_template({ files })`; container: none (local) |
421
469
  | `benchmarks dry-run` | none (local: Docker for harbor, the bundled Python runtime for a world) |
470
+ | `benchmarks dry-run --hosted` / `environment dry-run` | `dry_run_environment({ containerId, task, versionId? })`; `POST /api/public/environments/{id}/dry-run` (harbor only, oracle, one task, not evidence) |
422
471
  | `benchmarks push` | world: `create_world`, `POST /api/public/worlds`; container: `create_environment_version` (`bench push`) |
423
- | `benchmarks run` | `run_evaluation({ models, rolloutsPerExample })`; `POST /api/public/environments/{id}/run` |
472
+ | `benchmarks run` | `run_evaluation({ models, rolloutsPerExample })`; `POST /api/public/environments/{id}/run` (`rolloutsPerExample` 1–64) |
473
+ | `bench run <id> --cancel` / `environment run-status <id> --cancel` | `POST /api/public/environments/runs/{id}/cancel` (secret key; 409 once settled) |
424
474
  | `benchmarks results` | `get_task_performance`, `list_rollouts`; `GET /api/public/environments/{id}/performance` |
425
475
  | `worlds schema task add` | none (edits gateway-env.toml); stored tasks: `create_world_task` |
426
476
  | `gateway skill search <words>` | `guide({ topic })` |
@@ -433,20 +483,18 @@ Every run records the version it ran against; `results` shows the per-version tr
433
483
  writes nothing makes every trial ungraded (null), not 0. `mkdir -p /logs/verifier` first.
434
484
  - harbor: a `gateway-env.toml` at the root makes the runner read the tree as a world (`mock-tools`);
435
485
  `validate` names it.
436
- - harbor: `solution/solve.sh` is for `dry-run` and the oracle; the runner ignores it.
486
+ - harbor: `solution/solve.sh` is for `dry-run` (local, or `--hosted`: the oracle on the runner); a model run ignores it.
437
487
  - world: a verifier whose `initial_score` is 1.0 in `validate` — or whose `dry-run` reward is 1.0 — passes
438
488
  before the agent acts; write it against the state the task must produce. `verifiers/<name>.sql` must
439
489
  exist before `schema task add`; `dry-run` exits 1 naming a verifier whose SQL cannot run.
440
490
  - `run` needs a model per run; `benchmark.json` `models` fills in when `--model` is absent. A run
441
- that reads `settled: false` is still running, not failed.
491
+ that reads `settled: false` is still running, not failed; `bench run <id> --cancel` stops it.
442
492
  - `--kind image` is refused: the runner has no hub engine for a bare image (its generic engine runs
443
493
  registry entries only). Package the benchmark as a harbor task.
444
494
 
445
495
  ## Related
446
496
 
447
497
  - `worlds-getting-started`: the world itself — contract, data, sessions, stored tasks, links.
448
- - `world-data-ingestion`: rows in, calls out.
449
- - `connector-schema-authoring`: the contract, fidelity shapes, conventions.
450
- - Consumer pages: `gateway_documentation/pages/cli/benchmarks.mdx`, `pages/worlds/benchmarks.mdx`,
451
- `pages/evaluation/benchmarks-harbor.mdx`.
498
+ - `world-data-ingestion`: rows in, calls out. `connector-schema-authoring`: contract, fidelity shapes.
499
+ - Consumer pages: `gateway_documentation/pages/cli/benchmarks.mdx`, `pages/worlds/benchmarks.mdx`, `pages/evaluation/benchmarks-harbor.mdx`.
452
500
  - `documentation/features/benchmark-containers.md`: the container substrate, kinds, the runner.
@@ -55,6 +55,7 @@ registration, a platform builder, live ingestion, tools, or a registry framework
55
55
  | Command or path | Status |
56
56
  | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
57
57
  | `gateway worlds schema init PATH --connector <slug> [--name NAME] [--with-example-tool]` | exists (#219); `--connector custom` is the blank starting point for an arbitrary connector: its `$id` is derived from `--name` (`urn:gateway:connector:<name>`), it ships the `records` entity and a starter verifier but no tools unless `--with-example-tool` asks for the `list_records`/`get_record`/`create_record` example, and it writes no `data/records.json` (Brex retrospective, 2026-09-14) |
58
+ | `gateway worlds schema init PATH --from-openapi SOURCE [--plan-only] [--include GLOB\|TAG]… [--exclude GLOB\|TAG]…` | exists (2026-09-22); imports a Swagger 2.0 / OpenAPI 3.x document (JSON or YAML; a local path, `drive:<path>` read platform-side, or an http(s) URL) into `schema/world.json` + `connector.toml` + `schema/provenance.json`: keyed schemas the selected operations return become entities, every path+method a route (passthrough when the response is not an entity), the plan is printed first and `--plan-only` writes nothing; refusals (no `paths`, unknown `$ref`, cyclic entity root, undeclared path parameter) carry a JSON pointer. Rules: `worlds-getting-started` → `references/openapi-import.md` |
58
59
  | `gateway worlds connector validate\|capture\|ingest PATH` | exists; a `connector.toml` in the world describes base URL, auth scheme, pacing, redaction and operations; `capture` records paced, redacted pages (errors included) under `captures/` with the secret never written; `ingest` seeds `data/initial.json` through the contract in a scratch copy first; `ingest --dry-run [--report refusals.jsonl]` lists every violation of every capture (the `data import --report` records plus `capture`), exit 2, nothing written |
59
60
  | `gateway worlds schema reconcile PATH --from captures [--write]` | exists (runtime, offline CLI and `gateway worlds schema reconcile`); the contract against the captured rows: relaxes `required` for fields absent in captures, adds undocumented keys as optional properties typed from the values, drops formats the captured values fail; a report of every change, `schema/world.json` rewritten and recompiled only with `--write` |
60
61
  | `gateway worlds connector conform PATH --openapi vendor.yaml` | exists; serves the world in-process, calls every declared operation with arguments from its own seed, and checks status + body against the vendor's OpenAPI response schemas; non-zero exit on any mismatch. Run it before calling a world 1:1; its FAIL list is the work list (unseeded entities → 404s, creates the contract rejects → 4xx, honest 501s). Reads OpenAPI 3.0 `nullable`, reference cycles and `discriminator` unions; `[operations.api] sample_headers` / `sample_body` shape a request the route demands (`Idempotency-Key`, a body) |
@@ -92,7 +93,7 @@ deliverable can finish with synthetic fixtures and explicitly skipped live valid
92
93
  ### 0. Decide the shape
93
94
 
94
95
  - [ ] **Fidelity shape**: base, tenant-shaped, or mixed. See [CONNECTORS.md](CONNECTORS.md). Tenant-shaped means describe-on-bind is a requirement of the world, and the shared template is only a skeleton.
95
- - [ ] **Model source**: authored by hand (start from `--connector custom`), imported (`openapi`, `odata-csdl`, `discovery`, `wsdl-xsd`, `ldap-subschema`, `describe`, `docs`), or uploaded. Pin the spec URL and version now.
96
+ - [ ] **Model source**: authored by hand (start from `--connector custom`), imported from a Swagger/OpenAPI document (`schema init PATH --from-openapi SOURCE`, provenance pinned for you) or from another source by hand (`odata-csdl`, `discovery`, `wsdl-xsd`, `ldap-subschema`, `describe`, `docs`), or uploaded. Pin the spec URL and version now.
96
97
  - [ ] **Scope of the model**: which entities and fields, decided from what the connector's users need the world to be faithful about. Record the reasoning in `scope.md` or `scope.toml`; coverage of the vendor's full model is reported, not required.
97
98
  - [ ] **Sandbox**: record whether live validation is available and in scope. If it is not, record the reason and mark live checks `skipped`; a schema foundation can still ship as provisional `subset`.
98
99
  - [ ] **Tools and surface** (optional): none, generated defaults, hand-written, user's custom, or mapped to an MCP surface. Decide now so the entity scope covers what those tools touch.
@@ -135,6 +136,26 @@ deliverable can finish with synthetic fixtures and explicitly skipped live valid
135
136
  - [ ] Handlers never read `benchmark_runtime_context` or `benchmark_conversation_events`.
136
137
  - [ ] A user attaching custom tools to this world needs nothing from this checklist beyond the store API; document any conventions the model expects (key formats, enum values).
137
138
 
139
+ ### Store API
140
+
141
+ Tool handlers (`tools/*.py`) reach the world's rows through `from gateway_env import store`. It has
142
+ exactly these methods; `gateway worlds schema check .` refuses any other `store.<name>` at file:line
143
+ with this list, and a hosted call into such a handler answers 400 `{error: "handler", detail, handler}`
144
+ (an `AttributeError`/`TypeError`/`NameError` is the author's bug; anything else a handler raises is a
145
+ 500 `{error: "engine", detail, handler}`; never a bare exception string).
146
+
147
+ - `get(entity: str, key: dict) -> dict | None` — one row by primary key, or `None`.
148
+ - `list(entity: str) -> list[dict]` — every row, ordered by key.
149
+ - `list_where(entity: str, where: dict[str, list], limit: int | None = None, offset: int = 0) -> tuple[list[dict], int]` — a page of rows whose `where` fields hold one of the listed values, plus the total.
150
+ - `insert(entity: str, value: dict, *, source: dict | None = None) -> dict` — validate against the entity's schema and add; the key must be new.
151
+ - `update(entity: str, key: dict, patch: dict, *, source: dict | None = None) -> dict` — merge `patch` into the row at `key`, validate, return the row.
152
+ - `delete(entity: str, key: dict, *, source: dict | None = None) -> None` — remove the row at `key`; a missing row is a contract error.
153
+ - `apply_batch(batch: dict) -> dict` — one ingest batch `{id, target, mapping, coverage, mutations}`, atomically; an exact retry is a no-op.
154
+ - `lineage(entity: str, key: dict) -> list[dict]` — the batches that wrote the row at `key`, oldest first.
155
+ - `counts() -> dict[str, int]` — rows per entity, `{entity: count}`.
156
+
157
+ No `upsert`: `get`, then `insert` or `update`. Read-only SQL: `gateway_env.db.one/all`; raw SQL writes are refused. The scaffolded `tools/handlers.py` docstring carries the same table.
158
+
138
159
  ### 6. Surface binding (only if the world declares it mirrors an MCP surface)
139
160
 
140
161
  - [ ] Export a real connection's filtered `tools/list` through the MCP proxy under a grant, or inspect its `AgentSecurityResource` tool rows. Native Slack source capture has no live MCP catalog; its schema template tools are separate offline behavior.
@@ -142,6 +163,11 @@ deliverable can finish with synthetic fixtures and explicitly skipped live valid
142
163
  - [ ] Compare read-only tools manually against a live connection only when such comparison is authorized and in scope.
143
164
  - [ ] Native manifest changes bump `TOOL_VERSIONS`.
144
165
 
166
+ ### 6b. Reader plan (only if the template fills from an active Merge connection)
167
+
168
+ - [ ] `readers/merge.toml` names `transport = "merge"`, the Merge `provider` slug (`google_drive`, not the template slug) and one `[[pulls]]` per logical tool (`list_files`, unprefixed — `MergeConnection` adds `google_drive__`); arguments, paging (`page = { cursor, next }`) and `max_pages` live here and nowhere else: a caller (CLI, MCP, route) can never pick a tool or pass arguments. `readers/merge.ingest.toml` maps each tool's records to entities with the `data ingest --map` grammar.
169
+ - [ ] Prove it against the fake Agent Handler (`scripts/worlds-e2e/fake-merge-agent-handler.mjs`, dataset per fixture) through `gateway worlds create <slug> --from-connection <connector>` and `gateway worlds data pull <slug> --connection <id> --mode append`; every tool the plan names must land rows (`unmatched` empty). A live pull needs an owner-authorized active connection and is recorded as such, not implied by the fake.
170
+
145
171
  ### 7. Ship the template; register platform integration only when in scope
146
172
 
147
173
  - [ ] SDK-shipped: commit authored files under `gatewaysdk/sdk/src/gatewaysdk/environment/schema/connectors/<slug>/`; `list_templates()` and `test_schema_scaffold.py` cover the slug and preserve its authored dependencies/provenance during initialization. Schema-only templates declare an empty tools map and have no operation handlers.
@@ -167,7 +193,7 @@ When adding a bridge command to `schema/platform.py`, mirror it as a typed wrapp
167
193
  `worldSchemaRuntime.ts` (`compileWorldFiles` / `applyWorldBatch` are the pattern: file map
168
194
  in, `{entries, report, ...}` out, errors as `SchemaBridgeError`), cover it in
169
195
  `tests/environment/test_schema_platform.py`, and let the agent tool call the wrapper, never
170
- the subprocess. The bridge needs Python ≥ 3.11 (`tomllib`); set `GATEWAY_SCHEMA_PYTHON`
196
+ the subprocess. The bridge needs Python ≥ 3.12 (`tomllib`, and the store's sqlite `setconfig`); set `GATEWAY_SCHEMA_PYTHON`
171
197
  when the default interpreter is older.
172
198
 
173
199
  Use `worldSchemaRuntime.ts` for platform validation and `schema.platform` for
@@ -243,12 +269,19 @@ These source checks do not imply live-provider validation or executed fidelity s
243
269
 
244
270
  A world can describe the vendor it mirrors in `connector.toml` at its root: base URL,
245
271
  auth scheme (`header`, `bearer`, `basic`, `query`, `oauth2`, `none`) with the secret named by an
246
- environment variable, capture pacing, a redaction policy (`hash` or `drop` fields at any
272
+ environment variable (for `basic`, the credential is the Basic *user* and the password is ignored:
273
+ the host and `worlds serve` present `Basic b64("<key>:")`), capture pacing, a redaction policy (`hash` or `drop` fields at any
247
274
  depth), and the operations the agent actually calls (method, path with `{args}`, query
248
275
  params, and the entity plus results path their rows land in). The same file is meant to
249
276
  drive the platform's custom source later, so describe once.
250
277
 
251
- - `gateway worlds connector validate PATH` parses the spec and lists operations.
278
+ - `gateway worlds connector validate PATH` parses the spec, binds every operation's `entity`
279
+ (and each `explode` target) to the entities `schema/world.json` declares, and lists operations.
280
+ A route onto an undeclared entity is refused with `operation X lands on entity Y, which
281
+ schema/world.json does not declare (declared: …)` at `/operations/<i>/entity`; `schema
282
+ compile`, `schema check`, `worlds describe` and session open apply the same rule and write nothing
283
+ (it used to pass every gate, show under `served`, and 500 on the first call). A version stored
284
+ before this rule with such a route refuses to open instead; `worlds describe` names the fix.
252
285
  - `gateway worlds connector capture PATH --op NAME --arg k=v --param k=v` calls the live
253
286
  connection, follows the spec's cursor up to `max_pages`, and writes every page under
254
287
  `captures/<op>/` plus `captures/manifest.json` (sha256 per file). The recorded request
@@ -329,6 +362,23 @@ no `--host`/`--remote`/`workspace:<slug>`, platform env stripped, pinned to its
329
362
  so publishing always goes through the tools above. To parallelise several connectors, `spawn_subagent` with a brief
330
363
  naming only the tools that child needs (`allowSubagents` on the session).
331
364
 
365
+ ### From the CLI sandbox (no checkout, no Python)
366
+
367
+ `gateway worlds sandbox open <slug>@main` puts the same CLI in front of the world's source tree
368
+ server-side: `sandbox exec <id> 'gateway worlds schema entity add . <entity> --fields … -k id'`,
369
+ `'gateway worlds schema verifier add …'`, `'gateway worlds schema task add …'`, `'gateway worlds
370
+ schema compile . && gateway worlds schema check .'`, `'gateway worlds test .'` and `'gateway worlds
371
+ connector validate|conform .'` run on the uncommitted copy (`sqlite3` is on PATH too); `sandbox check`
372
+ runs the push gate and `sandbox commit` lands the version. `open` reports the staged CLI under `cli`
373
+ (`null` plus `warnings` when the image cannot host it — then only plain shell runs).
374
+
375
+ With an API key and no world yet, start the world itself there:
376
+ `gateway worlds sandbox open --new <slug> --from custom` (MCP `open_world_sandbox({ new, from })`)
377
+ opens on a world that does not exist yet, its tree the exact `schema init` scaffold (or
378
+ `--from <template>` for a shipped one); the same `sandbox exec` verbs run on that tree with the
379
+ staged CLI, `sandbox check` runs the push gate, and the first `sandbox commit -m …` creates the
380
+ world (first READY version on `main`). A taken slug is a 409 `world_exists`, never a rename.
381
+
332
382
  ### Serving the vendor's API from the world
333
383
 
334
384
  A provider's client speaks the vendor's REST paths, not tool calls, so the world answers
@@ -68,7 +68,7 @@ gateway worlds schema init ./spycloud-world --connector custom --name spycloud-w
68
68
  gateway worlds connector route add ./spycloud-world get_records_by_email \
69
69
  --path /breach/data/emails/{email} --entity breach_records --results results \
70
70
  --filter email=email
71
- gateway worlds connector validate ./spycloud-world
71
+ gateway worlds connector validate ./spycloud-world # syntax + every route's entity exists in schema/world.json
72
72
  gateway worlds connector capture ./spycloud-world --op get_records_by_email --arg email=probe@example.com \
73
73
  --param severity[]=high --param severity[]=critical # a repeated name goes out twice; an undeclared one is sent, with a note
74
74
  gateway worlds connector ingest ./spycloud-world --dry-run --report refusals.jsonl # every violation, nothing written
@@ -120,7 +120,7 @@ least one `[[operations]]` entry.
120
120
  | `[capture]` | `requests_per_minute`, `timeout_seconds`, `max_pages`, `cursor_param`, `cursor_field`, `cursor_header` | Pacing between pages; providers ask us to be sparing. The next cursor is read from a body field or a response header (Torchlight sends `Cursor:`), never both, and is sent back as `cursor_param`; a cursor that is an absolute URL on the connection's host (DRF's `next`) is followed by its query. |
121
121
  | `[redact]` | `hash`, `drop`, `round`, `preserve` | Applied before a body reaches disk. `preserve` hashes shape-preservingly (same length, digits stay digits, letters keep their case, separators stay; slug-salted, deterministic) so an account number still meets the field's format — `hash` made it a 71-char `sha256:` string the contract refused (Brex). `hash` writes `sha256:<hex>` salted with the slug so joins survive, element by element for arrays so the shape the contract expects survives too; `drop` removes; `round = { lat = 2, lon = 2 }` keeps numbers numeric but coarsens them. A bare name matches at any depth; a dotted path (`results.*.email`, `*` for list items) matches that place only, so a catalog's `assets` count map keeps its `email` key. Field names keep the vendor's casing. |
122
122
  | `[api]` | `envelope`, `hits_key`, `cursor_key`, `page`, `limit_param`, `offset_param`, `cursor_param`, `page_size`, `previous_key`, `page_links`, `page_headers`, `cursor_absent_when_done`, `unauthorized`, `not_found`, `invalid`, `*_status`, `error_content_type`, `token_path`, `handlers`, `before` | `envelope = "results"` wraps rows as `{results, hits, cursor}` under the vendor's key names; empty means a bare array. `hits_key = ""` leaves the count out and `cursor_absent_when_done = true` leaves the cursor key out of the last page (instead of `""`), together the `{items, next_cursor?}` shape Brex-style vendors answer and clients test with `"next_cursor" in body`. `page` is `none`, `offset` or `cursor`, with the vendor's parameter names; `previous_key` adds the previous page and `page_links = true` renders next/previous as absolute URLs of the served host, the way Django REST Framework answers; `page_headers = { cursor = "Cursor", total = "Total-Count" }` sends the same values as response headers for vendors that page that way (Torchlight). Error bodies are JSON strings replayed verbatim with the vendor's statuses and content type (`application/problem+json` for RFC 7807 vendors); `""` (or `'null'`) means no body at all, for vendors that answer a bare 401. An oauth2 world mints bearers on `token_path` (default the spec's `token_url`) and honours only those. `handlers` names the world's Python module (default `api/handlers.py` when present); `before` names a gate in it that runs before every operation, declarative ones included, and returns `None` to let the request through or an answer (quota, allow-list, key state) to stop it. |
123
- | `[[operations]]` | `name`, `method`, `path`, `params`, `entity`, `results` | `path` may carry `{args}`; a `params` name may end in `[]` (`expand[]`, `status[]`) for a key the vendor repeats, which capture then sends repeated and the served route filters on any value of; `results` is the dotted path to the row array (empty when the body is the row). An operation without `entity` never seeds and answers 501 `not_modelled`. |
123
+ | `[[operations]]` | `name`, `method`, `path`, `params`, `entity`, `results` | `path` may carry `{args}`; a `params` name may end in `[]` (`expand[]`, `status[]`) for a key the vendor repeats, which capture then sends repeated and the served route filters on any value of; `results` is the dotted path to the row array (empty when the body is the row). An operation without `entity` never seeds and answers 501 `not_modelled`. An `entity` (or an `explode` target) that `schema/world.json` does not declare is refused by `connector validate`, `schema compile`, `schema check` and the platform's describe (`gateway worlds describe`, local or platform) alike — `operation X lands on entity Y, which schema/world.json does not declare (declared: …)` at `/operations/<i>/entity` — and nothing is written; hand-edited TOML meets the same gate `route add` applies. |
124
124
  | `[operations.ingest]` | `drop`, `explode`, `carry_args` | `drop` removes vendor fields the entity does not carry; each `[[operations.ingest.explode]]` (`field`, `entity`, `carry`, `take`) turns a nested array into child rows; `carry_args = { account_id = "id" }` stamps the capture's path argument or query parameter onto every row before projection (a statement only knows its account from the URL it was fetched under). `connector ingest --dry-run [--report FILE]` lists every violation of every capture and writes nothing. |
125
125
  | `[operations.api]` | `filter`, `match`, `action`, `status`, `single`, `envelope`, `page`, `body`, `defaults`, `generate`, `handler`, `sample`, `sample_headers`, `sample_body`, `not_found`, `not_found_status`, `cursor_absent_when_done` | `filter` maps a path argument or declared query parameter to the row field it selects on (case-insensitive, any depth; an absent parameter does not narrow, one sent more than once matches rows carrying any of its values); `match = { name = "contains" }` switches an argument from equality to substring or `prefix`, and a list field matches when any element does. `action` is `read` (default), `create`, `update`, `delete` or `handler`: writes persist through the contract, `body` maps row fields (dotted for nested) to request-body paths, `defaults` fixes constants, `generate` fills `uuid`, `int` or `now`; `update` and `delete` find their row through `filter` and answer the vendor's 404 otherwise. `status` is the 2xx the vendor answers (201 for create, 204 for delete by default), `single` returns the row itself instead of an envelope, `envelope` and `page` override the world-wide setting per route. `handler` names a function `def name(ctx, args, query, body)` in the handlers module that returns a body, `(status, body)` or `(status, body, headers)`; `ctx` reads and writes the store (`rows`, `get`, `find`, `insert`, `update`, `delete`), keeps time (`now`, `elapsed`, `ladder` for status machines, frozen by `GATEWAY_WORLD_CLOCK`), mints ids (`uuid`) and shapes answers (`envelope`, `not_found`, `invalid`) and sees the request's `headers` and `query_all` (every value of every parameter, `{"expand[]": ["a", "b"]}`; the `query` argument keeps the first value only). `not_found` / `not_found_status` answer this route's missing row the vendor's own way (a 403 with its own message for one resource) and fall back to `[api]`; `cursor_absent_when_done` overrides `[api]` for this route. `sample = { ip = "203.0.113.5" }` gives the conformance run a value a validating route accepts when no seeded row can supply one; `sample_headers = { "Idempotency-Key" = "conform-1" }` and `sample_body = { reason = "conform" }` give it the headers and body a route demands (read by `connector conform` only; the served API never sends requests). |
126
126
 
@@ -142,11 +142,22 @@ its own (`uuid` for a string key, `int` for an integer one) unless `--body` maps
142
142
  `--generate` names it, so a POST that leaves the id out is created rather than refused with
143
143
  "Missing required property id". Composite keys are never minted.
144
144
 
145
- Refusals: a duplicate or malformed name, an unknown method, an undeclared entity, `--results`
146
- or a store action without `--entity`, a `--filter` key that is neither a `{path}` argument
147
- nor a `--param`, `action = "handler"` without `--handler`, a status outside 2xx, a `--generate`
148
- kind other than `uuid`/`int`/`now`, `--default`/`--generate` on a read route, or a file
149
- with no `[connector]` table. Source: `gatewaysdk/sdk-ts/src/cli/runtime/scaffold.ts`.
145
+ Refusals: a duplicate or malformed name (snake_case, at most 64 characters), an unknown method,
146
+ an undeclared entity, `--results` or a store action without `--entity`, a `--filter` key that is
147
+ neither a `{path}` argument nor a `--param`, a `{segment}` of the path that nothing binds (not
148
+ an entity field, not a `--filter`, not a `--param`; the refusal names the segment and the
149
+ `--param <segment>` that declares a scope argument), `action = "handler"` without `--handler`, a
150
+ status outside 2xx, a `--generate` kind other than `uuid`/`int`/`now`, `--default`/`--generate`
151
+ on a read route, or a file with no `[connector]` table. Source:
152
+ `gatewaysdk/sdk-ts/src/cli/runtime/scaffold.ts`; `schema compile` and `schema check` apply the
153
+ same binding rule to a hand-written `connector.toml` (`route_bindings.py`).
154
+
155
+ On a read route a declared `--param` that names a top-level field of the entity filters the
156
+ rows on its own (`GET /records?status=open` returns the open rows), the way a vendor's list
157
+ endpoint reads; `--filter` is for an argument whose name differs from the field's or that
158
+ selects at depth. The output of `route add` lists which arguments filter and which pass
159
+ through. A world built from a vendor document instead of by hand starts with
160
+ `gateway worlds schema init <dir> --from-openapi <spec>` ([world-schema](world-schema.md)).
150
161
 
151
162
  ## Writes and logic
152
163
 
@@ -209,6 +220,9 @@ its local runtime, with `GATEWAY_*`/`LANGFUSE_*` stripped and `--host`/`--remote
209
220
 
210
221
  ## Conformance and scenarios
211
222
 
223
+ Have the vendor's document and no world yet? `gateway worlds schema init <dir> --from-openapi
224
+ <spec>` writes the schema and the routes from it; `conform` is the check that comes after.
225
+
212
226
  `gateway worlds connector conform <world> --openapi <vendor.yaml>` serves the world in-process,
213
227
  drives every declared operation with arguments drawn from the world's own seeded rows (so lookups
214
228
  hit data), and checks each answer against the vendor's OpenAPI: the documented 2xx status, and a