@withgateway/sdk 0.2.6 → 0.2.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/README.md +24 -8
  2. package/dist/cli.js +15468 -11263
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +15233 -11028
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +291 -40
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +284 -37
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/hooks/index.d.mts +1 -1
  13. package/dist/hooks/index.d.ts +1 -1
  14. package/dist/hooks/index.js +203 -47
  15. package/dist/hooks/index.js.map +1 -1
  16. package/dist/hooks/index.mjs +198 -42
  17. package/dist/hooks/index.mjs.map +1 -1
  18. package/dist/{index-BaKu5ZSU.d.mts → index-C3UzD0G3.d.mts} +75 -1
  19. package/dist/{index-BaKu5ZSU.d.ts → index-C3UzD0G3.d.ts} +75 -1
  20. package/dist/{index-CLkMr6VG.d.ts → index-D-_4Ibcn.d.ts} +36 -4
  21. package/dist/{index-CdbmO4aX.d.mts → index-FfK4Nump.d.mts} +36 -4
  22. package/dist/index.d.mts +3 -3
  23. package/dist/index.d.ts +3 -3
  24. package/dist/index.js +2149 -615
  25. package/dist/index.js.map +1 -1
  26. package/dist/index.mjs +2053 -566
  27. package/dist/index.mjs.map +1 -1
  28. package/dist/traces/index.js +268 -31
  29. package/dist/traces/index.js.map +1 -1
  30. package/dist/traces/index.mjs +262 -28
  31. package/dist/traces/index.mjs.map +1 -1
  32. package/dist/tracing/index.js +19 -11
  33. package/dist/tracing/index.js.map +1 -1
  34. package/dist/tracing/index.mjs +19 -11
  35. package/dist/tracing/index.mjs.map +1 -1
  36. package/dist/worlds/index.d.mts +886 -117
  37. package/dist/worlds/index.d.ts +886 -117
  38. package/dist/worlds/index.js +1770 -452
  39. package/dist/worlds/index.js.map +1 -1
  40. package/dist/worlds/index.mjs +1752 -434
  41. package/dist/worlds/index.mjs.map +1 -1
  42. package/package.json +14 -10
  43. package/runtime/gatewaysdk/environment/_toml.py +3 -10
  44. package/runtime/gatewaysdk/environment/_world.py +95 -28
  45. package/runtime/gatewaysdk/environment/runtime.py +10 -6
  46. package/runtime/gatewaysdk/environment/schema/__main__.py +35 -0
  47. package/runtime/gatewaysdk/environment/schema/api.py +19 -23
  48. package/runtime/gatewaysdk/environment/schema/batch.py +68 -14
  49. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +4 -2
  50. package/runtime/gatewaysdk/environment/schema/cast.py +264 -0
  51. package/runtime/gatewaysdk/environment/schema/compiler.py +35 -1
  52. package/runtime/gatewaysdk/environment/schema/conform.py +28 -10
  53. package/runtime/gatewaysdk/environment/schema/connector.py +59 -14
  54. package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +38 -1
  55. package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.toml +3 -1
  56. package/runtime/gatewaysdk/environment/schema/credentials.py +20 -0
  57. package/runtime/gatewaysdk/environment/schema/csv_rows.py +117 -0
  58. package/runtime/gatewaysdk/environment/schema/cycles.py +121 -0
  59. package/runtime/gatewaysdk/environment/schema/data.py +21 -5
  60. package/runtime/gatewaysdk/environment/schema/grading.py +95 -13
  61. package/runtime/gatewaysdk/environment/schema/handler_errors.py +44 -0
  62. package/runtime/gatewaysdk/environment/schema/handler_scan.py +175 -0
  63. package/runtime/gatewaysdk/environment/schema/host.py +68 -115
  64. package/runtime/gatewaysdk/environment/schema/host_requests.py +78 -0
  65. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +12 -0
  66. package/runtime/gatewaysdk/environment/schema/ingest.py +5 -18
  67. package/runtime/gatewaysdk/environment/schema/local.py +7 -10
  68. package/runtime/gatewaysdk/environment/schema/openapi_document.py +322 -0
  69. package/runtime/gatewaysdk/environment/schema/openapi_import.py +382 -0
  70. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +441 -0
  71. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +629 -0
  72. package/runtime/gatewaysdk/environment/schema/openapi_source.py +210 -0
  73. package/runtime/gatewaysdk/environment/schema/platform.py +110 -299
  74. package/runtime/gatewaysdk/environment/schema/platform_import.py +182 -0
  75. package/runtime/gatewaysdk/environment/schema/platform_query.py +95 -0
  76. package/runtime/gatewaysdk/environment/schema/route_bindings.py +176 -0
  77. package/runtime/gatewaysdk/environment/schema/runtime.py +56 -7
  78. package/runtime/gatewaysdk/environment/schema/scaffold.py +64 -5
  79. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +70 -22
  80. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +54 -4
  81. package/runtime/gatewaysdk/environment/schema/skills/hooks-getting-started/SKILL.md +27 -2
  82. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +62 -0
  83. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +91 -54
  84. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +146 -146
  85. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +11 -7
  86. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +30 -0
  87. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/new-world-sandbox.md +43 -0
  88. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +90 -0
  89. package/runtime/gatewaysdk/environment/schema/store.py +15 -6
  90. package/runtime/gatewaysdk/environment/schema/store_contract.py +119 -0
  91. package/runtime/gatewaysdk/environment/schema/tasks.py +121 -32
  92. package/runtime/gatewaysdk/environment/schema/tools_world.py +32 -10
  93. package/runtime/gatewaysdk/environment/schema/tree_import.py +228 -0
  94. package/runtime/gatewaysdk/environment/schema/ui.py +18 -2
  95. package/runtime/gatewaysdk/environment/schema/validation.py +35 -0
  96. package/runtime/gatewaysdk/environment/schema/verbs.py +8 -2
  97. package/runtime/gatewaysdk/environment/world.py +42 -3
  98. package/runtime/gatewaysdk/environment/world_session_contract.py +57 -0
  99. package/skills/benchmarks-getting-started/SKILL.md +70 -22
  100. package/skills/connector-schema-authoring/SKILL.md +54 -4
  101. package/skills/connector-schema-authoring/connector-worlds.md +21 -7
  102. package/skills/connector-schema-authoring/world-schema.md +144 -83
  103. package/skills/hooks-getting-started/SKILL.md +27 -2
  104. package/skills/world-data-ingestion/ROWS.md +62 -0
  105. package/skills/world-data-ingestion/SKILL.md +91 -54
  106. package/skills/worlds-getting-started/SKILL.md +146 -146
  107. package/skills/worlds-getting-started/references/cli-mcp-map.md +11 -7
  108. package/skills/worlds-getting-started/references/gotchas.md +30 -0
  109. package/skills/worlds-getting-started/references/new-world-sandbox.md +43 -0
  110. package/skills/worlds-getting-started/references/openapi-import.md +90 -0
@@ -1,6 +1,6 @@
1
1
  ---
2
2
  name: benchmarks-getting-started
3
- description: Build, validate, dry-run, push, run and read a benchmark on the platform, for an agent (Claude Code, Cursor, Codex). A benchmark is either a world with tasks (tools + data, a schema when the vendor has one; a prompt and the SQL verifiers that grade the world's end state) or a container benchmark (a harbor task tree graded by its own tests/test.sh in a container, or a verifiers package graded by Python reward functions), run k times per task per model. Use when building a benchmark, an eval suite, a coding benchmark, a task set or a leaderboard for an agent; when you have a harbor task tree, task.toml files, a verifiers package, a Taskset, or tests that grade an agent; when choosing between a world benchmark and a container (benchmark runner) benchmark; when you need a dry run before pushing; when running k rollouts per task per model or reading per-task results. Every CLI step names its MCP twin (create_world, env_write_file, env_commit, create_environment_version, run_evaluation, get_task_performance, list_rollouts, get_rollout_samples) and where the result shows in the UI.
3
+ description: Build, validate, dry-run, push, run and read a benchmark on the platform, for an agent (Claude Code, Cursor, Codex). A benchmark is either a world with tasks (tools + data, a schema when the vendor has one; a prompt and the SQL verifiers that grade the world's end state) or a container benchmark (a harbor task tree graded by its own tests/test.sh in a container, or a verifiers package graded by Python reward functions), run k times per task per model. Use when building a benchmark, an eval suite, a coding benchmark, a task set or a leaderboard for an agent; when you have a harbor task tree, task.toml files, a verifiers package, a Taskset, or tests that grade an agent; when choosing between a world benchmark and a container (benchmark runner) benchmark; when you need a dry run before pushing (locally in Docker, or hosted on the benchmark runner with `--hosted` when the machine has no Docker); when running k rollouts per task per model or reading per-task results. Every CLI step names its MCP twin (create_world, env_write_file, env_commit, create_environment_version, run_evaluation, get_task_performance, list_rollouts, get_rollout_samples) and where the result shows in the UI.
4
4
  ---
5
5
 
6
6
  # Benchmarks: getting started
@@ -41,20 +41,23 @@ Segmentation, so one substrate never crashes into the other's infrastructure:
41
41
  | Storage | a world version (`POST /api/public/worlds`; later versions through the same hub as containers) | a benchmark-container version (`bench push`: resolve, versions, finalize) |
42
42
  | Engine | World Host (a hosted session per rollout; the mock-tools runner for the legacy bundle) | benchmark runner on AWS Batch: `harbor run` per task, or verifiers v1 `run_eval` in-process |
43
43
  | Results rows | evaluations, samples, `GET /environments/{id}/performance` | the same rows and endpoint |
44
+ | Dry run | local only: the task's session in the bundled runtime (`benchmarks dry-run <dir>`) | local (Docker, `benchmarks dry-run <dir>`) **or hosted** (`--hosted`: the runner, the oracle, the real image rule, `POST /environments/{id}/dry-run`, harbor only; a verifiers package or a world is refused with a 400 naming the local command); a hosted dry run is `kind: dry-run`, never evidence |
44
45
  | Refused when a tree crosses over | `worlds create` answers `world_kind_retired` for a harbor tree or a verifiers package, before any write; the hub's versions door answers `benchmark_substrate_mismatch` (409) for a container tree pushed under a world's slug | the hub's versions door answers `benchmark_substrate_mismatch` for a world tree pushed under a container's slug; the runner's harbor engine refuses a schema bundle (`require_legacy_bundle`) |
45
46
  | Read before writing | `benchmarks push` reads the slug's substrate (`GET /environments/{id}` → `substrate`) and refuses a cross-substrate push with exit 2 and no request; `--kind` that disagrees with the tree is refused before that | the same pre-check; a container benchmark that IS a container benchmark opens on the runner — `session open` / `task create` do not refuse it (only the legacy tools bundle is retired on serve) |
46
47
 
47
48
  ## 0. Authenticate
48
49
 
49
50
  ```bash
50
- npm install -g @withgateway/sdk # the gateway CLI (TypeScript; python3 >= 3.11 runs the local runtime)
51
+ npm install -g @withgateway/sdk # the gateway CLI (TypeScript; python3 >= 3.12 runs the local runtime)
51
52
  gateway auth login --host https://<your-gateway> --public-key pk-lf-... --api-key sk-lf-...
52
53
  ```
53
54
 
54
- `init`, `validate`, `dry-run` and the local world commands run without credentials; `push`, `run` and
55
- `results` need them. `dry-run` on a harbor tree executes in Docker when Docker is reachable and prints
56
- the exact plan when it is not; on a world it opens the task's session in the bundled Python runtime
57
- (python3 >= 3.11) and prints the plan when there is none.
55
+ `init`, `validate`, `dry-run` and the local world commands run without credentials; `push`, `run`,
56
+ `results` and `dry-run --hosted` need them. `dry-run` on a harbor tree executes in Docker when Docker
57
+ is reachable and prints the exact plan when it is not; `dry-run --hosted` runs the same one task on
58
+ the benchmark runner (harbor's oracle, no model), so a machine with nothing but the npm CLI and project
59
+ keys can author, dry-run, push and run a container benchmark; on a world it opens the task's session
60
+ in the bundled Python runtime (python3 >= 3.12) and prints the plan when there is none.
58
61
 
59
62
  ## A. A harbor (container) benchmark: init → validate → dry-run → push → run → results
60
63
 
@@ -177,6 +180,50 @@ docker-compose file requires EXA_API_KEY … export EXA_API_KEY here for the dry
177
180
  it from … gateway bench secrets set --env code-bench --key EXA_API_KEY`, exit 0, nothing spawned.
178
181
  Values never appear in the printed argv.
179
182
 
183
+ **Hosted (`--hosted`): the runner, the real image rule, no Docker here.** A Dockerfile-first local
184
+ dry-run passes a task whose `task.toml` names `docker_image`, because harbor never builds that
185
+ Dockerfile on the runner (A1); the hosted dry-run catches it in about two minutes, not in a
186
+ 15-minute model run.
187
+
188
+ ```bash
189
+ gateway benchmarks dry-run code-bench --task hello --hosted # a pushed slug (or slug@ref)
190
+ gateway benchmarks dry-run ./code-bench --task hello --hosted # a dir that tracks a slug (.gateway/.env-metadata.json)
191
+ gateway benchmarks dry-run ./code-bench --task hello --hosted --push # the dir differs from the pushed version: push first, then dry-run that version
192
+ gateway environment dry-run code-bench --task hello # the same run under the environment group (the Python CLI's spelling)
193
+ ```
194
+
195
+ ONE Batch job on the benchmark runner (`POST /api/public/environments/{id}/dry-run` with
196
+ `{ task, versionId? }`): harbor kind only, `--agent oracle` (the task's `solution/solve.sh` in place
197
+ of the agent), the ONE named task (the runner prunes the working copy to it, the same
198
+ `GATEWAY_TASK_NAMES` filter sharded runs use), one rollout, one example — capped on the server, a body
199
+ that asks for more is refused (400) — the image chosen by harbor's rule and the `bench secrets` compose
200
+ requirements enforced exactly like a real run (missing → refused before dispatch with the
201
+ `gateway bench secrets set …` command). Expected closing lines:
202
+
203
+ ```
204
+ image: build environment/Dockerfile # or `image: ghcr.io/acme/tool:1.2 (prebuilt: docker_image)`
205
+ ran: harbor run -p …/tasks -a oracle -e docker -o …/jobs -k 1 -n 1 -y -q -l 1 # no -m: the oracle calls no model; the working copy holds only `hello`
206
+ reward: 1 {"reward":1,"file_exists":1,"content_ok":1} · exit 0
207
+ where: hosted (code-bench 0.1.0 · 3f9a2c1d · task hello · agent oracle, no model · not evidence)
208
+ run: <run_id> · COMPLETED
209
+ page: https://<your-gateway>/project/<id>/world/runs/<evaluation_id>
210
+ status: gateway environment run-status <run_id>
211
+ ```
212
+
213
+ Exit 0 only when the oracle scored 1.0; lower → exit 2 naming it; nothing graded → exit 2
214
+ `reward: null (ungraded)`; an image pull/build failure → exit 2 with the runner's error tail (bounded,
215
+ no host paths, no secret values). `--wait` (default, `--wait-timeout 1200`) and `--no-wait` (return
216
+ after dispatch with the run id and the `run-status` command) mirror `benchmarks run`; `--json` prints
217
+ dispatch + verdict. A dir with uncommitted changes vs the pushed version is refused (exit 2, nothing
218
+ dispatched) unless `--push`, which goes through the ONE `benchmarks push` path. A world or a verifiers
219
+ package is refused with a 400 that names the local alternative (`benchmarks dry-run <dir>`).
220
+
221
+ **Not evidence.** The run is `kind: dry-run` (`evidence: false`, `agent: oracle`): the run page and
222
+ `environment run-status` show `dryRun`, the rollouts list labels the evaluation `dryRun: true`,
223
+ `benchmarks results` / `get_task_performance` exclude it, and the Batch job is named `bench-dryrun-…`
224
+ with `GATEWAY_DRY_RUN=oracle` in its environment, so an operator can tell it apart in the queue. MCP twin:
225
+ `dry_run_environment` (`containerId`, `task`, `versionId?`).
226
+
180
227
  ### A4. Push
181
228
 
182
229
  ```bash
@@ -211,15 +258,16 @@ MCP: `create_environment_version` (or `env_write_file` + `env_commit`).
211
258
  ```bash
212
259
  gateway benchmarks run code-bench --model openrouter/openai/gpt-5-mini --model anthropic/claude-sonnet-4-6 -k 3
213
260
  gateway benchmarks run code-bench -k 5 --no-wait # models from benchmark.json; return after dispatch
214
- gateway environment run-status <run_id> # one run's verdict; settled: false = still running
261
+ gateway environment run-status <run_id> # verdict; while running: `0/15 rollouts, 95s since dispatch`
262
+ gateway bench run <run_id> --cancel # CANCELLED here, runner job terminated on Batch (counts what Batch accepted)
215
263
  ```
216
264
 
217
265
  One runner job per model (`POST /api/public/environments/{id}/run`): it downloads the version, runs
218
266
  `harbor run -p tasks -a mini-swe-agent -m <model> -e docker -k <rollouts>` — the agent in each task's
219
267
  container, then `tests/test.sh` — and harvests every trial into an evaluation. Models are
220
268
  litellm-style, provider-prefixed (`openrouter/vendor/model`, `anthropic/…`); the project's LLM
221
- connection supplies the key. `--num-examples N` caps the tasks. The first run on a fresh version
222
- builds images and can take fifteen minutes.
269
+ connection supplies the key. `--num-examples N` caps the tasks; `-k` is 1–64 (the platform refuses
270
+ more before any row or job exists, whatever the client sent). The first run builds images: ~15 min.
223
271
 
224
272
  MCP: `run_evaluation({ models, versionId?, numExamples?, rolloutsPerExample? })`; `list_rollouts`.
225
273
 
@@ -227,7 +275,7 @@ MCP: `run_evaluation({ models, versionId?, numExamples?, rolloutsPerExample? })`
227
275
 
228
276
  ```bash
229
277
  gateway benchmarks results code-bench # per task: mean reward, pass rate, samples, no score
230
- gateway benchmarks results code-bench --runs --json
278
+ gateway benchmarks results code-bench --runs --json # also --trend | --all | --version 1.2.0
231
279
  gateway environment samples <container_id> <evaluation_id>
232
280
  ```
233
281
 
@@ -236,12 +284,12 @@ Expected:
236
284
  ```
237
285
  task mean reward pass rate samples no score
238
286
  hello 1.000 100% 3 0
239
- sort_csv 0.333 33% 3 0
240
- 6 samples, 0 without a score
287
+ 3 samples, 0 without a score
241
288
  ```
242
289
 
243
- `GET /api/public/environments/{id}/performance`; null scores are their own count. Sub-rewards from
244
- `reward.json` are on each sample. MCP: `get_task_performance`, `get_rollout_samples`.
290
+ `GET /api/public/environments/{id}/performance`; null scores are their own count; `reward.json` sub-rewards
291
+ are on each sample. MCP: `get_task_performance`, `get_rollout_samples`. Default is the newest version with
292
+ results; `--json` carries the table's selection (`view`, `version`, `tasks`), never every version's rows.
245
293
 
246
294
  | Thing | Where in the UI |
247
295
  | --- | --- |
@@ -361,7 +409,7 @@ reward is for the untouched world; an agent's rollout is what `benchmarks run` g
361
409
 
362
410
  A verifier that cannot run — bad SQL, a missing entity — prints `FAIL verifier 'x': …` with SQLite's
363
411
  words and exits 1; that is what the dry run is for. A reward of 1.0 here means the verifier passes
364
- before any agent acts: rewrite it. Without the bundled runtime (no python3 >= 3.11) the four steps
412
+ before any agent acts: rewrite it. Without the bundled runtime (no python3 >= 3.12) the four steps
365
413
  print and the run is marked not executed, exit 0.
366
414
 
367
415
  A hand-driven rollout, the way an agent's session goes:
@@ -419,8 +467,10 @@ Every run records the version it ran against; `results` shows the per-version tr
419
467
  | `benchmarks init` | none (local); `env_write_file` + `env_commit` on an existing environment |
420
468
  | `benchmarks validate` | world: `validate_connector_template({ files })`; container: none (local) |
421
469
  | `benchmarks dry-run` | none (local: Docker for harbor, the bundled Python runtime for a world) |
470
+ | `benchmarks dry-run --hosted` / `environment dry-run` | `dry_run_environment({ containerId, task, versionId? })`; `POST /api/public/environments/{id}/dry-run` (harbor only, oracle, one task, not evidence) |
422
471
  | `benchmarks push` | world: `create_world`, `POST /api/public/worlds`; container: `create_environment_version` (`bench push`) |
423
- | `benchmarks run` | `run_evaluation({ models, rolloutsPerExample })`; `POST /api/public/environments/{id}/run` |
472
+ | `benchmarks run` | `run_evaluation({ models, rolloutsPerExample })`; `POST /api/public/environments/{id}/run` (`rolloutsPerExample` 1–64) |
473
+ | `bench run <id> --cancel` / `environment run-status <id> --cancel` | `POST /api/public/environments/runs/{id}/cancel` (secret key; 409 once settled) |
424
474
  | `benchmarks results` | `get_task_performance`, `list_rollouts`; `GET /api/public/environments/{id}/performance` |
425
475
  | `worlds schema task add` | none (edits gateway-env.toml); stored tasks: `create_world_task` |
426
476
  | `gateway skill search <words>` | `guide({ topic })` |
@@ -433,20 +483,18 @@ Every run records the version it ran against; `results` shows the per-version tr
433
483
  writes nothing makes every trial ungraded (null), not 0. `mkdir -p /logs/verifier` first.
434
484
  - harbor: a `gateway-env.toml` at the root makes the runner read the tree as a world (`mock-tools`);
435
485
  `validate` names it.
436
- - harbor: `solution/solve.sh` is for `dry-run` and the oracle; the runner ignores it.
486
+ - harbor: `solution/solve.sh` is for `dry-run` (local, or `--hosted`: the oracle on the runner); a model run ignores it.
437
487
  - world: a verifier whose `initial_score` is 1.0 in `validate` — or whose `dry-run` reward is 1.0 — passes
438
488
  before the agent acts; write it against the state the task must produce. `verifiers/<name>.sql` must
439
489
  exist before `schema task add`; `dry-run` exits 1 naming a verifier whose SQL cannot run.
440
490
  - `run` needs a model per run; `benchmark.json` `models` fills in when `--model` is absent. A run
441
- that reads `settled: false` is still running, not failed.
491
+ that reads `settled: false` is still running, not failed; `bench run <id> --cancel` stops it.
442
492
  - `--kind image` is refused: the runner has no hub engine for a bare image (its generic engine runs
443
493
  registry entries only). Package the benchmark as a harbor task.
444
494
 
445
495
  ## Related
446
496
 
447
497
  - `worlds-getting-started`: the world itself — contract, data, sessions, stored tasks, links.
448
- - `world-data-ingestion`: rows in, calls out.
449
- - `connector-schema-authoring`: the contract, fidelity shapes, conventions.
450
- - Consumer pages: `gateway_documentation/pages/cli/benchmarks.mdx`, `pages/worlds/benchmarks.mdx`,
451
- `pages/evaluation/benchmarks-harbor.mdx`.
498
+ - `world-data-ingestion`: rows in, calls out. `connector-schema-authoring`: contract, fidelity shapes.
499
+ - Consumer pages: `gateway_documentation/pages/cli/benchmarks.mdx`, `pages/worlds/benchmarks.mdx`, `pages/evaluation/benchmarks-harbor.mdx`.
452
500
  - `documentation/features/benchmark-containers.md`: the container substrate, kinds, the runner.
@@ -55,6 +55,7 @@ registration, a platform builder, live ingestion, tools, or a registry framework
55
55
  | Command or path | Status |
56
56
  | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
57
57
  | `gateway worlds schema init PATH --connector <slug> [--name NAME] [--with-example-tool]` | exists (#219); `--connector custom` is the blank starting point for an arbitrary connector: its `$id` is derived from `--name` (`urn:gateway:connector:<name>`), it ships the `records` entity and a starter verifier but no tools unless `--with-example-tool` asks for the `list_records`/`get_record`/`create_record` example, and it writes no `data/records.json` (Brex retrospective, 2026-09-14) |
58
+ | `gateway worlds schema init PATH --from-openapi SOURCE [--plan-only] [--include GLOB\|TAG]… [--exclude GLOB\|TAG]…` | exists (2026-09-22); imports a Swagger 2.0 / OpenAPI 3.x document (JSON or YAML; a local path, `drive:<path>` read platform-side, or an http(s) URL) into `schema/world.json` + `connector.toml` + `schema/provenance.json`: keyed schemas the selected operations return become entities, every path+method a route (passthrough when the response is not an entity), the plan is printed first and `--plan-only` writes nothing; refusals (no `paths`, unknown `$ref`, cyclic entity root, undeclared path parameter) carry a JSON pointer. Rules: `worlds-getting-started` → `references/openapi-import.md` |
58
59
  | `gateway worlds connector validate\|capture\|ingest PATH` | exists; a `connector.toml` in the world describes base URL, auth scheme, pacing, redaction and operations; `capture` records paced, redacted pages (errors included) under `captures/` with the secret never written; `ingest` seeds `data/initial.json` through the contract in a scratch copy first; `ingest --dry-run [--report refusals.jsonl]` lists every violation of every capture (the `data import --report` records plus `capture`), exit 2, nothing written |
59
60
  | `gateway worlds schema reconcile PATH --from captures [--write]` | exists (runtime, offline CLI and `gateway worlds schema reconcile`); the contract against the captured rows: relaxes `required` for fields absent in captures, adds undocumented keys as optional properties typed from the values, drops formats the captured values fail; a report of every change, `schema/world.json` rewritten and recompiled only with `--write` |
60
61
  | `gateway worlds connector conform PATH --openapi vendor.yaml` | exists; serves the world in-process, calls every declared operation with arguments from its own seed, and checks status + body against the vendor's OpenAPI response schemas; non-zero exit on any mismatch. Run it before calling a world 1:1; its FAIL list is the work list (unseeded entities → 404s, creates the contract rejects → 4xx, honest 501s). Reads OpenAPI 3.0 `nullable`, reference cycles and `discriminator` unions; `[operations.api] sample_headers` / `sample_body` shape a request the route demands (`Idempotency-Key`, a body) |
@@ -92,7 +93,7 @@ deliverable can finish with synthetic fixtures and explicitly skipped live valid
92
93
  ### 0. Decide the shape
93
94
 
94
95
  - [ ] **Fidelity shape**: base, tenant-shaped, or mixed. See [CONNECTORS.md](CONNECTORS.md). Tenant-shaped means describe-on-bind is a requirement of the world, and the shared template is only a skeleton.
95
- - [ ] **Model source**: authored by hand (start from `--connector custom`), imported (`openapi`, `odata-csdl`, `discovery`, `wsdl-xsd`, `ldap-subschema`, `describe`, `docs`), or uploaded. Pin the spec URL and version now.
96
+ - [ ] **Model source**: authored by hand (start from `--connector custom`), imported from a Swagger/OpenAPI document (`schema init PATH --from-openapi SOURCE`, provenance pinned for you) or from another source by hand (`odata-csdl`, `discovery`, `wsdl-xsd`, `ldap-subschema`, `describe`, `docs`), or uploaded. Pin the spec URL and version now.
96
97
  - [ ] **Scope of the model**: which entities and fields, decided from what the connector's users need the world to be faithful about. Record the reasoning in `scope.md` or `scope.toml`; coverage of the vendor's full model is reported, not required.
97
98
  - [ ] **Sandbox**: record whether live validation is available and in scope. If it is not, record the reason and mark live checks `skipped`; a schema foundation can still ship as provisional `subset`.
98
99
  - [ ] **Tools and surface** (optional): none, generated defaults, hand-written, user's custom, or mapped to an MCP surface. Decide now so the entity scope covers what those tools touch.
@@ -135,6 +136,26 @@ deliverable can finish with synthetic fixtures and explicitly skipped live valid
135
136
  - [ ] Handlers never read `benchmark_runtime_context` or `benchmark_conversation_events`.
136
137
  - [ ] A user attaching custom tools to this world needs nothing from this checklist beyond the store API; document any conventions the model expects (key formats, enum values).
137
138
 
139
+ ### Store API
140
+
141
+ Tool handlers (`tools/*.py`) reach the world's rows through `from gateway_env import store`. It has
142
+ exactly these methods; `gateway worlds schema check .` refuses any other `store.<name>` at file:line
143
+ with this list, and a hosted call into such a handler answers 400 `{error: "handler", detail, handler}`
144
+ (an `AttributeError`/`TypeError`/`NameError` is the author's bug; anything else a handler raises is a
145
+ 500 `{error: "engine", detail, handler}`; never a bare exception string).
146
+
147
+ - `get(entity: str, key: dict) -> dict | None` — one row by primary key, or `None`.
148
+ - `list(entity: str) -> list[dict]` — every row, ordered by key.
149
+ - `list_where(entity: str, where: dict[str, list], limit: int | None = None, offset: int = 0) -> tuple[list[dict], int]` — a page of rows whose `where` fields hold one of the listed values, plus the total.
150
+ - `insert(entity: str, value: dict, *, source: dict | None = None) -> dict` — validate against the entity's schema and add; the key must be new.
151
+ - `update(entity: str, key: dict, patch: dict, *, source: dict | None = None) -> dict` — merge `patch` into the row at `key`, validate, return the row.
152
+ - `delete(entity: str, key: dict, *, source: dict | None = None) -> None` — remove the row at `key`; a missing row is a contract error.
153
+ - `apply_batch(batch: dict) -> dict` — one ingest batch `{id, target, mapping, coverage, mutations}`, atomically; an exact retry is a no-op.
154
+ - `lineage(entity: str, key: dict) -> list[dict]` — the batches that wrote the row at `key`, oldest first.
155
+ - `counts() -> dict[str, int]` — rows per entity, `{entity: count}`.
156
+
157
+ No `upsert`: `get`, then `insert` or `update`. Read-only SQL: `gateway_env.db.one/all`; raw SQL writes are refused. The scaffolded `tools/handlers.py` docstring carries the same table.
158
+
138
159
  ### 6. Surface binding (only if the world declares it mirrors an MCP surface)
139
160
 
140
161
  - [ ] Export a real connection's filtered `tools/list` through the MCP proxy under a grant, or inspect its `AgentSecurityResource` tool rows. Native Slack source capture has no live MCP catalog; its schema template tools are separate offline behavior.
@@ -142,6 +163,11 @@ deliverable can finish with synthetic fixtures and explicitly skipped live valid
142
163
  - [ ] Compare read-only tools manually against a live connection only when such comparison is authorized and in scope.
143
164
  - [ ] Native manifest changes bump `TOOL_VERSIONS`.
144
165
 
166
+ ### 6b. Reader plan (only if the template fills from an active Merge connection)
167
+
168
+ - [ ] `readers/merge.toml` names `transport = "merge"`, the Merge `provider` slug (`google_drive`, not the template slug) and one `[[pulls]]` per logical tool (`list_files`, unprefixed — `MergeConnection` adds `google_drive__`); arguments, paging (`page = { cursor, next }`) and `max_pages` live here and nowhere else: a caller (CLI, MCP, route) can never pick a tool or pass arguments. `readers/merge.ingest.toml` maps each tool's records to entities with the `data ingest --map` grammar.
169
+ - [ ] Prove it against the fake Agent Handler (`scripts/worlds-e2e/fake-merge-agent-handler.mjs`, dataset per fixture) through `gateway worlds create <slug> --from-connection <connector>` and `gateway worlds data pull <slug> --connection <id> --mode append`; every tool the plan names must land rows (`unmatched` empty). A live pull needs an owner-authorized active connection and is recorded as such, not implied by the fake.
170
+
145
171
  ### 7. Ship the template; register platform integration only when in scope
146
172
 
147
173
  - [ ] SDK-shipped: commit authored files under `gatewaysdk/sdk/src/gatewaysdk/environment/schema/connectors/<slug>/`; `list_templates()` and `test_schema_scaffold.py` cover the slug and preserve its authored dependencies/provenance during initialization. Schema-only templates declare an empty tools map and have no operation handlers.
@@ -167,7 +193,7 @@ When adding a bridge command to `schema/platform.py`, mirror it as a typed wrapp
167
193
  `worldSchemaRuntime.ts` (`compileWorldFiles` / `applyWorldBatch` are the pattern: file map
168
194
  in, `{entries, report, ...}` out, errors as `SchemaBridgeError`), cover it in
169
195
  `tests/environment/test_schema_platform.py`, and let the agent tool call the wrapper, never
170
- the subprocess. The bridge needs Python ≥ 3.11 (`tomllib`); set `GATEWAY_SCHEMA_PYTHON`
196
+ the subprocess. The bridge needs Python ≥ 3.12 (`tomllib`, and the store's sqlite `setconfig`); set `GATEWAY_SCHEMA_PYTHON`
171
197
  when the default interpreter is older.
172
198
 
173
199
  Use `worldSchemaRuntime.ts` for platform validation and `schema.platform` for
@@ -243,12 +269,19 @@ These source checks do not imply live-provider validation or executed fidelity s
243
269
 
244
270
  A world can describe the vendor it mirrors in `connector.toml` at its root: base URL,
245
271
  auth scheme (`header`, `bearer`, `basic`, `query`, `oauth2`, `none`) with the secret named by an
246
- environment variable, capture pacing, a redaction policy (`hash` or `drop` fields at any
272
+ environment variable (for `basic`, the credential is the Basic *user* and the password is ignored:
273
+ the host and `worlds serve` present `Basic b64("<key>:")`), capture pacing, a redaction policy (`hash` or `drop` fields at any
247
274
  depth), and the operations the agent actually calls (method, path with `{args}`, query
248
275
  params, and the entity plus results path their rows land in). The same file is meant to
249
276
  drive the platform's custom source later, so describe once.
250
277
 
251
- - `gateway worlds connector validate PATH` parses the spec and lists operations.
278
+ - `gateway worlds connector validate PATH` parses the spec, binds every operation's `entity`
279
+ (and each `explode` target) to the entities `schema/world.json` declares, and lists operations.
280
+ A route onto an undeclared entity is refused with `operation X lands on entity Y, which
281
+ schema/world.json does not declare (declared: …)` at `/operations/<i>/entity`; `schema
282
+ compile`, `schema check`, `worlds describe` and session open apply the same rule and write nothing
283
+ (it used to pass every gate, show under `served`, and 500 on the first call). A version stored
284
+ before this rule with such a route refuses to open instead; `worlds describe` names the fix.
252
285
  - `gateway worlds connector capture PATH --op NAME --arg k=v --param k=v` calls the live
253
286
  connection, follows the spec's cursor up to `max_pages`, and writes every page under
254
287
  `captures/<op>/` plus `captures/manifest.json` (sha256 per file). The recorded request
@@ -329,6 +362,23 @@ no `--host`/`--remote`/`workspace:<slug>`, platform env stripped, pinned to its
329
362
  so publishing always goes through the tools above. To parallelise several connectors, `spawn_subagent` with a brief
330
363
  naming only the tools that child needs (`allowSubagents` on the session).
331
364
 
365
+ ### From the CLI sandbox (no checkout, no Python)
366
+
367
+ `gateway worlds sandbox open <slug>@main` puts the same CLI in front of the world's source tree
368
+ server-side: `sandbox exec <id> 'gateway worlds schema entity add . <entity> --fields … -k id'`,
369
+ `'gateway worlds schema verifier add …'`, `'gateway worlds schema task add …'`, `'gateway worlds
370
+ schema compile . && gateway worlds schema check .'`, `'gateway worlds test .'` and `'gateway worlds
371
+ connector validate|conform .'` run on the uncommitted copy (`sqlite3` is on PATH too); `sandbox check`
372
+ runs the push gate and `sandbox commit` lands the version. `open` reports the staged CLI under `cli`
373
+ (`null` plus `warnings` when the image cannot host it — then only plain shell runs).
374
+
375
+ With an API key and no world yet, start the world itself there:
376
+ `gateway worlds sandbox open --new <slug> --from custom` (MCP `open_world_sandbox({ new, from })`)
377
+ opens on a world that does not exist yet, its tree the exact `schema init` scaffold (or
378
+ `--from <template>` for a shipped one); the same `sandbox exec` verbs run on that tree with the
379
+ staged CLI, `sandbox check` runs the push gate, and the first `sandbox commit -m …` creates the
380
+ world (first READY version on `main`). A taken slug is a 409 `world_exists`, never a rename.
381
+
332
382
  ### Serving the vendor's API from the world
333
383
 
334
384
  A provider's client speaks the vendor's REST paths, not tool calls, so the world answers
@@ -78,8 +78,31 @@ person so they stay consistent; `role: "public"` is never replaced. Every varian
78
78
  tracked (bare surname, handle spellings, `(at)` emails, phone formats, percent-encoding, a
79
79
  world's `sha256:` stored form), and in any case: `Alexr` or `ALEXR` for a seeded `alexr` is swapped
80
80
  where it stands alone, and the stand-in keeps the case it arrived in. The door swaps and never
81
- blocks. What slips through is counted (`leak_through_inbound`), and a real value in the outgoing
82
- report fails the run (`leaks_outbound`).
81
+ blocks. A leak is about a **case subject**, an entity the task seeded: what slips through is counted
82
+ (`leak_through_inbound`), and a subject's real value in the outgoing report fails the run
83
+ (`leaks_outbound`). Someone who only turned up in a result and got a stand-in on sight is a bystander:
84
+ swapped the same way, counted apart (`bystander_leak_through_inbound`, `bystander_leaks_outbound`),
85
+ never a failure.
86
+
87
+ ## Choose the stand-in yourself
88
+
89
+ When a case must look a particular way to the model, or the sources spell a real value in
90
+ ways the door cannot guess, declare it on the seed instead of relying on the pools:
91
+
92
+ ```json
93
+ {"type": "username", "value": "<real handle>", "alias": "kmarsh7", "variants": ["<the forum's misspelling>"]}
94
+ {"type": "person_name", "value": "<real name>", "alias": "Kim Marsh"}
95
+ ```
96
+
97
+ `alias` is the stand-in the model sees; every form the kind has (name order, `(at)` emails, phone
98
+ formats, case) is paired from it. `variants` are other spellings of the real value; each is paired
99
+ with the same stand-in, so a misspelling in a forum post or a phone written without its country
100
+ digit is swapped like the value itself. Leave both out and the door generates a persona.
101
+
102
+ When the case must not show a generated persona at all (a model that recognises the reserved IP
103
+ range or the made-up domains stops trusting the data), set `declared_only = true` under `[alias]`:
104
+ every private seed then needs its own `alias`, a seed without one is refused before the run, and
105
+ `mint_inbound` stays off.
83
106
 
84
107
  ## Identities that arrive, not seeded
85
108
 
@@ -175,6 +198,8 @@ gateway hooks audit runs/<run id>/ --json # the
175
198
  ```
176
199
 
177
200
  `fields.json` is `{"*": {"full_name": "person_name", "email": "email"}}`, or keyed by tool name.
201
+ A directory with neither `raw.jsonl` nor `shown.jsonl` (a `task run --out-dir`, say) never ran under
202
+ hooks: the audit says so and exits 2 rather than reading the run's own `run.json` as hooks state.
178
203
 
179
204
  It knows a value is real from three places, strongest first: your legend; every value of a field
180
205
  whose kind you declared (a name in a vendor row that reappears in a web snippet is caught for
@@ -15,6 +15,8 @@ Companion to [SKILL.md](SKILL.md). The contract is fixed in
15
15
  8. [MCP payloads](#mcp-payloads)
16
16
  9. [Chunks, batches and refusal records](#chunks-batches-and-refusal-records)
17
17
  10. [Extract adapters](#extract-adapters)
18
+ 11. [CSV and TSV cells](#csv-and-tsv-cells)
19
+ 12. [Data snapshots](#data-snapshots)
18
20
 
19
21
  ## Shapes
20
22
 
@@ -46,6 +48,16 @@ gateway worlds data import ./world users.json --entity users
46
48
  gateway worlds data import ./world users.jsonl --entity users
47
49
  ```
48
50
 
51
+ **CSV/TSV with `--entity NAME`.** A `.csv`/`.tsv` (or `.gz`) whose header row names the fields;
52
+ every cell is cast by the entity field's schema type (integer, number, boolean, array/object as
53
+ JSON, null for an empty nullable cell) — [CSV and TSV cells](#csv-and-tsv-cells). `--delimiter`
54
+ picks the separator. Not accepted from `drive:` (import it from the machine that holds it, or
55
+ `worlds sandbox import` it from a sandbox).
56
+
57
+ ```bash
58
+ gateway worlds data import ./world tickets.csv --entity hd_tickets
59
+ ```
60
+
49
61
  A file that mixes shapes is refused. Several files on one command are applied as one batch:
50
62
  all or nothing (`--per-chunk` on a platform import relaxes that to one chunk at a time). Files
51
63
  are streamed, so size is not a concern: pass `-` for stdin, `.gz` files are gunzipped.
@@ -241,3 +253,53 @@ rows add missing fields; `upsert`: last wins); the world's `[redact]` policy app
241
253
  by element, `round` rounded). Output is tagged JSONL; stderr gets
242
254
  `{"read": N, "rows": M, "unknownFields": {name: n}, "redacted": {"dropped": n, "hashed": n}}`.
243
255
  The fixtures under `gatewaysdk/fixtures/world-data/extract/` are the worked examples.
256
+
257
+ ## CSV and TSV cells
258
+
259
+ A `.csv`/`.tsv` (optionally `.gz`) is read as a stream — memory is bounded at any size — and
260
+ split by `--delimiter` (one character, `\t` for a tab; default `,`, tab for a `.tsv`). The header
261
+ row names the fields; `--entity` says which entity they fill. Every cell is cast by that field's
262
+ schema type, the same rule in the TypeScript CLI (`worldData/cast.ts`) and the Python package
263
+ (`schema/cast.py`), pinned by `gatewaysdk/sdk/tests/conformance/csv_cast.json`:
264
+
265
+ | Field type | Cell | Value |
266
+ | --- | --- | --- |
267
+ | `integer` | `42`, `-7` | the number; `4.5` or `abc` is refused |
268
+ | `number` | `12.5`, `1e3`, `3` | the number (always a float) |
269
+ | `boolean` | `true`/`false`/`1`/`0`, any case | the boolean; `yes` is refused |
270
+ | `array` / `object` | JSON text of that shape (quote it in the CSV: `"[""a"",""b""]"`) | the parsed value |
271
+ | `string` | anything | the text as given |
272
+ | `integer \| string` (union) | each member tried in order, `string` last | the first that fits |
273
+ | any, empty cell | `` | null when the type admits null; left out when the field is optional; `""` for a required string; refused otherwise |
274
+
275
+ A cell that fits none is a row-level refusal naming the field and the cell —
276
+ `/taxi_trip/3: fare = "n/a" is not a number` — with the usual pointer; nothing is written. A
277
+ column the schema does not name stays text and the gate refuses it as an unknown field. A CSV
278
+ on the drive (`drive:<path>`) is still refused: import it from the machine that holds it, or
279
+ through `worlds sandbox import` when it sits in a sandbox.
280
+
281
+ ```bash
282
+ gateway worlds data import ./world trips.csv --entity taxi_trip
283
+ gateway worlds data import ./world trips.tsv --entity taxi_trip # tab by name
284
+ gateway worlds data import acme export.txt.csv --entity rows --delimiter ';'
285
+ ```
286
+
287
+ ## Data snapshots
288
+
289
+ Why a served world starts in under a second.
290
+
291
+ A schema world's validated database is kept, not rebuilt. `gateway worlds schema compile`
292
+ builds `db/state.sqlite` (the default snapshot: `data/initial.json` applied through the
293
+ contract) and `scenarios/<name>/state.sqlite` (the default with the scenario's overlays),
294
+ each with a `state.json` sidecar `{contractHash, batchDigest, entityCounts, builtAt,
295
+ runtime, fileSha256}`; the same key is never rebuilt, `schema check` verifies the sidecars,
296
+ and a data batch the contract refuses leaves no snapshot behind. `gateway worlds serve` and
297
+ the api bundle open the snapshot (copy + `SQLiteStore.open`, no row replay) and reset by
298
+ copying it again; without one the JSON is replayed as before. A hosted pod gets
299
+ `WORLD_SNAPSHOT_URL` (presigned GET, `.gz` or `Content-Encoding: gzip` is gunzipped),
300
+ `WORLD_SNAPSHOT_SHA256` and `WORLD_SNAPSHOT_DIGEST`; the server downloads, verifies, writes
301
+ the sidecar and only then serves; a mismatch exits 2 naming the digest. `/__world/state`
302
+ reports `"snapshot": {batchDigest, contractHash}` (or null) and `/__world/seed` takes the
303
+ overlay form `{"base": digest, "rows": {...}, "mode": append|replace}` beside the legacy
304
+ whole-state body. Snapshot files are build outputs: gitignore `db/state.sqlite`,
305
+ `db/state.json`, `scenarios/*/state.*`.