@withgateway/sdk 0.2.6 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -8
- package/dist/cli.js +15468 -11263
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +15233 -11028
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +291 -40
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +284 -37
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +203 -47
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +198 -42
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-BaKu5ZSU.d.mts → index-C3UzD0G3.d.mts} +75 -1
- package/dist/{index-BaKu5ZSU.d.ts → index-C3UzD0G3.d.ts} +75 -1
- package/dist/{index-CLkMr6VG.d.ts → index-D-_4Ibcn.d.ts} +36 -4
- package/dist/{index-CdbmO4aX.d.mts → index-FfK4Nump.d.mts} +36 -4
- package/dist/index.d.mts +3 -3
- package/dist/index.d.ts +3 -3
- package/dist/index.js +2149 -615
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2053 -566
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js +268 -31
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +262 -28
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.js +19 -11
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +19 -11
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +886 -117
- package/dist/worlds/index.d.ts +886 -117
- package/dist/worlds/index.js +1770 -452
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1752 -434
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +14 -10
- package/runtime/gatewaysdk/environment/_toml.py +3 -10
- package/runtime/gatewaysdk/environment/_world.py +95 -28
- package/runtime/gatewaysdk/environment/runtime.py +10 -6
- package/runtime/gatewaysdk/environment/schema/__main__.py +35 -0
- package/runtime/gatewaysdk/environment/schema/api.py +19 -23
- package/runtime/gatewaysdk/environment/schema/batch.py +68 -14
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +4 -2
- package/runtime/gatewaysdk/environment/schema/cast.py +264 -0
- package/runtime/gatewaysdk/environment/schema/compiler.py +35 -1
- package/runtime/gatewaysdk/environment/schema/conform.py +28 -10
- package/runtime/gatewaysdk/environment/schema/connector.py +59 -14
- package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +38 -1
- package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.toml +3 -1
- package/runtime/gatewaysdk/environment/schema/credentials.py +20 -0
- package/runtime/gatewaysdk/environment/schema/csv_rows.py +117 -0
- package/runtime/gatewaysdk/environment/schema/cycles.py +121 -0
- package/runtime/gatewaysdk/environment/schema/data.py +21 -5
- package/runtime/gatewaysdk/environment/schema/grading.py +95 -13
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +44 -0
- package/runtime/gatewaysdk/environment/schema/handler_scan.py +175 -0
- package/runtime/gatewaysdk/environment/schema/host.py +68 -115
- package/runtime/gatewaysdk/environment/schema/host_requests.py +78 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +12 -0
- package/runtime/gatewaysdk/environment/schema/ingest.py +5 -18
- package/runtime/gatewaysdk/environment/schema/local.py +7 -10
- package/runtime/gatewaysdk/environment/schema/openapi_document.py +322 -0
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +382 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +441 -0
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +629 -0
- package/runtime/gatewaysdk/environment/schema/openapi_source.py +210 -0
- package/runtime/gatewaysdk/environment/schema/platform.py +110 -299
- package/runtime/gatewaysdk/environment/schema/platform_import.py +182 -0
- package/runtime/gatewaysdk/environment/schema/platform_query.py +95 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +176 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +56 -7
- package/runtime/gatewaysdk/environment/schema/scaffold.py +64 -5
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +70 -22
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +54 -4
- package/runtime/gatewaysdk/environment/schema/skills/hooks-getting-started/SKILL.md +27 -2
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +62 -0
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +91 -54
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +146 -146
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +11 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +30 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/new-world-sandbox.md +43 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +90 -0
- package/runtime/gatewaysdk/environment/schema/store.py +15 -6
- package/runtime/gatewaysdk/environment/schema/store_contract.py +119 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +121 -32
- package/runtime/gatewaysdk/environment/schema/tools_world.py +32 -10
- package/runtime/gatewaysdk/environment/schema/tree_import.py +228 -0
- package/runtime/gatewaysdk/environment/schema/ui.py +18 -2
- package/runtime/gatewaysdk/environment/schema/validation.py +35 -0
- package/runtime/gatewaysdk/environment/schema/verbs.py +8 -2
- package/runtime/gatewaysdk/environment/world.py +42 -3
- package/runtime/gatewaysdk/environment/world_session_contract.py +57 -0
- package/skills/benchmarks-getting-started/SKILL.md +70 -22
- package/skills/connector-schema-authoring/SKILL.md +54 -4
- package/skills/connector-schema-authoring/connector-worlds.md +21 -7
- package/skills/connector-schema-authoring/world-schema.md +144 -83
- package/skills/hooks-getting-started/SKILL.md +27 -2
- package/skills/world-data-ingestion/ROWS.md +62 -0
- package/skills/world-data-ingestion/SKILL.md +91 -54
- package/skills/worlds-getting-started/SKILL.md +146 -146
- package/skills/worlds-getting-started/references/cli-mcp-map.md +11 -7
- package/skills/worlds-getting-started/references/gotchas.md +30 -0
- package/skills/worlds-getting-started/references/new-world-sandbox.md +43 -0
- package/skills/worlds-getting-started/references/openapi-import.md +90 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: benchmarks-getting-started
|
|
3
|
-
description: Build, validate, dry-run, push, run and read a benchmark on the platform, for an agent (Claude Code, Cursor, Codex). A benchmark is either a world with tasks (tools + data, a schema when the vendor has one; a prompt and the SQL verifiers that grade the world's end state) or a container benchmark (a harbor task tree graded by its own tests/test.sh in a container, or a verifiers package graded by Python reward functions), run k times per task per model. Use when building a benchmark, an eval suite, a coding benchmark, a task set or a leaderboard for an agent; when you have a harbor task tree, task.toml files, a verifiers package, a Taskset, or tests that grade an agent; when choosing between a world benchmark and a container (benchmark runner) benchmark; when you need a dry run before pushing; when running k rollouts per task per model or reading per-task results. Every CLI step names its MCP twin (create_world, env_write_file, env_commit, create_environment_version, run_evaluation, get_task_performance, list_rollouts, get_rollout_samples) and where the result shows in the UI.
|
|
3
|
+
description: Build, validate, dry-run, push, run and read a benchmark on the platform, for an agent (Claude Code, Cursor, Codex). A benchmark is either a world with tasks (tools + data, a schema when the vendor has one; a prompt and the SQL verifiers that grade the world's end state) or a container benchmark (a harbor task tree graded by its own tests/test.sh in a container, or a verifiers package graded by Python reward functions), run k times per task per model. Use when building a benchmark, an eval suite, a coding benchmark, a task set or a leaderboard for an agent; when you have a harbor task tree, task.toml files, a verifiers package, a Taskset, or tests that grade an agent; when choosing between a world benchmark and a container (benchmark runner) benchmark; when you need a dry run before pushing (locally in Docker, or hosted on the benchmark runner with `--hosted` when the machine has no Docker); when running k rollouts per task per model or reading per-task results. Every CLI step names its MCP twin (create_world, env_write_file, env_commit, create_environment_version, run_evaluation, get_task_performance, list_rollouts, get_rollout_samples) and where the result shows in the UI.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# Benchmarks: getting started
|
|
@@ -41,20 +41,23 @@ Segmentation, so one substrate never crashes into the other's infrastructure:
|
|
|
41
41
|
| Storage | a world version (`POST /api/public/worlds`; later versions through the same hub as containers) | a benchmark-container version (`bench push`: resolve, versions, finalize) |
|
|
42
42
|
| Engine | World Host (a hosted session per rollout; the mock-tools runner for the legacy bundle) | benchmark runner on AWS Batch: `harbor run` per task, or verifiers v1 `run_eval` in-process |
|
|
43
43
|
| Results rows | evaluations, samples, `GET /environments/{id}/performance` | the same rows and endpoint |
|
|
44
|
+
| Dry run | local only: the task's session in the bundled runtime (`benchmarks dry-run <dir>`) | local (Docker, `benchmarks dry-run <dir>`) **or hosted** (`--hosted`: the runner, the oracle, the real image rule, `POST /environments/{id}/dry-run`, harbor only; a verifiers package or a world is refused with a 400 naming the local command); a hosted dry run is `kind: dry-run`, never evidence |
|
|
44
45
|
| Refused when a tree crosses over | `worlds create` answers `world_kind_retired` for a harbor tree or a verifiers package, before any write; the hub's versions door answers `benchmark_substrate_mismatch` (409) for a container tree pushed under a world's slug | the hub's versions door answers `benchmark_substrate_mismatch` for a world tree pushed under a container's slug; the runner's harbor engine refuses a schema bundle (`require_legacy_bundle`) |
|
|
45
46
|
| Read before writing | `benchmarks push` reads the slug's substrate (`GET /environments/{id}` → `substrate`) and refuses a cross-substrate push with exit 2 and no request; `--kind` that disagrees with the tree is refused before that | the same pre-check; a container benchmark that IS a container benchmark opens on the runner — `session open` / `task create` do not refuse it (only the legacy tools bundle is retired on serve) |
|
|
46
47
|
|
|
47
48
|
## 0. Authenticate
|
|
48
49
|
|
|
49
50
|
```bash
|
|
50
|
-
npm install -g @withgateway/sdk # the gateway CLI (TypeScript; python3 >= 3.
|
|
51
|
+
npm install -g @withgateway/sdk # the gateway CLI (TypeScript; python3 >= 3.12 runs the local runtime)
|
|
51
52
|
gateway auth login --host https://<your-gateway> --public-key pk-lf-... --api-key sk-lf-...
|
|
52
53
|
```
|
|
53
54
|
|
|
54
|
-
`init`, `validate`, `dry-run` and the local world commands run without credentials; `push`, `run
|
|
55
|
-
`results` need them. `dry-run` on a harbor tree executes in Docker when Docker
|
|
56
|
-
the exact plan when it is not;
|
|
57
|
-
(
|
|
55
|
+
`init`, `validate`, `dry-run` and the local world commands run without credentials; `push`, `run`,
|
|
56
|
+
`results` and `dry-run --hosted` need them. `dry-run` on a harbor tree executes in Docker when Docker
|
|
57
|
+
is reachable and prints the exact plan when it is not; `dry-run --hosted` runs the same one task on
|
|
58
|
+
the benchmark runner (harbor's oracle, no model), so a machine with nothing but the npm CLI and project
|
|
59
|
+
keys can author, dry-run, push and run a container benchmark; on a world it opens the task's session
|
|
60
|
+
in the bundled Python runtime (python3 >= 3.12) and prints the plan when there is none.
|
|
58
61
|
|
|
59
62
|
## A. A harbor (container) benchmark: init → validate → dry-run → push → run → results
|
|
60
63
|
|
|
@@ -177,6 +180,50 @@ docker-compose file requires EXA_API_KEY … export EXA_API_KEY here for the dry
|
|
|
177
180
|
it from … gateway bench secrets set --env code-bench --key EXA_API_KEY`, exit 0, nothing spawned.
|
|
178
181
|
Values never appear in the printed argv.
|
|
179
182
|
|
|
183
|
+
**Hosted (`--hosted`): the runner, the real image rule, no Docker here.** A Dockerfile-first local
|
|
184
|
+
dry-run passes a task whose `task.toml` names `docker_image`, because harbor never builds that
|
|
185
|
+
Dockerfile on the runner (A1); the hosted dry-run catches it in about two minutes, not in a
|
|
186
|
+
15-minute model run.
|
|
187
|
+
|
|
188
|
+
```bash
|
|
189
|
+
gateway benchmarks dry-run code-bench --task hello --hosted # a pushed slug (or slug@ref)
|
|
190
|
+
gateway benchmarks dry-run ./code-bench --task hello --hosted # a dir that tracks a slug (.gateway/.env-metadata.json)
|
|
191
|
+
gateway benchmarks dry-run ./code-bench --task hello --hosted --push # the dir differs from the pushed version: push first, then dry-run that version
|
|
192
|
+
gateway environment dry-run code-bench --task hello # the same run under the environment group (the Python CLI's spelling)
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
ONE Batch job on the benchmark runner (`POST /api/public/environments/{id}/dry-run` with
|
|
196
|
+
`{ task, versionId? }`): harbor kind only, `--agent oracle` (the task's `solution/solve.sh` in place
|
|
197
|
+
of the agent), the ONE named task (the runner prunes the working copy to it, the same
|
|
198
|
+
`GATEWAY_TASK_NAMES` filter sharded runs use), one rollout, one example — capped on the server, a body
|
|
199
|
+
that asks for more is refused (400) — the image chosen by harbor's rule and the `bench secrets` compose
|
|
200
|
+
requirements enforced exactly like a real run (missing → refused before dispatch with the
|
|
201
|
+
`gateway bench secrets set …` command). Expected closing lines:
|
|
202
|
+
|
|
203
|
+
```
|
|
204
|
+
image: build environment/Dockerfile # or `image: ghcr.io/acme/tool:1.2 (prebuilt: docker_image)`
|
|
205
|
+
ran: harbor run -p …/tasks -a oracle -e docker -o …/jobs -k 1 -n 1 -y -q -l 1 # no -m: the oracle calls no model; the working copy holds only `hello`
|
|
206
|
+
reward: 1 {"reward":1,"file_exists":1,"content_ok":1} · exit 0
|
|
207
|
+
where: hosted (code-bench 0.1.0 · 3f9a2c1d · task hello · agent oracle, no model · not evidence)
|
|
208
|
+
run: <run_id> · COMPLETED
|
|
209
|
+
page: https://<your-gateway>/project/<id>/world/runs/<evaluation_id>
|
|
210
|
+
status: gateway environment run-status <run_id>
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
Exit 0 only when the oracle scored 1.0; lower → exit 2 naming it; nothing graded → exit 2
|
|
214
|
+
`reward: null (ungraded)`; an image pull/build failure → exit 2 with the runner's error tail (bounded,
|
|
215
|
+
no host paths, no secret values). `--wait` (default, `--wait-timeout 1200`) and `--no-wait` (return
|
|
216
|
+
after dispatch with the run id and the `run-status` command) mirror `benchmarks run`; `--json` prints
|
|
217
|
+
dispatch + verdict. A dir with uncommitted changes vs the pushed version is refused (exit 2, nothing
|
|
218
|
+
dispatched) unless `--push`, which goes through the ONE `benchmarks push` path. A world or a verifiers
|
|
219
|
+
package is refused with a 400 that names the local alternative (`benchmarks dry-run <dir>`).
|
|
220
|
+
|
|
221
|
+
**Not evidence.** The run is `kind: dry-run` (`evidence: false`, `agent: oracle`): the run page and
|
|
222
|
+
`environment run-status` show `dryRun`, the rollouts list labels the evaluation `dryRun: true`,
|
|
223
|
+
`benchmarks results` / `get_task_performance` exclude it, and the Batch job is named `bench-dryrun-…`
|
|
224
|
+
with `GATEWAY_DRY_RUN=oracle` in its environment, so an operator can tell it apart in the queue. MCP twin:
|
|
225
|
+
`dry_run_environment` (`containerId`, `task`, `versionId?`).
|
|
226
|
+
|
|
180
227
|
### A4. Push
|
|
181
228
|
|
|
182
229
|
```bash
|
|
@@ -211,15 +258,16 @@ MCP: `create_environment_version` (or `env_write_file` + `env_commit`).
|
|
|
211
258
|
```bash
|
|
212
259
|
gateway benchmarks run code-bench --model openrouter/openai/gpt-5-mini --model anthropic/claude-sonnet-4-6 -k 3
|
|
213
260
|
gateway benchmarks run code-bench -k 5 --no-wait # models from benchmark.json; return after dispatch
|
|
214
|
-
gateway environment run-status <run_id> #
|
|
261
|
+
gateway environment run-status <run_id> # verdict; while running: `0/15 rollouts, 95s since dispatch`
|
|
262
|
+
gateway bench run <run_id> --cancel # CANCELLED here, runner job terminated on Batch (counts what Batch accepted)
|
|
215
263
|
```
|
|
216
264
|
|
|
217
265
|
One runner job per model (`POST /api/public/environments/{id}/run`): it downloads the version, runs
|
|
218
266
|
`harbor run -p tasks -a mini-swe-agent -m <model> -e docker -k <rollouts>` — the agent in each task's
|
|
219
267
|
container, then `tests/test.sh` — and harvests every trial into an evaluation. Models are
|
|
220
268
|
litellm-style, provider-prefixed (`openrouter/vendor/model`, `anthropic/…`); the project's LLM
|
|
221
|
-
connection supplies the key. `--num-examples N` caps the tasks
|
|
222
|
-
|
|
269
|
+
connection supplies the key. `--num-examples N` caps the tasks; `-k` is 1–64 (the platform refuses
|
|
270
|
+
more before any row or job exists, whatever the client sent). The first run builds images: ~15 min.
|
|
223
271
|
|
|
224
272
|
MCP: `run_evaluation({ models, versionId?, numExamples?, rolloutsPerExample? })`; `list_rollouts`.
|
|
225
273
|
|
|
@@ -227,7 +275,7 @@ MCP: `run_evaluation({ models, versionId?, numExamples?, rolloutsPerExample? })`
|
|
|
227
275
|
|
|
228
276
|
```bash
|
|
229
277
|
gateway benchmarks results code-bench # per task: mean reward, pass rate, samples, no score
|
|
230
|
-
gateway benchmarks results code-bench --runs --json
|
|
278
|
+
gateway benchmarks results code-bench --runs --json # also --trend | --all | --version 1.2.0
|
|
231
279
|
gateway environment samples <container_id> <evaluation_id>
|
|
232
280
|
```
|
|
233
281
|
|
|
@@ -236,12 +284,12 @@ Expected:
|
|
|
236
284
|
```
|
|
237
285
|
task mean reward pass rate samples no score
|
|
238
286
|
hello 1.000 100% 3 0
|
|
239
|
-
|
|
240
|
-
6 samples, 0 without a score
|
|
287
|
+
3 samples, 0 without a score
|
|
241
288
|
```
|
|
242
289
|
|
|
243
|
-
`GET /api/public/environments/{id}/performance`; null scores are their own count.
|
|
244
|
-
|
|
290
|
+
`GET /api/public/environments/{id}/performance`; null scores are their own count; `reward.json` sub-rewards
|
|
291
|
+
are on each sample. MCP: `get_task_performance`, `get_rollout_samples`. Default is the newest version with
|
|
292
|
+
results; `--json` carries the table's selection (`view`, `version`, `tasks`), never every version's rows.
|
|
245
293
|
|
|
246
294
|
| Thing | Where in the UI |
|
|
247
295
|
| --- | --- |
|
|
@@ -361,7 +409,7 @@ reward is for the untouched world; an agent's rollout is what `benchmarks run` g
|
|
|
361
409
|
|
|
362
410
|
A verifier that cannot run — bad SQL, a missing entity — prints `FAIL verifier 'x': …` with SQLite's
|
|
363
411
|
words and exits 1; that is what the dry run is for. A reward of 1.0 here means the verifier passes
|
|
364
|
-
before any agent acts: rewrite it. Without the bundled runtime (no python3 >= 3.
|
|
412
|
+
before any agent acts: rewrite it. Without the bundled runtime (no python3 >= 3.12) the four steps
|
|
365
413
|
print and the run is marked not executed, exit 0.
|
|
366
414
|
|
|
367
415
|
A hand-driven rollout, the way an agent's session goes:
|
|
@@ -419,8 +467,10 @@ Every run records the version it ran against; `results` shows the per-version tr
|
|
|
419
467
|
| `benchmarks init` | none (local); `env_write_file` + `env_commit` on an existing environment |
|
|
420
468
|
| `benchmarks validate` | world: `validate_connector_template({ files })`; container: none (local) |
|
|
421
469
|
| `benchmarks dry-run` | none (local: Docker for harbor, the bundled Python runtime for a world) |
|
|
470
|
+
| `benchmarks dry-run --hosted` / `environment dry-run` | `dry_run_environment({ containerId, task, versionId? })`; `POST /api/public/environments/{id}/dry-run` (harbor only, oracle, one task, not evidence) |
|
|
422
471
|
| `benchmarks push` | world: `create_world`, `POST /api/public/worlds`; container: `create_environment_version` (`bench push`) |
|
|
423
|
-
| `benchmarks run` | `run_evaluation({ models, rolloutsPerExample })`; `POST /api/public/environments/{id}/run` |
|
|
472
|
+
| `benchmarks run` | `run_evaluation({ models, rolloutsPerExample })`; `POST /api/public/environments/{id}/run` (`rolloutsPerExample` 1–64) |
|
|
473
|
+
| `bench run <id> --cancel` / `environment run-status <id> --cancel` | `POST /api/public/environments/runs/{id}/cancel` (secret key; 409 once settled) |
|
|
424
474
|
| `benchmarks results` | `get_task_performance`, `list_rollouts`; `GET /api/public/environments/{id}/performance` |
|
|
425
475
|
| `worlds schema task add` | none (edits gateway-env.toml); stored tasks: `create_world_task` |
|
|
426
476
|
| `gateway skill search <words>` | `guide({ topic })` |
|
|
@@ -433,20 +483,18 @@ Every run records the version it ran against; `results` shows the per-version tr
|
|
|
433
483
|
writes nothing makes every trial ungraded (null), not 0. `mkdir -p /logs/verifier` first.
|
|
434
484
|
- harbor: a `gateway-env.toml` at the root makes the runner read the tree as a world (`mock-tools`);
|
|
435
485
|
`validate` names it.
|
|
436
|
-
- harbor: `solution/solve.sh` is for `dry-run`
|
|
486
|
+
- harbor: `solution/solve.sh` is for `dry-run` (local, or `--hosted`: the oracle on the runner); a model run ignores it.
|
|
437
487
|
- world: a verifier whose `initial_score` is 1.0 in `validate` — or whose `dry-run` reward is 1.0 — passes
|
|
438
488
|
before the agent acts; write it against the state the task must produce. `verifiers/<name>.sql` must
|
|
439
489
|
exist before `schema task add`; `dry-run` exits 1 naming a verifier whose SQL cannot run.
|
|
440
490
|
- `run` needs a model per run; `benchmark.json` `models` fills in when `--model` is absent. A run
|
|
441
|
-
that reads `settled: false` is still running, not failed.
|
|
491
|
+
that reads `settled: false` is still running, not failed; `bench run <id> --cancel` stops it.
|
|
442
492
|
- `--kind image` is refused: the runner has no hub engine for a bare image (its generic engine runs
|
|
443
493
|
registry entries only). Package the benchmark as a harbor task.
|
|
444
494
|
|
|
445
495
|
## Related
|
|
446
496
|
|
|
447
497
|
- `worlds-getting-started`: the world itself — contract, data, sessions, stored tasks, links.
|
|
448
|
-
- `world-data-ingestion`: rows in, calls out.
|
|
449
|
-
-
|
|
450
|
-
- Consumer pages: `gateway_documentation/pages/cli/benchmarks.mdx`, `pages/worlds/benchmarks.mdx`,
|
|
451
|
-
`pages/evaluation/benchmarks-harbor.mdx`.
|
|
498
|
+
- `world-data-ingestion`: rows in, calls out. `connector-schema-authoring`: contract, fidelity shapes.
|
|
499
|
+
- Consumer pages: `gateway_documentation/pages/cli/benchmarks.mdx`, `pages/worlds/benchmarks.mdx`, `pages/evaluation/benchmarks-harbor.mdx`.
|
|
452
500
|
- `documentation/features/benchmark-containers.md`: the container substrate, kinds, the runner.
|
|
@@ -55,6 +55,7 @@ registration, a platform builder, live ingestion, tools, or a registry framework
|
|
|
55
55
|
| Command or path | Status |
|
|
56
56
|
| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
|
57
57
|
| `gateway worlds schema init PATH --connector <slug> [--name NAME] [--with-example-tool]` | exists (#219); `--connector custom` is the blank starting point for an arbitrary connector: its `$id` is derived from `--name` (`urn:gateway:connector:<name>`), it ships the `records` entity and a starter verifier but no tools unless `--with-example-tool` asks for the `list_records`/`get_record`/`create_record` example, and it writes no `data/records.json` (Brex retrospective, 2026-09-14) |
|
|
58
|
+
| `gateway worlds schema init PATH --from-openapi SOURCE [--plan-only] [--include GLOB\|TAG]… [--exclude GLOB\|TAG]…` | exists (2026-09-22); imports a Swagger 2.0 / OpenAPI 3.x document (JSON or YAML; a local path, `drive:<path>` read platform-side, or an http(s) URL) into `schema/world.json` + `connector.toml` + `schema/provenance.json`: keyed schemas the selected operations return become entities, every path+method a route (passthrough when the response is not an entity), the plan is printed first and `--plan-only` writes nothing; refusals (no `paths`, unknown `$ref`, cyclic entity root, undeclared path parameter) carry a JSON pointer. Rules: `worlds-getting-started` → `references/openapi-import.md` |
|
|
58
59
|
| `gateway worlds connector validate\|capture\|ingest PATH` | exists; a `connector.toml` in the world describes base URL, auth scheme, pacing, redaction and operations; `capture` records paced, redacted pages (errors included) under `captures/` with the secret never written; `ingest` seeds `data/initial.json` through the contract in a scratch copy first; `ingest --dry-run [--report refusals.jsonl]` lists every violation of every capture (the `data import --report` records plus `capture`), exit 2, nothing written |
|
|
59
60
|
| `gateway worlds schema reconcile PATH --from captures [--write]` | exists (runtime, offline CLI and `gateway worlds schema reconcile`); the contract against the captured rows: relaxes `required` for fields absent in captures, adds undocumented keys as optional properties typed from the values, drops formats the captured values fail; a report of every change, `schema/world.json` rewritten and recompiled only with `--write` |
|
|
60
61
|
| `gateway worlds connector conform PATH --openapi vendor.yaml` | exists; serves the world in-process, calls every declared operation with arguments from its own seed, and checks status + body against the vendor's OpenAPI response schemas; non-zero exit on any mismatch. Run it before calling a world 1:1; its FAIL list is the work list (unseeded entities → 404s, creates the contract rejects → 4xx, honest 501s). Reads OpenAPI 3.0 `nullable`, reference cycles and `discriminator` unions; `[operations.api] sample_headers` / `sample_body` shape a request the route demands (`Idempotency-Key`, a body) |
|
|
@@ -92,7 +93,7 @@ deliverable can finish with synthetic fixtures and explicitly skipped live valid
|
|
|
92
93
|
### 0. Decide the shape
|
|
93
94
|
|
|
94
95
|
- [ ] **Fidelity shape**: base, tenant-shaped, or mixed. See [CONNECTORS.md](CONNECTORS.md). Tenant-shaped means describe-on-bind is a requirement of the world, and the shared template is only a skeleton.
|
|
95
|
-
- [ ] **Model source**: authored by hand (start from `--connector custom`), imported (`openapi`, `odata-csdl`, `discovery`, `wsdl-xsd`, `ldap-subschema`, `describe`, `docs`), or uploaded. Pin the spec URL and version now.
|
|
96
|
+
- [ ] **Model source**: authored by hand (start from `--connector custom`), imported from a Swagger/OpenAPI document (`schema init PATH --from-openapi SOURCE`, provenance pinned for you) or from another source by hand (`odata-csdl`, `discovery`, `wsdl-xsd`, `ldap-subschema`, `describe`, `docs`), or uploaded. Pin the spec URL and version now.
|
|
96
97
|
- [ ] **Scope of the model**: which entities and fields, decided from what the connector's users need the world to be faithful about. Record the reasoning in `scope.md` or `scope.toml`; coverage of the vendor's full model is reported, not required.
|
|
97
98
|
- [ ] **Sandbox**: record whether live validation is available and in scope. If it is not, record the reason and mark live checks `skipped`; a schema foundation can still ship as provisional `subset`.
|
|
98
99
|
- [ ] **Tools and surface** (optional): none, generated defaults, hand-written, user's custom, or mapped to an MCP surface. Decide now so the entity scope covers what those tools touch.
|
|
@@ -135,6 +136,26 @@ deliverable can finish with synthetic fixtures and explicitly skipped live valid
|
|
|
135
136
|
- [ ] Handlers never read `benchmark_runtime_context` or `benchmark_conversation_events`.
|
|
136
137
|
- [ ] A user attaching custom tools to this world needs nothing from this checklist beyond the store API; document any conventions the model expects (key formats, enum values).
|
|
137
138
|
|
|
139
|
+
### Store API
|
|
140
|
+
|
|
141
|
+
Tool handlers (`tools/*.py`) reach the world's rows through `from gateway_env import store`. It has
|
|
142
|
+
exactly these methods; `gateway worlds schema check .` refuses any other `store.<name>` at file:line
|
|
143
|
+
with this list, and a hosted call into such a handler answers 400 `{error: "handler", detail, handler}`
|
|
144
|
+
(an `AttributeError`/`TypeError`/`NameError` is the author's bug; anything else a handler raises is a
|
|
145
|
+
500 `{error: "engine", detail, handler}`; never a bare exception string).
|
|
146
|
+
|
|
147
|
+
- `get(entity: str, key: dict) -> dict | None` — one row by primary key, or `None`.
|
|
148
|
+
- `list(entity: str) -> list[dict]` — every row, ordered by key.
|
|
149
|
+
- `list_where(entity: str, where: dict[str, list], limit: int | None = None, offset: int = 0) -> tuple[list[dict], int]` — a page of rows whose `where` fields hold one of the listed values, plus the total.
|
|
150
|
+
- `insert(entity: str, value: dict, *, source: dict | None = None) -> dict` — validate against the entity's schema and add; the key must be new.
|
|
151
|
+
- `update(entity: str, key: dict, patch: dict, *, source: dict | None = None) -> dict` — merge `patch` into the row at `key`, validate, return the row.
|
|
152
|
+
- `delete(entity: str, key: dict, *, source: dict | None = None) -> None` — remove the row at `key`; a missing row is a contract error.
|
|
153
|
+
- `apply_batch(batch: dict) -> dict` — one ingest batch `{id, target, mapping, coverage, mutations}`, atomically; an exact retry is a no-op.
|
|
154
|
+
- `lineage(entity: str, key: dict) -> list[dict]` — the batches that wrote the row at `key`, oldest first.
|
|
155
|
+
- `counts() -> dict[str, int]` — rows per entity, `{entity: count}`.
|
|
156
|
+
|
|
157
|
+
No `upsert`: `get`, then `insert` or `update`. Read-only SQL: `gateway_env.db.one/all`; raw SQL writes are refused. The scaffolded `tools/handlers.py` docstring carries the same table.
|
|
158
|
+
|
|
138
159
|
### 6. Surface binding (only if the world declares it mirrors an MCP surface)
|
|
139
160
|
|
|
140
161
|
- [ ] Export a real connection's filtered `tools/list` through the MCP proxy under a grant, or inspect its `AgentSecurityResource` tool rows. Native Slack source capture has no live MCP catalog; its schema template tools are separate offline behavior.
|
|
@@ -142,6 +163,11 @@ deliverable can finish with synthetic fixtures and explicitly skipped live valid
|
|
|
142
163
|
- [ ] Compare read-only tools manually against a live connection only when such comparison is authorized and in scope.
|
|
143
164
|
- [ ] Native manifest changes bump `TOOL_VERSIONS`.
|
|
144
165
|
|
|
166
|
+
### 6b. Reader plan (only if the template fills from an active Merge connection)
|
|
167
|
+
|
|
168
|
+
- [ ] `readers/merge.toml` names `transport = "merge"`, the Merge `provider` slug (`google_drive`, not the template slug) and one `[[pulls]]` per logical tool (`list_files`, unprefixed — `MergeConnection` adds `google_drive__`); arguments, paging (`page = { cursor, next }`) and `max_pages` live here and nowhere else: a caller (CLI, MCP, route) can never pick a tool or pass arguments. `readers/merge.ingest.toml` maps each tool's records to entities with the `data ingest --map` grammar.
|
|
169
|
+
- [ ] Prove it against the fake Agent Handler (`scripts/worlds-e2e/fake-merge-agent-handler.mjs`, dataset per fixture) through `gateway worlds create <slug> --from-connection <connector>` and `gateway worlds data pull <slug> --connection <id> --mode append`; every tool the plan names must land rows (`unmatched` empty). A live pull needs an owner-authorized active connection and is recorded as such, not implied by the fake.
|
|
170
|
+
|
|
145
171
|
### 7. Ship the template; register platform integration only when in scope
|
|
146
172
|
|
|
147
173
|
- [ ] SDK-shipped: commit authored files under `gatewaysdk/sdk/src/gatewaysdk/environment/schema/connectors/<slug>/`; `list_templates()` and `test_schema_scaffold.py` cover the slug and preserve its authored dependencies/provenance during initialization. Schema-only templates declare an empty tools map and have no operation handlers.
|
|
@@ -167,7 +193,7 @@ When adding a bridge command to `schema/platform.py`, mirror it as a typed wrapp
|
|
|
167
193
|
`worldSchemaRuntime.ts` (`compileWorldFiles` / `applyWorldBatch` are the pattern: file map
|
|
168
194
|
in, `{entries, report, ...}` out, errors as `SchemaBridgeError`), cover it in
|
|
169
195
|
`tests/environment/test_schema_platform.py`, and let the agent tool call the wrapper, never
|
|
170
|
-
the subprocess. The bridge needs Python ≥ 3.
|
|
196
|
+
the subprocess. The bridge needs Python ≥ 3.12 (`tomllib`, and the store's sqlite `setconfig`); set `GATEWAY_SCHEMA_PYTHON`
|
|
171
197
|
when the default interpreter is older.
|
|
172
198
|
|
|
173
199
|
Use `worldSchemaRuntime.ts` for platform validation and `schema.platform` for
|
|
@@ -243,12 +269,19 @@ These source checks do not imply live-provider validation or executed fidelity s
|
|
|
243
269
|
|
|
244
270
|
A world can describe the vendor it mirrors in `connector.toml` at its root: base URL,
|
|
245
271
|
auth scheme (`header`, `bearer`, `basic`, `query`, `oauth2`, `none`) with the secret named by an
|
|
246
|
-
environment variable
|
|
272
|
+
environment variable (for `basic`, the credential is the Basic *user* and the password is ignored:
|
|
273
|
+
the host and `worlds serve` present `Basic b64("<key>:")`), capture pacing, a redaction policy (`hash` or `drop` fields at any
|
|
247
274
|
depth), and the operations the agent actually calls (method, path with `{args}`, query
|
|
248
275
|
params, and the entity plus results path their rows land in). The same file is meant to
|
|
249
276
|
drive the platform's custom source later, so describe once.
|
|
250
277
|
|
|
251
|
-
- `gateway worlds connector validate PATH` parses the spec
|
|
278
|
+
- `gateway worlds connector validate PATH` parses the spec, binds every operation's `entity`
|
|
279
|
+
(and each `explode` target) to the entities `schema/world.json` declares, and lists operations.
|
|
280
|
+
A route onto an undeclared entity is refused with `operation X lands on entity Y, which
|
|
281
|
+
schema/world.json does not declare (declared: …)` at `/operations/<i>/entity`; `schema
|
|
282
|
+
compile`, `schema check`, `worlds describe` and session open apply the same rule and write nothing
|
|
283
|
+
(it used to pass every gate, show under `served`, and 500 on the first call). A version stored
|
|
284
|
+
before this rule with such a route refuses to open instead; `worlds describe` names the fix.
|
|
252
285
|
- `gateway worlds connector capture PATH --op NAME --arg k=v --param k=v` calls the live
|
|
253
286
|
connection, follows the spec's cursor up to `max_pages`, and writes every page under
|
|
254
287
|
`captures/<op>/` plus `captures/manifest.json` (sha256 per file). The recorded request
|
|
@@ -329,6 +362,23 @@ no `--host`/`--remote`/`workspace:<slug>`, platform env stripped, pinned to its
|
|
|
329
362
|
so publishing always goes through the tools above. To parallelise several connectors, `spawn_subagent` with a brief
|
|
330
363
|
naming only the tools that child needs (`allowSubagents` on the session).
|
|
331
364
|
|
|
365
|
+
### From the CLI sandbox (no checkout, no Python)
|
|
366
|
+
|
|
367
|
+
`gateway worlds sandbox open <slug>@main` puts the same CLI in front of the world's source tree
|
|
368
|
+
server-side: `sandbox exec <id> 'gateway worlds schema entity add . <entity> --fields … -k id'`,
|
|
369
|
+
`'gateway worlds schema verifier add …'`, `'gateway worlds schema task add …'`, `'gateway worlds
|
|
370
|
+
schema compile . && gateway worlds schema check .'`, `'gateway worlds test .'` and `'gateway worlds
|
|
371
|
+
connector validate|conform .'` run on the uncommitted copy (`sqlite3` is on PATH too); `sandbox check`
|
|
372
|
+
runs the push gate and `sandbox commit` lands the version. `open` reports the staged CLI under `cli`
|
|
373
|
+
(`null` plus `warnings` when the image cannot host it — then only plain shell runs).
|
|
374
|
+
|
|
375
|
+
With an API key and no world yet, start the world itself there:
|
|
376
|
+
`gateway worlds sandbox open --new <slug> --from custom` (MCP `open_world_sandbox({ new, from })`)
|
|
377
|
+
opens on a world that does not exist yet, its tree the exact `schema init` scaffold (or
|
|
378
|
+
`--from <template>` for a shipped one); the same `sandbox exec` verbs run on that tree with the
|
|
379
|
+
staged CLI, `sandbox check` runs the push gate, and the first `sandbox commit -m …` creates the
|
|
380
|
+
world (first READY version on `main`). A taken slug is a 409 `world_exists`, never a rename.
|
|
381
|
+
|
|
332
382
|
### Serving the vendor's API from the world
|
|
333
383
|
|
|
334
384
|
A provider's client speaks the vendor's REST paths, not tool calls, so the world answers
|
|
@@ -78,8 +78,31 @@ person so they stay consistent; `role: "public"` is never replaced. Every varian
|
|
|
78
78
|
tracked (bare surname, handle spellings, `(at)` emails, phone formats, percent-encoding, a
|
|
79
79
|
world's `sha256:` stored form), and in any case: `Alexr` or `ALEXR` for a seeded `alexr` is swapped
|
|
80
80
|
where it stands alone, and the stand-in keeps the case it arrived in. The door swaps and never
|
|
81
|
-
blocks.
|
|
82
|
-
report fails the run
|
|
81
|
+
blocks. A leak is about a **case subject**, an entity the task seeded: what slips through is counted
|
|
82
|
+
(`leak_through_inbound`), and a subject's real value in the outgoing report fails the run
|
|
83
|
+
(`leaks_outbound`). Someone who only turned up in a result and got a stand-in on sight is a bystander:
|
|
84
|
+
swapped the same way, counted apart (`bystander_leak_through_inbound`, `bystander_leaks_outbound`),
|
|
85
|
+
never a failure.
|
|
86
|
+
|
|
87
|
+
## Choose the stand-in yourself
|
|
88
|
+
|
|
89
|
+
When a case must look a particular way to the model, or the sources spell a real value in
|
|
90
|
+
ways the door cannot guess, declare it on the seed instead of relying on the pools:
|
|
91
|
+
|
|
92
|
+
```json
|
|
93
|
+
{"type": "username", "value": "<real handle>", "alias": "kmarsh7", "variants": ["<the forum's misspelling>"]}
|
|
94
|
+
{"type": "person_name", "value": "<real name>", "alias": "Kim Marsh"}
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
`alias` is the stand-in the model sees; every form the kind has (name order, `(at)` emails, phone
|
|
98
|
+
formats, case) is paired from it. `variants` are other spellings of the real value; each is paired
|
|
99
|
+
with the same stand-in, so a misspelling in a forum post or a phone written without its country
|
|
100
|
+
digit is swapped like the value itself. Leave both out and the door generates a persona.
|
|
101
|
+
|
|
102
|
+
When the case must not show a generated persona at all (a model that recognises the reserved IP
|
|
103
|
+
range or the made-up domains stops trusting the data), set `declared_only = true` under `[alias]`:
|
|
104
|
+
every private seed then needs its own `alias`, a seed without one is refused before the run, and
|
|
105
|
+
`mint_inbound` stays off.
|
|
83
106
|
|
|
84
107
|
## Identities that arrive, not seeded
|
|
85
108
|
|
|
@@ -175,6 +198,8 @@ gateway hooks audit runs/<run id>/ --json # the
|
|
|
175
198
|
```
|
|
176
199
|
|
|
177
200
|
`fields.json` is `{"*": {"full_name": "person_name", "email": "email"}}`, or keyed by tool name.
|
|
201
|
+
A directory with neither `raw.jsonl` nor `shown.jsonl` (a `task run --out-dir`, say) never ran under
|
|
202
|
+
hooks: the audit says so and exits 2 rather than reading the run's own `run.json` as hooks state.
|
|
178
203
|
|
|
179
204
|
It knows a value is real from three places, strongest first: your legend; every value of a field
|
|
180
205
|
whose kind you declared (a name in a vendor row that reappears in a web snippet is caught for
|
|
@@ -15,6 +15,8 @@ Companion to [SKILL.md](SKILL.md). The contract is fixed in
|
|
|
15
15
|
8. [MCP payloads](#mcp-payloads)
|
|
16
16
|
9. [Chunks, batches and refusal records](#chunks-batches-and-refusal-records)
|
|
17
17
|
10. [Extract adapters](#extract-adapters)
|
|
18
|
+
11. [CSV and TSV cells](#csv-and-tsv-cells)
|
|
19
|
+
12. [Data snapshots](#data-snapshots)
|
|
18
20
|
|
|
19
21
|
## Shapes
|
|
20
22
|
|
|
@@ -46,6 +48,16 @@ gateway worlds data import ./world users.json --entity users
|
|
|
46
48
|
gateway worlds data import ./world users.jsonl --entity users
|
|
47
49
|
```
|
|
48
50
|
|
|
51
|
+
**CSV/TSV with `--entity NAME`.** A `.csv`/`.tsv` (or `.gz`) whose header row names the fields;
|
|
52
|
+
every cell is cast by the entity field's schema type (integer, number, boolean, array/object as
|
|
53
|
+
JSON, null for an empty nullable cell) — [CSV and TSV cells](#csv-and-tsv-cells). `--delimiter`
|
|
54
|
+
picks the separator. Not accepted from `drive:` (import it from the machine that holds it, or
|
|
55
|
+
`worlds sandbox import` it from a sandbox).
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
gateway worlds data import ./world tickets.csv --entity hd_tickets
|
|
59
|
+
```
|
|
60
|
+
|
|
49
61
|
A file that mixes shapes is refused. Several files on one command are applied as one batch:
|
|
50
62
|
all or nothing (`--per-chunk` on a platform import relaxes that to one chunk at a time). Files
|
|
51
63
|
are streamed, so size is not a concern: pass `-` for stdin, `.gz` files are gunzipped.
|
|
@@ -241,3 +253,53 @@ rows add missing fields; `upsert`: last wins); the world's `[redact]` policy app
|
|
|
241
253
|
by element, `round` rounded). Output is tagged JSONL; stderr gets
|
|
242
254
|
`{"read": N, "rows": M, "unknownFields": {name: n}, "redacted": {"dropped": n, "hashed": n}}`.
|
|
243
255
|
The fixtures under `gatewaysdk/fixtures/world-data/extract/` are the worked examples.
|
|
256
|
+
|
|
257
|
+
## CSV and TSV cells
|
|
258
|
+
|
|
259
|
+
A `.csv`/`.tsv` (optionally `.gz`) is read as a stream — memory is bounded at any size — and
|
|
260
|
+
split by `--delimiter` (one character, `\t` for a tab; default `,`, tab for a `.tsv`). The header
|
|
261
|
+
row names the fields; `--entity` says which entity they fill. Every cell is cast by that field's
|
|
262
|
+
schema type, the same rule in the TypeScript CLI (`worldData/cast.ts`) and the Python package
|
|
263
|
+
(`schema/cast.py`), pinned by `gatewaysdk/sdk/tests/conformance/csv_cast.json`:
|
|
264
|
+
|
|
265
|
+
| Field type | Cell | Value |
|
|
266
|
+
| --- | --- | --- |
|
|
267
|
+
| `integer` | `42`, `-7` | the number; `4.5` or `abc` is refused |
|
|
268
|
+
| `number` | `12.5`, `1e3`, `3` | the number (always a float) |
|
|
269
|
+
| `boolean` | `true`/`false`/`1`/`0`, any case | the boolean; `yes` is refused |
|
|
270
|
+
| `array` / `object` | JSON text of that shape (quote it in the CSV: `"[""a"",""b""]"`) | the parsed value |
|
|
271
|
+
| `string` | anything | the text as given |
|
|
272
|
+
| `integer \| string` (union) | each member tried in order, `string` last | the first that fits |
|
|
273
|
+
| any, empty cell | `` | null when the type admits null; left out when the field is optional; `""` for a required string; refused otherwise |
|
|
274
|
+
|
|
275
|
+
A cell that fits none is a row-level refusal naming the field and the cell —
|
|
276
|
+
`/taxi_trip/3: fare = "n/a" is not a number` — with the usual pointer; nothing is written. A
|
|
277
|
+
column the schema does not name stays text and the gate refuses it as an unknown field. A CSV
|
|
278
|
+
on the drive (`drive:<path>`) is still refused: import it from the machine that holds it, or
|
|
279
|
+
through `worlds sandbox import` when it sits in a sandbox.
|
|
280
|
+
|
|
281
|
+
```bash
|
|
282
|
+
gateway worlds data import ./world trips.csv --entity taxi_trip
|
|
283
|
+
gateway worlds data import ./world trips.tsv --entity taxi_trip # tab by name
|
|
284
|
+
gateway worlds data import acme export.txt.csv --entity rows --delimiter ';'
|
|
285
|
+
```
|
|
286
|
+
|
|
287
|
+
## Data snapshots
|
|
288
|
+
|
|
289
|
+
Why a served world starts in under a second.
|
|
290
|
+
|
|
291
|
+
A schema world's validated database is kept, not rebuilt. `gateway worlds schema compile`
|
|
292
|
+
builds `db/state.sqlite` (the default snapshot: `data/initial.json` applied through the
|
|
293
|
+
contract) and `scenarios/<name>/state.sqlite` (the default with the scenario's overlays),
|
|
294
|
+
each with a `state.json` sidecar `{contractHash, batchDigest, entityCounts, builtAt,
|
|
295
|
+
runtime, fileSha256}`; the same key is never rebuilt, `schema check` verifies the sidecars,
|
|
296
|
+
and a data batch the contract refuses leaves no snapshot behind. `gateway worlds serve` and
|
|
297
|
+
the api bundle open the snapshot (copy + `SQLiteStore.open`, no row replay) and reset by
|
|
298
|
+
copying it again; without one the JSON is replayed as before. A hosted pod gets
|
|
299
|
+
`WORLD_SNAPSHOT_URL` (presigned GET, `.gz` or `Content-Encoding: gzip` is gunzipped),
|
|
300
|
+
`WORLD_SNAPSHOT_SHA256` and `WORLD_SNAPSHOT_DIGEST`; the server downloads, verifies, writes
|
|
301
|
+
the sidecar and only then serves; a mismatch exits 2 naming the digest. `/__world/state`
|
|
302
|
+
reports `"snapshot": {batchDigest, contractHash}` (or null) and `/__world/seed` takes the
|
|
303
|
+
overlay form `{"base": digest, "rows": {...}, "mode": append|replace}` beside the legacy
|
|
304
|
+
whole-state body. Snapshot files are build outputs: gitignore `db/state.sqlite`,
|
|
305
|
+
`db/state.json`, `scenarios/*/state.*`.
|