@withgateway/sdk 0.2.6 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +24 -8
- package/dist/cli.js +15468 -11263
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +15233 -11028
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +291 -40
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +284 -37
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +203 -47
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +198 -42
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-BaKu5ZSU.d.mts → index-C3UzD0G3.d.mts} +75 -1
- package/dist/{index-BaKu5ZSU.d.ts → index-C3UzD0G3.d.ts} +75 -1
- package/dist/{index-CLkMr6VG.d.ts → index-D-_4Ibcn.d.ts} +36 -4
- package/dist/{index-CdbmO4aX.d.mts → index-FfK4Nump.d.mts} +36 -4
- package/dist/index.d.mts +3 -3
- package/dist/index.d.ts +3 -3
- package/dist/index.js +2149 -615
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2053 -566
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js +268 -31
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +262 -28
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.js +19 -11
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +19 -11
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +886 -117
- package/dist/worlds/index.d.ts +886 -117
- package/dist/worlds/index.js +1770 -452
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1752 -434
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +14 -10
- package/runtime/gatewaysdk/environment/_toml.py +3 -10
- package/runtime/gatewaysdk/environment/_world.py +95 -28
- package/runtime/gatewaysdk/environment/runtime.py +10 -6
- package/runtime/gatewaysdk/environment/schema/__main__.py +35 -0
- package/runtime/gatewaysdk/environment/schema/api.py +19 -23
- package/runtime/gatewaysdk/environment/schema/batch.py +68 -14
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +4 -2
- package/runtime/gatewaysdk/environment/schema/cast.py +264 -0
- package/runtime/gatewaysdk/environment/schema/compiler.py +35 -1
- package/runtime/gatewaysdk/environment/schema/conform.py +28 -10
- package/runtime/gatewaysdk/environment/schema/connector.py +59 -14
- package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +38 -1
- package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.toml +3 -1
- package/runtime/gatewaysdk/environment/schema/credentials.py +20 -0
- package/runtime/gatewaysdk/environment/schema/csv_rows.py +117 -0
- package/runtime/gatewaysdk/environment/schema/cycles.py +121 -0
- package/runtime/gatewaysdk/environment/schema/data.py +21 -5
- package/runtime/gatewaysdk/environment/schema/grading.py +95 -13
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +44 -0
- package/runtime/gatewaysdk/environment/schema/handler_scan.py +175 -0
- package/runtime/gatewaysdk/environment/schema/host.py +68 -115
- package/runtime/gatewaysdk/environment/schema/host_requests.py +78 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +12 -0
- package/runtime/gatewaysdk/environment/schema/ingest.py +5 -18
- package/runtime/gatewaysdk/environment/schema/local.py +7 -10
- package/runtime/gatewaysdk/environment/schema/openapi_document.py +322 -0
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +382 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +441 -0
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +629 -0
- package/runtime/gatewaysdk/environment/schema/openapi_source.py +210 -0
- package/runtime/gatewaysdk/environment/schema/platform.py +110 -299
- package/runtime/gatewaysdk/environment/schema/platform_import.py +182 -0
- package/runtime/gatewaysdk/environment/schema/platform_query.py +95 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +176 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +56 -7
- package/runtime/gatewaysdk/environment/schema/scaffold.py +64 -5
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +70 -22
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +54 -4
- package/runtime/gatewaysdk/environment/schema/skills/hooks-getting-started/SKILL.md +27 -2
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +62 -0
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +91 -54
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +146 -146
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +11 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +30 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/new-world-sandbox.md +43 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +90 -0
- package/runtime/gatewaysdk/environment/schema/store.py +15 -6
- package/runtime/gatewaysdk/environment/schema/store_contract.py +119 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +121 -32
- package/runtime/gatewaysdk/environment/schema/tools_world.py +32 -10
- package/runtime/gatewaysdk/environment/schema/tree_import.py +228 -0
- package/runtime/gatewaysdk/environment/schema/ui.py +18 -2
- package/runtime/gatewaysdk/environment/schema/validation.py +35 -0
- package/runtime/gatewaysdk/environment/schema/verbs.py +8 -2
- package/runtime/gatewaysdk/environment/world.py +42 -3
- package/runtime/gatewaysdk/environment/world_session_contract.py +57 -0
- package/skills/benchmarks-getting-started/SKILL.md +70 -22
- package/skills/connector-schema-authoring/SKILL.md +54 -4
- package/skills/connector-schema-authoring/connector-worlds.md +21 -7
- package/skills/connector-schema-authoring/world-schema.md +144 -83
- package/skills/hooks-getting-started/SKILL.md +27 -2
- package/skills/world-data-ingestion/ROWS.md +62 -0
- package/skills/world-data-ingestion/SKILL.md +91 -54
- package/skills/worlds-getting-started/SKILL.md +146 -146
- package/skills/worlds-getting-started/references/cli-mcp-map.md +11 -7
- package/skills/worlds-getting-started/references/gotchas.md +30 -0
- package/skills/worlds-getting-started/references/new-world-sandbox.md +43 -0
- package/skills/worlds-getting-started/references/openapi-import.md +90 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: benchmarks-getting-started
|
|
3
|
-
description: Build, validate, dry-run, push, run and read a benchmark on the platform, for an agent (Claude Code, Cursor, Codex). A benchmark is either a world with tasks (tools + data, a schema when the vendor has one; a prompt and the SQL verifiers that grade the world's end state) or a container benchmark (a harbor task tree graded by its own tests/test.sh in a container, or a verifiers package graded by Python reward functions), run k times per task per model. Use when building a benchmark, an eval suite, a coding benchmark, a task set or a leaderboard for an agent; when you have a harbor task tree, task.toml files, a verifiers package, a Taskset, or tests that grade an agent; when choosing between a world benchmark and a container (benchmark runner) benchmark; when you need a dry run before pushing; when running k rollouts per task per model or reading per-task results. Every CLI step names its MCP twin (create_world, env_write_file, env_commit, create_environment_version, run_evaluation, get_task_performance, list_rollouts, get_rollout_samples) and where the result shows in the UI.
|
|
3
|
+
description: Build, validate, dry-run, push, run and read a benchmark on the platform, for an agent (Claude Code, Cursor, Codex). A benchmark is either a world with tasks (tools + data, a schema when the vendor has one; a prompt and the SQL verifiers that grade the world's end state) or a container benchmark (a harbor task tree graded by its own tests/test.sh in a container, or a verifiers package graded by Python reward functions), run k times per task per model. Use when building a benchmark, an eval suite, a coding benchmark, a task set or a leaderboard for an agent; when you have a harbor task tree, task.toml files, a verifiers package, a Taskset, or tests that grade an agent; when choosing between a world benchmark and a container (benchmark runner) benchmark; when you need a dry run before pushing (locally in Docker, or hosted on the benchmark runner with `--hosted` when the machine has no Docker); when running k rollouts per task per model or reading per-task results. Every CLI step names its MCP twin (create_world, env_write_file, env_commit, create_environment_version, run_evaluation, get_task_performance, list_rollouts, get_rollout_samples) and where the result shows in the UI.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# Benchmarks: getting started
|
|
@@ -41,20 +41,23 @@ Segmentation, so one substrate never crashes into the other's infrastructure:
|
|
|
41
41
|
| Storage | a world version (`POST /api/public/worlds`; later versions through the same hub as containers) | a benchmark-container version (`bench push`: resolve, versions, finalize) |
|
|
42
42
|
| Engine | World Host (a hosted session per rollout; the mock-tools runner for the legacy bundle) | benchmark runner on AWS Batch: `harbor run` per task, or verifiers v1 `run_eval` in-process |
|
|
43
43
|
| Results rows | evaluations, samples, `GET /environments/{id}/performance` | the same rows and endpoint |
|
|
44
|
+
| Dry run | local only: the task's session in the bundled runtime (`benchmarks dry-run <dir>`) | local (Docker, `benchmarks dry-run <dir>`) **or hosted** (`--hosted`: the runner, the oracle, the real image rule, `POST /environments/{id}/dry-run`, harbor only; a verifiers package or a world is refused with a 400 naming the local command); a hosted dry run is `kind: dry-run`, never evidence |
|
|
44
45
|
| Refused when a tree crosses over | `worlds create` answers `world_kind_retired` for a harbor tree or a verifiers package, before any write; the hub's versions door answers `benchmark_substrate_mismatch` (409) for a container tree pushed under a world's slug | the hub's versions door answers `benchmark_substrate_mismatch` for a world tree pushed under a container's slug; the runner's harbor engine refuses a schema bundle (`require_legacy_bundle`) |
|
|
45
46
|
| Read before writing | `benchmarks push` reads the slug's substrate (`GET /environments/{id}` → `substrate`) and refuses a cross-substrate push with exit 2 and no request; `--kind` that disagrees with the tree is refused before that | the same pre-check; a container benchmark that IS a container benchmark opens on the runner — `session open` / `task create` do not refuse it (only the legacy tools bundle is retired on serve) |
|
|
46
47
|
|
|
47
48
|
## 0. Authenticate
|
|
48
49
|
|
|
49
50
|
```bash
|
|
50
|
-
npm install -g @withgateway/sdk # the gateway CLI (TypeScript; python3 >= 3.
|
|
51
|
+
npm install -g @withgateway/sdk # the gateway CLI (TypeScript; python3 >= 3.12 runs the local runtime)
|
|
51
52
|
gateway auth login --host https://<your-gateway> --public-key pk-lf-... --api-key sk-lf-...
|
|
52
53
|
```
|
|
53
54
|
|
|
54
|
-
`init`, `validate`, `dry-run` and the local world commands run without credentials; `push`, `run
|
|
55
|
-
`results` need them. `dry-run` on a harbor tree executes in Docker when Docker
|
|
56
|
-
the exact plan when it is not;
|
|
57
|
-
(
|
|
55
|
+
`init`, `validate`, `dry-run` and the local world commands run without credentials; `push`, `run`,
|
|
56
|
+
`results` and `dry-run --hosted` need them. `dry-run` on a harbor tree executes in Docker when Docker
|
|
57
|
+
is reachable and prints the exact plan when it is not; `dry-run --hosted` runs the same one task on
|
|
58
|
+
the benchmark runner (harbor's oracle, no model), so a machine with nothing but the npm CLI and project
|
|
59
|
+
keys can author, dry-run, push and run a container benchmark; on a world it opens the task's session
|
|
60
|
+
in the bundled Python runtime (python3 >= 3.12) and prints the plan when there is none.
|
|
58
61
|
|
|
59
62
|
## A. A harbor (container) benchmark: init → validate → dry-run → push → run → results
|
|
60
63
|
|
|
@@ -177,6 +180,50 @@ docker-compose file requires EXA_API_KEY … export EXA_API_KEY here for the dry
|
|
|
177
180
|
it from … gateway bench secrets set --env code-bench --key EXA_API_KEY`, exit 0, nothing spawned.
|
|
178
181
|
Values never appear in the printed argv.
|
|
179
182
|
|
|
183
|
+
**Hosted (`--hosted`): the runner, the real image rule, no Docker here.** A Dockerfile-first local
|
|
184
|
+
dry-run passes a task whose `task.toml` names `docker_image`, because harbor never builds that
|
|
185
|
+
Dockerfile on the runner (A1); the hosted dry-run catches it in about two minutes, not in a
|
|
186
|
+
15-minute model run.
|
|
187
|
+
|
|
188
|
+
```bash
|
|
189
|
+
gateway benchmarks dry-run code-bench --task hello --hosted # a pushed slug (or slug@ref)
|
|
190
|
+
gateway benchmarks dry-run ./code-bench --task hello --hosted # a dir that tracks a slug (.gateway/.env-metadata.json)
|
|
191
|
+
gateway benchmarks dry-run ./code-bench --task hello --hosted --push # the dir differs from the pushed version: push first, then dry-run that version
|
|
192
|
+
gateway environment dry-run code-bench --task hello # the same run under the environment group (the Python CLI's spelling)
|
|
193
|
+
```
|
|
194
|
+
|
|
195
|
+
ONE Batch job on the benchmark runner (`POST /api/public/environments/{id}/dry-run` with
|
|
196
|
+
`{ task, versionId? }`): harbor kind only, `--agent oracle` (the task's `solution/solve.sh` in place
|
|
197
|
+
of the agent), the ONE named task (the runner prunes the working copy to it, the same
|
|
198
|
+
`GATEWAY_TASK_NAMES` filter sharded runs use), one rollout, one example — capped on the server, a body
|
|
199
|
+
that asks for more is refused (400) — the image chosen by harbor's rule and the `bench secrets` compose
|
|
200
|
+
requirements enforced exactly like a real run (missing → refused before dispatch with the
|
|
201
|
+
`gateway bench secrets set …` command). Expected closing lines:
|
|
202
|
+
|
|
203
|
+
```
|
|
204
|
+
image: build environment/Dockerfile # or `image: ghcr.io/acme/tool:1.2 (prebuilt: docker_image)`
|
|
205
|
+
ran: harbor run -p …/tasks -a oracle -e docker -o …/jobs -k 1 -n 1 -y -q -l 1 # no -m: the oracle calls no model; the working copy holds only `hello`
|
|
206
|
+
reward: 1 {"reward":1,"file_exists":1,"content_ok":1} · exit 0
|
|
207
|
+
where: hosted (code-bench 0.1.0 · 3f9a2c1d · task hello · agent oracle, no model · not evidence)
|
|
208
|
+
run: <run_id> · COMPLETED
|
|
209
|
+
page: https://<your-gateway>/project/<id>/world/runs/<evaluation_id>
|
|
210
|
+
status: gateway environment run-status <run_id>
|
|
211
|
+
```
|
|
212
|
+
|
|
213
|
+
Exit 0 only when the oracle scored 1.0; lower → exit 2 naming it; nothing graded → exit 2
|
|
214
|
+
`reward: null (ungraded)`; an image pull/build failure → exit 2 with the runner's error tail (bounded,
|
|
215
|
+
no host paths, no secret values). `--wait` (default, `--wait-timeout 1200`) and `--no-wait` (return
|
|
216
|
+
after dispatch with the run id and the `run-status` command) mirror `benchmarks run`; `--json` prints
|
|
217
|
+
dispatch + verdict. A dir with uncommitted changes vs the pushed version is refused (exit 2, nothing
|
|
218
|
+
dispatched) unless `--push`, which goes through the ONE `benchmarks push` path. A world or a verifiers
|
|
219
|
+
package is refused with a 400 that names the local alternative (`benchmarks dry-run <dir>`).
|
|
220
|
+
|
|
221
|
+
**Not evidence.** The run is `kind: dry-run` (`evidence: false`, `agent: oracle`): the run page and
|
|
222
|
+
`environment run-status` show `dryRun`, the rollouts list labels the evaluation `dryRun: true`,
|
|
223
|
+
`benchmarks results` / `get_task_performance` exclude it, and the Batch job is named `bench-dryrun-…`
|
|
224
|
+
with `GATEWAY_DRY_RUN=oracle` in its environment, so an operator can tell it apart in the queue. MCP twin:
|
|
225
|
+
`dry_run_environment` (`containerId`, `task`, `versionId?`).
|
|
226
|
+
|
|
180
227
|
### A4. Push
|
|
181
228
|
|
|
182
229
|
```bash
|
|
@@ -211,15 +258,16 @@ MCP: `create_environment_version` (or `env_write_file` + `env_commit`).
|
|
|
211
258
|
```bash
|
|
212
259
|
gateway benchmarks run code-bench --model openrouter/openai/gpt-5-mini --model anthropic/claude-sonnet-4-6 -k 3
|
|
213
260
|
gateway benchmarks run code-bench -k 5 --no-wait # models from benchmark.json; return after dispatch
|
|
214
|
-
gateway environment run-status <run_id> #
|
|
261
|
+
gateway environment run-status <run_id> # verdict; while running: `0/15 rollouts, 95s since dispatch`
|
|
262
|
+
gateway bench run <run_id> --cancel # CANCELLED here, runner job terminated on Batch (counts what Batch accepted)
|
|
215
263
|
```
|
|
216
264
|
|
|
217
265
|
One runner job per model (`POST /api/public/environments/{id}/run`): it downloads the version, runs
|
|
218
266
|
`harbor run -p tasks -a mini-swe-agent -m <model> -e docker -k <rollouts>` — the agent in each task's
|
|
219
267
|
container, then `tests/test.sh` — and harvests every trial into an evaluation. Models are
|
|
220
268
|
litellm-style, provider-prefixed (`openrouter/vendor/model`, `anthropic/…`); the project's LLM
|
|
221
|
-
connection supplies the key. `--num-examples N` caps the tasks
|
|
222
|
-
|
|
269
|
+
connection supplies the key. `--num-examples N` caps the tasks; `-k` is 1–64 (the platform refuses
|
|
270
|
+
more before any row or job exists, whatever the client sent). The first run builds images: ~15 min.
|
|
223
271
|
|
|
224
272
|
MCP: `run_evaluation({ models, versionId?, numExamples?, rolloutsPerExample? })`; `list_rollouts`.
|
|
225
273
|
|
|
@@ -227,7 +275,7 @@ MCP: `run_evaluation({ models, versionId?, numExamples?, rolloutsPerExample? })`
|
|
|
227
275
|
|
|
228
276
|
```bash
|
|
229
277
|
gateway benchmarks results code-bench # per task: mean reward, pass rate, samples, no score
|
|
230
|
-
gateway benchmarks results code-bench --runs --json
|
|
278
|
+
gateway benchmarks results code-bench --runs --json # also --trend | --all | --version 1.2.0
|
|
231
279
|
gateway environment samples <container_id> <evaluation_id>
|
|
232
280
|
```
|
|
233
281
|
|
|
@@ -236,12 +284,12 @@ Expected:
|
|
|
236
284
|
```
|
|
237
285
|
task mean reward pass rate samples no score
|
|
238
286
|
hello 1.000 100% 3 0
|
|
239
|
-
|
|
240
|
-
6 samples, 0 without a score
|
|
287
|
+
3 samples, 0 without a score
|
|
241
288
|
```
|
|
242
289
|
|
|
243
|
-
`GET /api/public/environments/{id}/performance`; null scores are their own count.
|
|
244
|
-
|
|
290
|
+
`GET /api/public/environments/{id}/performance`; null scores are their own count; `reward.json` sub-rewards
|
|
291
|
+
are on each sample. MCP: `get_task_performance`, `get_rollout_samples`. Default is the newest version with
|
|
292
|
+
results; `--json` carries the table's selection (`view`, `version`, `tasks`), never every version's rows.
|
|
245
293
|
|
|
246
294
|
| Thing | Where in the UI |
|
|
247
295
|
| --- | --- |
|
|
@@ -361,7 +409,7 @@ reward is for the untouched world; an agent's rollout is what `benchmarks run` g
|
|
|
361
409
|
|
|
362
410
|
A verifier that cannot run — bad SQL, a missing entity — prints `FAIL verifier 'x': …` with SQLite's
|
|
363
411
|
words and exits 1; that is what the dry run is for. A reward of 1.0 here means the verifier passes
|
|
364
|
-
before any agent acts: rewrite it. Without the bundled runtime (no python3 >= 3.
|
|
412
|
+
before any agent acts: rewrite it. Without the bundled runtime (no python3 >= 3.12) the four steps
|
|
365
413
|
print and the run is marked not executed, exit 0.
|
|
366
414
|
|
|
367
415
|
A hand-driven rollout, the way an agent's session goes:
|
|
@@ -419,8 +467,10 @@ Every run records the version it ran against; `results` shows the per-version tr
|
|
|
419
467
|
| `benchmarks init` | none (local); `env_write_file` + `env_commit` on an existing environment |
|
|
420
468
|
| `benchmarks validate` | world: `validate_connector_template({ files })`; container: none (local) |
|
|
421
469
|
| `benchmarks dry-run` | none (local: Docker for harbor, the bundled Python runtime for a world) |
|
|
470
|
+
| `benchmarks dry-run --hosted` / `environment dry-run` | `dry_run_environment({ containerId, task, versionId? })`; `POST /api/public/environments/{id}/dry-run` (harbor only, oracle, one task, not evidence) |
|
|
422
471
|
| `benchmarks push` | world: `create_world`, `POST /api/public/worlds`; container: `create_environment_version` (`bench push`) |
|
|
423
|
-
| `benchmarks run` | `run_evaluation({ models, rolloutsPerExample })`; `POST /api/public/environments/{id}/run` |
|
|
472
|
+
| `benchmarks run` | `run_evaluation({ models, rolloutsPerExample })`; `POST /api/public/environments/{id}/run` (`rolloutsPerExample` 1–64) |
|
|
473
|
+
| `bench run <id> --cancel` / `environment run-status <id> --cancel` | `POST /api/public/environments/runs/{id}/cancel` (secret key; 409 once settled) |
|
|
424
474
|
| `benchmarks results` | `get_task_performance`, `list_rollouts`; `GET /api/public/environments/{id}/performance` |
|
|
425
475
|
| `worlds schema task add` | none (edits gateway-env.toml); stored tasks: `create_world_task` |
|
|
426
476
|
| `gateway skill search <words>` | `guide({ topic })` |
|
|
@@ -433,20 +483,18 @@ Every run records the version it ran against; `results` shows the per-version tr
|
|
|
433
483
|
writes nothing makes every trial ungraded (null), not 0. `mkdir -p /logs/verifier` first.
|
|
434
484
|
- harbor: a `gateway-env.toml` at the root makes the runner read the tree as a world (`mock-tools`);
|
|
435
485
|
`validate` names it.
|
|
436
|
-
- harbor: `solution/solve.sh` is for `dry-run`
|
|
486
|
+
- harbor: `solution/solve.sh` is for `dry-run` (local, or `--hosted`: the oracle on the runner); a model run ignores it.
|
|
437
487
|
- world: a verifier whose `initial_score` is 1.0 in `validate` — or whose `dry-run` reward is 1.0 — passes
|
|
438
488
|
before the agent acts; write it against the state the task must produce. `verifiers/<name>.sql` must
|
|
439
489
|
exist before `schema task add`; `dry-run` exits 1 naming a verifier whose SQL cannot run.
|
|
440
490
|
- `run` needs a model per run; `benchmark.json` `models` fills in when `--model` is absent. A run
|
|
441
|
-
that reads `settled: false` is still running, not failed.
|
|
491
|
+
that reads `settled: false` is still running, not failed; `bench run <id> --cancel` stops it.
|
|
442
492
|
- `--kind image` is refused: the runner has no hub engine for a bare image (its generic engine runs
|
|
443
493
|
registry entries only). Package the benchmark as a harbor task.
|
|
444
494
|
|
|
445
495
|
## Related
|
|
446
496
|
|
|
447
497
|
- `worlds-getting-started`: the world itself — contract, data, sessions, stored tasks, links.
|
|
448
|
-
- `world-data-ingestion`: rows in, calls out.
|
|
449
|
-
-
|
|
450
|
-
- Consumer pages: `gateway_documentation/pages/cli/benchmarks.mdx`, `pages/worlds/benchmarks.mdx`,
|
|
451
|
-
`pages/evaluation/benchmarks-harbor.mdx`.
|
|
498
|
+
- `world-data-ingestion`: rows in, calls out. `connector-schema-authoring`: contract, fidelity shapes.
|
|
499
|
+
- Consumer pages: `gateway_documentation/pages/cli/benchmarks.mdx`, `pages/worlds/benchmarks.mdx`, `pages/evaluation/benchmarks-harbor.mdx`.
|
|
452
500
|
- `documentation/features/benchmark-containers.md`: the container substrate, kinds, the runner.
|
|
@@ -55,6 +55,7 @@ registration, a platform builder, live ingestion, tools, or a registry framework
|
|
|
55
55
|
| Command or path | Status |
|
|
56
56
|
| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
|
57
57
|
| `gateway worlds schema init PATH --connector <slug> [--name NAME] [--with-example-tool]` | exists (#219); `--connector custom` is the blank starting point for an arbitrary connector: its `$id` is derived from `--name` (`urn:gateway:connector:<name>`), it ships the `records` entity and a starter verifier but no tools unless `--with-example-tool` asks for the `list_records`/`get_record`/`create_record` example, and it writes no `data/records.json` (Brex retrospective, 2026-09-14) |
|
|
58
|
+
| `gateway worlds schema init PATH --from-openapi SOURCE [--plan-only] [--include GLOB\|TAG]… [--exclude GLOB\|TAG]…` | exists (2026-09-22); imports a Swagger 2.0 / OpenAPI 3.x document (JSON or YAML; a local path, `drive:<path>` read platform-side, or an http(s) URL) into `schema/world.json` + `connector.toml` + `schema/provenance.json`: keyed schemas the selected operations return become entities, every path+method a route (passthrough when the response is not an entity), the plan is printed first and `--plan-only` writes nothing; refusals (no `paths`, unknown `$ref`, cyclic entity root, undeclared path parameter) carry a JSON pointer. Rules: `worlds-getting-started` → `references/openapi-import.md` |
|
|
58
59
|
| `gateway worlds connector validate\|capture\|ingest PATH` | exists; a `connector.toml` in the world describes base URL, auth scheme, pacing, redaction and operations; `capture` records paced, redacted pages (errors included) under `captures/` with the secret never written; `ingest` seeds `data/initial.json` through the contract in a scratch copy first; `ingest --dry-run [--report refusals.jsonl]` lists every violation of every capture (the `data import --report` records plus `capture`), exit 2, nothing written |
|
|
59
60
|
| `gateway worlds schema reconcile PATH --from captures [--write]` | exists (runtime, offline CLI and `gateway worlds schema reconcile`); the contract against the captured rows: relaxes `required` for fields absent in captures, adds undocumented keys as optional properties typed from the values, drops formats the captured values fail; a report of every change, `schema/world.json` rewritten and recompiled only with `--write` |
|
|
60
61
|
| `gateway worlds connector conform PATH --openapi vendor.yaml` | exists; serves the world in-process, calls every declared operation with arguments from its own seed, and checks status + body against the vendor's OpenAPI response schemas; non-zero exit on any mismatch. Run it before calling a world 1:1; its FAIL list is the work list (unseeded entities → 404s, creates the contract rejects → 4xx, honest 501s). Reads OpenAPI 3.0 `nullable`, reference cycles and `discriminator` unions; `[operations.api] sample_headers` / `sample_body` shape a request the route demands (`Idempotency-Key`, a body) |
|
|
@@ -92,7 +93,7 @@ deliverable can finish with synthetic fixtures and explicitly skipped live valid
|
|
|
92
93
|
### 0. Decide the shape
|
|
93
94
|
|
|
94
95
|
- [ ] **Fidelity shape**: base, tenant-shaped, or mixed. See [CONNECTORS.md](CONNECTORS.md). Tenant-shaped means describe-on-bind is a requirement of the world, and the shared template is only a skeleton.
|
|
95
|
-
- [ ] **Model source**: authored by hand (start from `--connector custom`), imported (`openapi`, `odata-csdl`, `discovery`, `wsdl-xsd`, `ldap-subschema`, `describe`, `docs`), or uploaded. Pin the spec URL and version now.
|
|
96
|
+
- [ ] **Model source**: authored by hand (start from `--connector custom`), imported from a Swagger/OpenAPI document (`schema init PATH --from-openapi SOURCE`, provenance pinned for you) or from another source by hand (`odata-csdl`, `discovery`, `wsdl-xsd`, `ldap-subschema`, `describe`, `docs`), or uploaded. Pin the spec URL and version now.
|
|
96
97
|
- [ ] **Scope of the model**: which entities and fields, decided from what the connector's users need the world to be faithful about. Record the reasoning in `scope.md` or `scope.toml`; coverage of the vendor's full model is reported, not required.
|
|
97
98
|
- [ ] **Sandbox**: record whether live validation is available and in scope. If it is not, record the reason and mark live checks `skipped`; a schema foundation can still ship as provisional `subset`.
|
|
98
99
|
- [ ] **Tools and surface** (optional): none, generated defaults, hand-written, user's custom, or mapped to an MCP surface. Decide now so the entity scope covers what those tools touch.
|
|
@@ -135,6 +136,26 @@ deliverable can finish with synthetic fixtures and explicitly skipped live valid
|
|
|
135
136
|
- [ ] Handlers never read `benchmark_runtime_context` or `benchmark_conversation_events`.
|
|
136
137
|
- [ ] A user attaching custom tools to this world needs nothing from this checklist beyond the store API; document any conventions the model expects (key formats, enum values).
|
|
137
138
|
|
|
139
|
+
### Store API
|
|
140
|
+
|
|
141
|
+
Tool handlers (`tools/*.py`) reach the world's rows through `from gateway_env import store`. It has
|
|
142
|
+
exactly these methods; `gateway worlds schema check .` refuses any other `store.<name>` at file:line
|
|
143
|
+
with this list, and a hosted call into such a handler answers 400 `{error: "handler", detail, handler}`
|
|
144
|
+
(an `AttributeError`/`TypeError`/`NameError` is the author's bug; anything else a handler raises is a
|
|
145
|
+
500 `{error: "engine", detail, handler}`; never a bare exception string).
|
|
146
|
+
|
|
147
|
+
- `get(entity: str, key: dict) -> dict | None` — one row by primary key, or `None`.
|
|
148
|
+
- `list(entity: str) -> list[dict]` — every row, ordered by key.
|
|
149
|
+
- `list_where(entity: str, where: dict[str, list], limit: int | None = None, offset: int = 0) -> tuple[list[dict], int]` — a page of rows whose `where` fields hold one of the listed values, plus the total.
|
|
150
|
+
- `insert(entity: str, value: dict, *, source: dict | None = None) -> dict` — validate against the entity's schema and add; the key must be new.
|
|
151
|
+
- `update(entity: str, key: dict, patch: dict, *, source: dict | None = None) -> dict` — merge `patch` into the row at `key`, validate, return the row.
|
|
152
|
+
- `delete(entity: str, key: dict, *, source: dict | None = None) -> None` — remove the row at `key`; a missing row is a contract error.
|
|
153
|
+
- `apply_batch(batch: dict) -> dict` — one ingest batch `{id, target, mapping, coverage, mutations}`, atomically; an exact retry is a no-op.
|
|
154
|
+
- `lineage(entity: str, key: dict) -> list[dict]` — the batches that wrote the row at `key`, oldest first.
|
|
155
|
+
- `counts() -> dict[str, int]` — rows per entity, `{entity: count}`.
|
|
156
|
+
|
|
157
|
+
No `upsert`: `get`, then `insert` or `update`. Read-only SQL: `gateway_env.db.one/all`; raw SQL writes are refused. The scaffolded `tools/handlers.py` docstring carries the same table.
|
|
158
|
+
|
|
138
159
|
### 6. Surface binding (only if the world declares it mirrors an MCP surface)
|
|
139
160
|
|
|
140
161
|
- [ ] Export a real connection's filtered `tools/list` through the MCP proxy under a grant, or inspect its `AgentSecurityResource` tool rows. Native Slack source capture has no live MCP catalog; its schema template tools are separate offline behavior.
|
|
@@ -142,6 +163,11 @@ deliverable can finish with synthetic fixtures and explicitly skipped live valid
|
|
|
142
163
|
- [ ] Compare read-only tools manually against a live connection only when such comparison is authorized and in scope.
|
|
143
164
|
- [ ] Native manifest changes bump `TOOL_VERSIONS`.
|
|
144
165
|
|
|
166
|
+
### 6b. Reader plan (only if the template fills from an active Merge connection)
|
|
167
|
+
|
|
168
|
+
- [ ] `readers/merge.toml` names `transport = "merge"`, the Merge `provider` slug (`google_drive`, not the template slug) and one `[[pulls]]` per logical tool (`list_files`, unprefixed — `MergeConnection` adds `google_drive__`); arguments, paging (`page = { cursor, next }`) and `max_pages` live here and nowhere else: a caller (CLI, MCP, route) can never pick a tool or pass arguments. `readers/merge.ingest.toml` maps each tool's records to entities with the `data ingest --map` grammar.
|
|
169
|
+
- [ ] Prove it against the fake Agent Handler (`scripts/worlds-e2e/fake-merge-agent-handler.mjs`, dataset per fixture) through `gateway worlds create <slug> --from-connection <connector>` and `gateway worlds data pull <slug> --connection <id> --mode append`; every tool the plan names must land rows (`unmatched` empty). A live pull needs an owner-authorized active connection and is recorded as such, not implied by the fake.
|
|
170
|
+
|
|
145
171
|
### 7. Ship the template; register platform integration only when in scope
|
|
146
172
|
|
|
147
173
|
- [ ] SDK-shipped: commit authored files under `gatewaysdk/sdk/src/gatewaysdk/environment/schema/connectors/<slug>/`; `list_templates()` and `test_schema_scaffold.py` cover the slug and preserve its authored dependencies/provenance during initialization. Schema-only templates declare an empty tools map and have no operation handlers.
|
|
@@ -167,7 +193,7 @@ When adding a bridge command to `schema/platform.py`, mirror it as a typed wrapp
|
|
|
167
193
|
`worldSchemaRuntime.ts` (`compileWorldFiles` / `applyWorldBatch` are the pattern: file map
|
|
168
194
|
in, `{entries, report, ...}` out, errors as `SchemaBridgeError`), cover it in
|
|
169
195
|
`tests/environment/test_schema_platform.py`, and let the agent tool call the wrapper, never
|
|
170
|
-
the subprocess. The bridge needs Python ≥ 3.
|
|
196
|
+
the subprocess. The bridge needs Python ≥ 3.12 (`tomllib`, and the store's sqlite `setconfig`); set `GATEWAY_SCHEMA_PYTHON`
|
|
171
197
|
when the default interpreter is older.
|
|
172
198
|
|
|
173
199
|
Use `worldSchemaRuntime.ts` for platform validation and `schema.platform` for
|
|
@@ -243,12 +269,19 @@ These source checks do not imply live-provider validation or executed fidelity s
|
|
|
243
269
|
|
|
244
270
|
A world can describe the vendor it mirrors in `connector.toml` at its root: base URL,
|
|
245
271
|
auth scheme (`header`, `bearer`, `basic`, `query`, `oauth2`, `none`) with the secret named by an
|
|
246
|
-
environment variable
|
|
272
|
+
environment variable (for `basic`, the credential is the Basic *user* and the password is ignored:
|
|
273
|
+
the host and `worlds serve` present `Basic b64("<key>:")`), capture pacing, a redaction policy (`hash` or `drop` fields at any
|
|
247
274
|
depth), and the operations the agent actually calls (method, path with `{args}`, query
|
|
248
275
|
params, and the entity plus results path their rows land in). The same file is meant to
|
|
249
276
|
drive the platform's custom source later, so describe once.
|
|
250
277
|
|
|
251
|
-
- `gateway worlds connector validate PATH` parses the spec
|
|
278
|
+
- `gateway worlds connector validate PATH` parses the spec, binds every operation's `entity`
|
|
279
|
+
(and each `explode` target) to the entities `schema/world.json` declares, and lists operations.
|
|
280
|
+
A route onto an undeclared entity is refused with `operation X lands on entity Y, which
|
|
281
|
+
schema/world.json does not declare (declared: …)` at `/operations/<i>/entity`; `schema
|
|
282
|
+
compile`, `schema check`, `worlds describe` and session open apply the same rule and write nothing
|
|
283
|
+
(it used to pass every gate, show under `served`, and 500 on the first call). A version stored
|
|
284
|
+
before this rule with such a route refuses to open instead; `worlds describe` names the fix.
|
|
252
285
|
- `gateway worlds connector capture PATH --op NAME --arg k=v --param k=v` calls the live
|
|
253
286
|
connection, follows the spec's cursor up to `max_pages`, and writes every page under
|
|
254
287
|
`captures/<op>/` plus `captures/manifest.json` (sha256 per file). The recorded request
|
|
@@ -329,6 +362,23 @@ no `--host`/`--remote`/`workspace:<slug>`, platform env stripped, pinned to its
|
|
|
329
362
|
so publishing always goes through the tools above. To parallelise several connectors, `spawn_subagent` with a brief
|
|
330
363
|
naming only the tools that child needs (`allowSubagents` on the session).
|
|
331
364
|
|
|
365
|
+
### From the CLI sandbox (no checkout, no Python)
|
|
366
|
+
|
|
367
|
+
`gateway worlds sandbox open <slug>@main` puts the same CLI in front of the world's source tree
|
|
368
|
+
server-side: `sandbox exec <id> 'gateway worlds schema entity add . <entity> --fields … -k id'`,
|
|
369
|
+
`'gateway worlds schema verifier add …'`, `'gateway worlds schema task add …'`, `'gateway worlds
|
|
370
|
+
schema compile . && gateway worlds schema check .'`, `'gateway worlds test .'` and `'gateway worlds
|
|
371
|
+
connector validate|conform .'` run on the uncommitted copy (`sqlite3` is on PATH too); `sandbox check`
|
|
372
|
+
runs the push gate and `sandbox commit` lands the version. `open` reports the staged CLI under `cli`
|
|
373
|
+
(`null` plus `warnings` when the image cannot host it — then only plain shell runs).
|
|
374
|
+
|
|
375
|
+
With an API key and no world yet, start the world itself there:
|
|
376
|
+
`gateway worlds sandbox open --new <slug> --from custom` (MCP `open_world_sandbox({ new, from })`)
|
|
377
|
+
opens on a world that does not exist yet, its tree the exact `schema init` scaffold (or
|
|
378
|
+
`--from <template>` for a shipped one); the same `sandbox exec` verbs run on that tree with the
|
|
379
|
+
staged CLI, `sandbox check` runs the push gate, and the first `sandbox commit -m …` creates the
|
|
380
|
+
world (first READY version on `main`). A taken slug is a 409 `world_exists`, never a rename.
|
|
381
|
+
|
|
332
382
|
### Serving the vendor's API from the world
|
|
333
383
|
|
|
334
384
|
A provider's client speaks the vendor's REST paths, not tool calls, so the world answers
|
|
@@ -68,7 +68,7 @@ gateway worlds schema init ./spycloud-world --connector custom --name spycloud-w
|
|
|
68
68
|
gateway worlds connector route add ./spycloud-world get_records_by_email \
|
|
69
69
|
--path /breach/data/emails/{email} --entity breach_records --results results \
|
|
70
70
|
--filter email=email
|
|
71
|
-
gateway worlds connector validate ./spycloud-world
|
|
71
|
+
gateway worlds connector validate ./spycloud-world # syntax + every route's entity exists in schema/world.json
|
|
72
72
|
gateway worlds connector capture ./spycloud-world --op get_records_by_email --arg email=probe@example.com \
|
|
73
73
|
--param severity[]=high --param severity[]=critical # a repeated name goes out twice; an undeclared one is sent, with a note
|
|
74
74
|
gateway worlds connector ingest ./spycloud-world --dry-run --report refusals.jsonl # every violation, nothing written
|
|
@@ -120,7 +120,7 @@ least one `[[operations]]` entry.
|
|
|
120
120
|
| `[capture]` | `requests_per_minute`, `timeout_seconds`, `max_pages`, `cursor_param`, `cursor_field`, `cursor_header` | Pacing between pages; providers ask us to be sparing. The next cursor is read from a body field or a response header (Torchlight sends `Cursor:`), never both, and is sent back as `cursor_param`; a cursor that is an absolute URL on the connection's host (DRF's `next`) is followed by its query. |
|
|
121
121
|
| `[redact]` | `hash`, `drop`, `round`, `preserve` | Applied before a body reaches disk. `preserve` hashes shape-preservingly (same length, digits stay digits, letters keep their case, separators stay; slug-salted, deterministic) so an account number still meets the field's format — `hash` made it a 71-char `sha256:` string the contract refused (Brex). `hash` writes `sha256:<hex>` salted with the slug so joins survive, element by element for arrays so the shape the contract expects survives too; `drop` removes; `round = { lat = 2, lon = 2 }` keeps numbers numeric but coarsens them. A bare name matches at any depth; a dotted path (`results.*.email`, `*` for list items) matches that place only, so a catalog's `assets` count map keeps its `email` key. Field names keep the vendor's casing. |
|
|
122
122
|
| `[api]` | `envelope`, `hits_key`, `cursor_key`, `page`, `limit_param`, `offset_param`, `cursor_param`, `page_size`, `previous_key`, `page_links`, `page_headers`, `cursor_absent_when_done`, `unauthorized`, `not_found`, `invalid`, `*_status`, `error_content_type`, `token_path`, `handlers`, `before` | `envelope = "results"` wraps rows as `{results, hits, cursor}` under the vendor's key names; empty means a bare array. `hits_key = ""` leaves the count out and `cursor_absent_when_done = true` leaves the cursor key out of the last page (instead of `""`), together the `{items, next_cursor?}` shape Brex-style vendors answer and clients test with `"next_cursor" in body`. `page` is `none`, `offset` or `cursor`, with the vendor's parameter names; `previous_key` adds the previous page and `page_links = true` renders next/previous as absolute URLs of the served host, the way Django REST Framework answers; `page_headers = { cursor = "Cursor", total = "Total-Count" }` sends the same values as response headers for vendors that page that way (Torchlight). Error bodies are JSON strings replayed verbatim with the vendor's statuses and content type (`application/problem+json` for RFC 7807 vendors); `""` (or `'null'`) means no body at all, for vendors that answer a bare 401. An oauth2 world mints bearers on `token_path` (default the spec's `token_url`) and honours only those. `handlers` names the world's Python module (default `api/handlers.py` when present); `before` names a gate in it that runs before every operation, declarative ones included, and returns `None` to let the request through or an answer (quota, allow-list, key state) to stop it. |
|
|
123
|
-
| `[[operations]]` | `name`, `method`, `path`, `params`, `entity`, `results` | `path` may carry `{args}`; a `params` name may end in `[]` (`expand[]`, `status[]`) for a key the vendor repeats, which capture then sends repeated and the served route filters on any value of; `results` is the dotted path to the row array (empty when the body is the row). An operation without `entity` never seeds and answers 501 `not_modelled`. |
|
|
123
|
+
| `[[operations]]` | `name`, `method`, `path`, `params`, `entity`, `results` | `path` may carry `{args}`; a `params` name may end in `[]` (`expand[]`, `status[]`) for a key the vendor repeats, which capture then sends repeated and the served route filters on any value of; `results` is the dotted path to the row array (empty when the body is the row). An operation without `entity` never seeds and answers 501 `not_modelled`. An `entity` (or an `explode` target) that `schema/world.json` does not declare is refused by `connector validate`, `schema compile`, `schema check` and the platform's describe (`gateway worlds describe`, local or platform) alike — `operation X lands on entity Y, which schema/world.json does not declare (declared: …)` at `/operations/<i>/entity` — and nothing is written; hand-edited TOML meets the same gate `route add` applies. |
|
|
124
124
|
| `[operations.ingest]` | `drop`, `explode`, `carry_args` | `drop` removes vendor fields the entity does not carry; each `[[operations.ingest.explode]]` (`field`, `entity`, `carry`, `take`) turns a nested array into child rows; `carry_args = { account_id = "id" }` stamps the capture's path argument or query parameter onto every row before projection (a statement only knows its account from the URL it was fetched under). `connector ingest --dry-run [--report FILE]` lists every violation of every capture and writes nothing. |
|
|
125
125
|
| `[operations.api]` | `filter`, `match`, `action`, `status`, `single`, `envelope`, `page`, `body`, `defaults`, `generate`, `handler`, `sample`, `sample_headers`, `sample_body`, `not_found`, `not_found_status`, `cursor_absent_when_done` | `filter` maps a path argument or declared query parameter to the row field it selects on (case-insensitive, any depth; an absent parameter does not narrow, one sent more than once matches rows carrying any of its values); `match = { name = "contains" }` switches an argument from equality to substring or `prefix`, and a list field matches when any element does. `action` is `read` (default), `create`, `update`, `delete` or `handler`: writes persist through the contract, `body` maps row fields (dotted for nested) to request-body paths, `defaults` fixes constants, `generate` fills `uuid`, `int` or `now`; `update` and `delete` find their row through `filter` and answer the vendor's 404 otherwise. `status` is the 2xx the vendor answers (201 for create, 204 for delete by default), `single` returns the row itself instead of an envelope, `envelope` and `page` override the world-wide setting per route. `handler` names a function `def name(ctx, args, query, body)` in the handlers module that returns a body, `(status, body)` or `(status, body, headers)`; `ctx` reads and writes the store (`rows`, `get`, `find`, `insert`, `update`, `delete`), keeps time (`now`, `elapsed`, `ladder` for status machines, frozen by `GATEWAY_WORLD_CLOCK`), mints ids (`uuid`) and shapes answers (`envelope`, `not_found`, `invalid`) and sees the request's `headers` and `query_all` (every value of every parameter, `{"expand[]": ["a", "b"]}`; the `query` argument keeps the first value only). `not_found` / `not_found_status` answer this route's missing row the vendor's own way (a 403 with its own message for one resource) and fall back to `[api]`; `cursor_absent_when_done` overrides `[api]` for this route. `sample = { ip = "203.0.113.5" }` gives the conformance run a value a validating route accepts when no seeded row can supply one; `sample_headers = { "Idempotency-Key" = "conform-1" }` and `sample_body = { reason = "conform" }` give it the headers and body a route demands (read by `connector conform` only; the served API never sends requests). |
|
|
126
126
|
|
|
@@ -142,11 +142,22 @@ its own (`uuid` for a string key, `int` for an integer one) unless `--body` maps
|
|
|
142
142
|
`--generate` names it, so a POST that leaves the id out is created rather than refused with
|
|
143
143
|
"Missing required property id". Composite keys are never minted.
|
|
144
144
|
|
|
145
|
-
Refusals: a duplicate or malformed name,
|
|
146
|
-
or a store action without `--entity`, a `--filter` key that is
|
|
147
|
-
nor a `--param`,
|
|
148
|
-
|
|
149
|
-
|
|
145
|
+
Refusals: a duplicate or malformed name (snake_case, at most 64 characters), an unknown method,
|
|
146
|
+
an undeclared entity, `--results` or a store action without `--entity`, a `--filter` key that is
|
|
147
|
+
neither a `{path}` argument nor a `--param`, a `{segment}` of the path that nothing binds (not
|
|
148
|
+
an entity field, not a `--filter`, not a `--param`; the refusal names the segment and the
|
|
149
|
+
`--param <segment>` that declares a scope argument), `action = "handler"` without `--handler`, a
|
|
150
|
+
status outside 2xx, a `--generate` kind other than `uuid`/`int`/`now`, `--default`/`--generate`
|
|
151
|
+
on a read route, or a file with no `[connector]` table. Source:
|
|
152
|
+
`gatewaysdk/sdk-ts/src/cli/runtime/scaffold.ts`; `schema compile` and `schema check` apply the
|
|
153
|
+
same binding rule to a hand-written `connector.toml` (`route_bindings.py`).
|
|
154
|
+
|
|
155
|
+
On a read route a declared `--param` that names a top-level field of the entity filters the
|
|
156
|
+
rows on its own (`GET /records?status=open` returns the open rows), the way a vendor's list
|
|
157
|
+
endpoint reads; `--filter` is for an argument whose name differs from the field's or that
|
|
158
|
+
selects at depth. The output of `route add` lists which arguments filter and which pass
|
|
159
|
+
through. A world built from a vendor document instead of by hand starts with
|
|
160
|
+
`gateway worlds schema init <dir> --from-openapi <spec>` ([world-schema](world-schema.md)).
|
|
150
161
|
|
|
151
162
|
## Writes and logic
|
|
152
163
|
|
|
@@ -209,6 +220,9 @@ its local runtime, with `GATEWAY_*`/`LANGFUSE_*` stripped and `--host`/`--remote
|
|
|
209
220
|
|
|
210
221
|
## Conformance and scenarios
|
|
211
222
|
|
|
223
|
+
Have the vendor's document and no world yet? `gateway worlds schema init <dir> --from-openapi
|
|
224
|
+
<spec>` writes the schema and the routes from it; `conform` is the check that comes after.
|
|
225
|
+
|
|
212
226
|
`gateway worlds connector conform <world> --openapi <vendor.yaml>` serves the world in-process,
|
|
213
227
|
drives every declared operation with arguments drawn from the world's own seeded rows (so lookups
|
|
214
228
|
hit data), and checks each answer against the vendor's OpenAPI: the documented 2xx status, and a
|