@withgateway/sdk 0.2.4 → 0.2.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. package/dist/cli.js +4304 -2381
  2. package/dist/cli.js.map +1 -1
  3. package/dist/cli.mjs +4266 -2343
  4. package/dist/cli.mjs.map +1 -1
  5. package/dist/files/index.js +56 -31
  6. package/dist/files/index.js.map +1 -1
  7. package/dist/files/index.mjs +52 -27
  8. package/dist/files/index.mjs.map +1 -1
  9. package/dist/hooks/index.d.mts +1 -1
  10. package/dist/hooks/index.d.ts +1 -1
  11. package/dist/hooks/index.js +280 -8
  12. package/dist/hooks/index.js.map +1 -1
  13. package/dist/hooks/index.mjs +272 -8
  14. package/dist/hooks/index.mjs.map +1 -1
  15. package/dist/{index-1xnxt0Cc.d.ts → index-BO2SNSgy.d.ts} +12 -1
  16. package/dist/{index-B0R3IgsV.d.mts → index-BaKu5ZSU.d.mts} +124 -2
  17. package/dist/{index-B0R3IgsV.d.ts → index-BaKu5ZSU.d.ts} +124 -2
  18. package/dist/{index-C1wDEwH4.d.mts → index-Cz8Q6Keo.d.mts} +12 -1
  19. package/dist/index.d.mts +4 -4
  20. package/dist/index.d.ts +4 -4
  21. package/dist/index.js +1100 -438
  22. package/dist/index.js.map +1 -1
  23. package/dist/index.mjs +1079 -424
  24. package/dist/index.mjs.map +1 -1
  25. package/dist/traces/index.js +45 -20
  26. package/dist/traces/index.js.map +1 -1
  27. package/dist/traces/index.mjs +41 -16
  28. package/dist/traces/index.mjs.map +1 -1
  29. package/dist/tracing/index.d.mts +1 -1
  30. package/dist/tracing/index.d.ts +1 -1
  31. package/dist/tracing/index.js +98 -61
  32. package/dist/tracing/index.js.map +1 -1
  33. package/dist/tracing/index.mjs +98 -61
  34. package/dist/tracing/index.mjs.map +1 -1
  35. package/dist/worlds/index.d.mts +179 -25
  36. package/dist/worlds/index.d.ts +179 -25
  37. package/dist/worlds/index.js +639 -284
  38. package/dist/worlds/index.js.map +1 -1
  39. package/dist/worlds/index.mjs +631 -276
  40. package/dist/worlds/index.mjs.map +1 -1
  41. package/package.json +2 -1
  42. package/runtime/gatewaysdk/environment/_bundle.py +15 -25
  43. package/runtime/gatewaysdk/environment/_materialize.py +53 -9
  44. package/runtime/gatewaysdk/environment/_toml.py +13 -0
  45. package/runtime/gatewaysdk/environment/schema/api.py +40 -25
  46. package/runtime/gatewaysdk/environment/schema/batch.py +15 -11
  47. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +24 -0
  48. package/runtime/gatewaysdk/environment/schema/compiler.py +60 -28
  49. package/runtime/gatewaysdk/environment/schema/conform.py +1 -1
  50. package/runtime/gatewaysdk/environment/schema/connector.py +1 -1
  51. package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.ingest.toml +25 -0
  52. package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.toml +17 -0
  53. package/runtime/gatewaysdk/environment/schema/data.py +21 -19
  54. package/runtime/gatewaysdk/environment/schema/extract.py +1 -5
  55. package/runtime/gatewaysdk/environment/schema/grading.py +225 -0
  56. package/runtime/gatewaysdk/environment/schema/host.py +156 -546
  57. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +1 -5
  58. package/runtime/gatewaysdk/environment/schema/ingest.py +3 -11
  59. package/runtime/gatewaysdk/environment/schema/local.py +32 -126
  60. package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
  61. package/runtime/gatewaysdk/environment/schema/platform.py +30 -26
  62. package/runtime/gatewaysdk/environment/schema/proof.py +237 -0
  63. package/runtime/gatewaysdk/environment/schema/redaction.py +1 -1
  64. package/runtime/gatewaysdk/environment/schema/runtime.py +686 -0
  65. package/runtime/gatewaysdk/environment/schema/scaffold.py +9 -2
  66. package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
  67. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +67 -9
  68. package/runtime/gatewaysdk/environment/schema/skills/hooks-getting-started/SKILL.md +67 -6
  69. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +4 -1
  70. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +11 -11
  71. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +17 -1
  72. package/runtime/gatewaysdk/environment/schema/snapshot.py +1 -5
  73. package/runtime/gatewaysdk/environment/schema/tools_world.py +71 -23
  74. package/runtime/gatewaysdk/environment/schema/ui.py +1 -5
  75. package/runtime/gatewaysdk/environment/schema/verbs.py +136 -0
  76. package/runtime/gatewaysdk/environment/schema/workers.py +14 -0
  77. package/runtime/gatewaysdk/environment/schema/world_tests.py +1 -1
  78. package/runtime/gatewaysdk/environment/world.py +507 -0
  79. package/skills/benchmarks-getting-started/SKILL.md +67 -9
  80. package/skills/connector-schema-authoring/connector-worlds.md +8 -1
  81. package/skills/connector-schema-authoring/world-schema.md +1 -1
  82. package/skills/hooks-getting-started/SKILL.md +67 -6
  83. package/skills/world-data-ingestion/SKILL.md +4 -1
  84. package/skills/worlds-getting-started/SKILL.md +11 -11
  85. package/skills/worlds-getting-started/references/cli-mcp-map.md +17 -1
@@ -81,6 +81,11 @@ TEMPLATE_CODE = {"handlers.py", "api/handlers.py"}
81
81
  #: Python (``api/handlers.py``), its served scenarios (``scenarios/…``) and the
82
82
  #: tests it ships for its tools (``tests/…``, run by ``gateway worlds test``).
83
83
  TEMPLATE_DIRECTORIES = {"api", "scenarios", "tests", "ui"}
84
+ #: Template subdirectories that stay with the connector package and never enter a
85
+ #: world: authoring evidence, fidelity reports, and ``readers/`` — how the platform
86
+ #: fills a world on this contract from a live connection (``readers/merge.toml``
87
+ #: names the tools to pull, ``readers/merge.ingest.toml`` maps their payloads).
88
+ CONNECTOR_ONLY_DIRECTORIES = {"evidence", "reports", "readers"}
84
89
 
85
90
 
86
91
  def _template_directory_files(directory: Path) -> list[Path]:
@@ -149,8 +154,8 @@ def init_bundle_from_files(
149
154
  target = destination / "connector.toml"
150
155
  elif path.startswith(("api/", "scenarios/", "tests/", "ui/")):
151
156
  target = destination / path
152
- elif path.startswith("evidence/") or path.startswith("reports/"):
153
- continue # authoring evidence stays with the connector, not the world
157
+ elif path.split("/", 1)[0] in CONNECTOR_ONLY_DIRECTORIES:
158
+ continue # evidence, reports and reader plans stay with the connector, not the world
154
159
  elif (
155
160
  Path(path).suffix in SUPPORTING_SUFFIXES and Path(path).name not in GENERATED_NAMES and "/" not in path
156
161
  ):
@@ -231,6 +236,8 @@ def init_bundle(
231
236
  if not (path.name == "tests" and connector == CUSTOM and not with_example_tool):
232
237
  code_files.extend(_template_directory_files(path))
233
238
  continue
239
+ if path.name in CONNECTOR_ONLY_DIRECTORIES and path.is_dir() and not path.is_symlink():
240
+ continue
234
241
  _resource(path)
235
242
  if path.name in {"world.json", "handlers.py"}:
236
243
  continue
@@ -32,6 +32,7 @@ NEXT_STEPS = {
32
32
  "world-data-ingestion": "gateway worlds data import <world> rows.json",
33
33
  "worlds-getting-started": "gateway worlds schema init ./my-world --connector custom",
34
34
  "benchmarks-getting-started": "gateway benchmarks init ./my-benchmark --connector custom",
35
+ "hooks-getting-started": "gateway hooks check gateway-hooks.toml",
35
36
  }
36
37
 
37
38
 
@@ -70,9 +70,9 @@ Expected output (JSON): `"kind": "harbor"`, `"tasks": ["hello"]`, the files, and
70
70
  code-bench/
71
71
  benchmark.json # kind, models, rollouts: the defaults `benchmarks run` reads
72
72
  pyproject.toml # [project] name + version: the slug and the version `push` records; not a Python package
73
- tasks/hello/task.toml # version, [metadata], [agent] timeout_sec, [verifier] timeout_sec, [environment] docker_image
73
+ tasks/hello/task.toml # version, [metadata], [agent] timeout_sec, [verifier] timeout_sec; docker_image only as a comment
74
74
  tasks/hello/instruction.md # what the agent is told
75
- tasks/hello/environment/Dockerfile # the task's container (FROM docker_image; WORKDIR /app)
75
+ tasks/hello/environment/Dockerfile # the task's container, built by the runner (FROM --image; WORKDIR /app): the ONE image source
76
76
  tasks/hello/tests/test.sh # the grader: runs after the agent, writes /logs/verifier/reward.json
77
77
  tasks/hello/solution/solve.sh # the reference solution; must score 1.0 (harbor's oracle agent runs it)
78
78
  ```
@@ -82,7 +82,40 @@ The reward contract is the only thing the platform reads: `tests/test.sh` writes
82
82
  other key a sub-reward — or a bare number to `/logs/verifier/reward.txt`. Any language, any method.
83
83
 
84
84
  Add a task: copy `tasks/hello` to `tasks/<name>`, edit `instruction.md`, `tests/test.sh`,
85
- `solution/solve.sh`, and the image (`docker_image` or the Dockerfile). Two tasks minimum.
85
+ `solution/solve.sh`, and the image. Two tasks minimum.
86
+
87
+ **One image source.** Harbor's rule (`should_use_prebuilt_docker_image`): when `task.toml` names
88
+ `[environment] docker_image`, the runner pulls that prebuilt image and runs it as it is —
89
+ `environment/Dockerfile` is **never built**, so anything it installs is absent on the runner (a tool the
90
+ agent needs, a pip package) while a Dockerfile-first local run passes. Without `docker_image` the
91
+ runner builds the Dockerfile. The scaffold therefore writes only the Dockerfile; put what the agent
92
+ needs in it. To run a prebuilt image instead, delete the Dockerfile and set `docker_image`.
93
+ `validate` fails a task that has both when the Dockerfile adds anything beyond `FROM`/`WORKDIR`
94
+ (a note when it does not), and `dry-run` follows the runner's choice (`image … (prebuilt: docker_image)`,
95
+ `pull`, no `build`).
96
+
97
+ **Secrets (an API key the task needs).** Declare it in the task's `environment/docker-compose.yaml`
98
+ — an overlay harbor applies on top of the image, prebuilt or built — and set it on the hosted
99
+ environment:
100
+
101
+ ```yaml
102
+ # tasks/q1/environment/docker-compose.yaml
103
+ services:
104
+ main:
105
+ environment:
106
+ EXA_API_KEY: ${EXA_API_KEY:?set with gateway bench secrets set}
107
+ ```
108
+
109
+ ```bash
110
+ gateway bench secrets set --env code-bench --key EXA_API_KEY # paste the value, then Ctrl-D; never on the command line
111
+ gateway bench secrets list --env code-bench
112
+ ```
113
+
114
+ `${NAME:?…}` (and a bare `${NAME}`) is a requirement; `${NAME:-default}` is not. `validate` lists a
115
+ task's required names as `secrets …`; `run` refuses before dispatch when one is not set on the
116
+ environment, naming the exact `bench secrets set` command. `gateway secret …` is a different store
117
+ (the platform's secret vault for agents and hooks) and does **not** feed hosted harbor runs.
118
+ `run_parameters.json` does not either. Locally, `dry-run` exports what your shell holds — see A3.
86
119
 
87
120
  MCP: no twin for a local scaffold; on an existing environment, `env_write_file({ containerId, path,
88
121
  newContent })` for each file, then `env_commit`.
@@ -97,15 +130,18 @@ Expected output:
97
130
 
98
131
  ```
99
132
  ./code-bench: harbor benchmark — runner reads it as harbor — ok
100
- ok hello: image python:3.11-slim; instruction yes; tests/test.sh yes; solution yes
133
+ ok hello: image build tasks/hello/environment/Dockerfile; instruction yes; tests/test.sh yes; solution yes
101
134
  benchmark ok (harbor): 1 task(s): hello; next: gateway benchmarks dry-run ./code-bench
102
135
  ```
103
136
 
104
- Per task: `task.toml` present, an image (`[environment] docker_image` or `environment/Dockerfile`),
105
- `instruction.md` non-empty, `tests/test.sh` present and writing the reward file, `solution/solve.sh`
106
- noted when absent. Tree-level: `pyproject.toml` (a note when absent) and that the runner's own
107
- dispatch rule (`detect_env_kind`) reads the tree as harbor — a `gateway-env.toml` at the root would
108
- make it a world to the runner. Exit 2 on any problem; notes never fail it.
137
+ Per task: `task.toml` present, ONE image source (`[environment] docker_image` → `image <ref> (prebuilt:
138
+ docker_image)`, else `environment/Dockerfile` → `image build …`; both is a problem when the Dockerfile
139
+ adds anything), the compose overlay and its `secrets` when the task has one, `instruction.md`
140
+ non-empty, `tests/test.sh` present and writing the reward file, `solution/solve.sh` noted when absent.
141
+ Tree-level: `pyproject.toml` (a note when absent) and that the runner's own dispatch rule
142
+ (`detect_env_kind`) reads the tree as harbor — a `gateway-env.toml` at the root would make it a world
143
+ to the runner. Exit 2 on any problem; notes never fail it. The closing line counts the tasks and names
144
+ the first five (`24 tasks: a, b, c, d, e, … (19 more)`); `--json` lists them all.
109
145
 
110
146
  ### A3. Dry-run one task
111
147
 
@@ -131,6 +167,16 @@ runner's `harbor run` line, the static check of `tests/test.sh`) followed by
131
167
  `not executed: … is not installed; execution needs Docker — the plan is what would run`, exit 0.
132
168
  `GATEWAY_DOCKER` names the docker binary.
133
169
 
170
+ The image is chosen by harbor's rule (A1): `docker_image` set → `docker pull`, the Dockerfile is not
171
+ built and the plan says so; otherwise `docker build`. A task with `environment/docker-compose.yaml`
172
+ runs through `docker compose` instead of `docker run` — harbor's base for `main` (the image, the
173
+ mounts, written to `<tmp>/docker-compose.base.yaml`) plus the task's file, then `compose down` — so
174
+ its extra services and `${NAME:?…}` secrets are exercised locally. The names must be exported in
175
+ your shell (`export EXA_API_KEY=…`); a missing one is reported as `not executed: the task's
176
+ docker-compose file requires EXA_API_KEY … export EXA_API_KEY here for the dry-run. Hosted runs read
177
+ it from … gateway bench secrets set --env code-bench --key EXA_API_KEY`, exit 0, nothing spawned.
178
+ Values never appear in the printed argv.
179
+
134
180
  ### A4. Push
135
181
 
136
182
  ```bash
@@ -146,6 +192,18 @@ tree under a world's slug (exit 2, nothing sent); the hub's versions door refuse
146
192
  `benchmark_substrate_mismatch`. Later pushes are new content-hashed versions (`gateway bench log`,
147
193
  `diff`, `branch`, `tag`).
148
194
 
195
+ **Which slug.** stderr says where it came from: `slug code-bench (from benchmark.json)`. Order:
196
+ `--slug` > the dir's `.gateway/.env-metadata.json` (written by every push and pull, so a dir keeps
197
+ pushing to the slug it became) > `benchmark.json` > `pyproject.toml [project].name` > the directory
198
+ name. A tracked slug that disagrees with what the tree declares is refused with exit 2 and nothing
199
+ sent — `./vendor tracks slug websearch-bench-e2e (.gateway/.env-metadata.json, from an earlier push)
200
+ but its benchmark.json says websearch-bench` — because that is how one `--slug X` push made every
201
+ later bare push land on X. Settle it with `--slug <the one you mean>` (the dir retracks) or delete the
202
+ metadata file. The platform's collision suffix (`code-bench-2` minted for a taken `code-bench`) is
203
+ not a disagreement. A 409 `Branch "main" has moved` (another push landed on the slug since this dir's
204
+ base) names its way out: `gateway benchmarks push ./code-bench --force` moves the branch anyway;
205
+ `gateway bench pull code-bench` into a fresh dir builds on the tip.
206
+
149
207
  MCP: `create_environment_version` (or `env_write_file` + `env_commit`).
150
208
 
151
209
  ### A5. Run: models × tasks × rollouts
@@ -76,8 +76,10 @@ Kinds: `person_name`, `email`, `username`, `phone`, `company`, `domain`, `ip`, `
76
76
  `machine`, `log`; anything else gets a `REF-nnnn` stand-in. `identity` ties attributes to one
77
77
  person so they stay consistent; `role: "public"` is never replaced. Every variant of a value is
78
78
  tracked (bare surname, handle spellings, `(at)` emails, phone formats, percent-encoding, a
79
- world's `sha256:` stored form); what slips through is counted (`leak_through_inbound`), and a
80
- real value in the outgoing report fails the run (`leaks_outbound`).
79
+ world's `sha256:` stored form), and in any case: `Alexr` or `ALEXR` for a seeded `alexr` is swapped
80
+ where it stands alone, and the stand-in keeps the case it arrived in. The door swaps and never
81
+ blocks. What slips through is counted (`leak_through_inbound`), and a real value in the outgoing
82
+ report fails the run (`leaks_outbound`).
81
83
 
82
84
  ## Identities that arrive, not seeded
83
85
 
@@ -127,12 +129,71 @@ spelling back in a JSON or text file; a `[redacted …]` token stays — redacti
127
129
  value, and its raw copy is a `record` sink placed before the `redact` rule. The legend maps
128
130
  personas to people: keep it with the run key, out of version control, never beside world data.
129
131
 
132
+ ## Did anything real get through? (the audit)
133
+
134
+ The door's own meters only count values it knew about, so they read zero for anything it never
135
+ recognised. After a run, the audit checks from the other side: it compares what your tools
136
+ **returned** with what the model was **shown**, and lists every real identifier that survived.
137
+
138
+ Write both sides with two `record` rules: one beside the inbound `alias` rule (raw), one after it
139
+ (shown). The outbound `alias` rule stays as you have it, so the tool is called with real values.
140
+
141
+ ```toml
142
+ [[hooks.pre_tool_call]]
143
+ matcher = "*"
144
+ handlers = [{ type = "alias" }]
145
+
146
+ [[hooks.post_tool_call]]
147
+ matcher = "*"
148
+ handlers = [{ type = "record", sink = "{run_dir}/raw.jsonl" }, { type = "alias" }]
149
+
150
+ [[hooks.post_tool_call]]
151
+ matcher = "*"
152
+ handlers = [{ type = "record", sink = "{run_dir}/shown.jsonl" }]
153
+ ```
154
+
155
+ ```python
156
+ from gatewaysdk.hooks import RedactionAudit
157
+
158
+ audit = RedactionAudit.for_run(
159
+ legend=run.alias.legend(), # your subjects and found personas
160
+ field_kinds={"*": {"full_name": "person_name", "email": "email", "handle": "username"}},
161
+ )
162
+ report = audit.run(raw_records, shown_records) # lists of the JSONL lines
163
+ report.ok # False: something real reached the model
164
+ report.to_dict() # findings, census, detectors, coverage
165
+ report.review_sample(10, seed=1) # unflagged text for a person to read
166
+ ```
167
+
168
+ TypeScript is the same: `RedactionAudit.forRun({ legend, fieldKinds }).run(raw, shown)`.
169
+
170
+ From the command line, when the run directory holds `raw.jsonl`, `shown.jsonl` and `legend.json`:
171
+
172
+ ```bash
173
+ gateway hooks audit runs/<run id>/ --field-kinds fields.json --sample 10 # exit 1 when anything leaked
174
+ gateway hooks audit runs/<run id>/ --json # the same report, for a script
175
+ ```
176
+
177
+ `fields.json` is `{"*": {"full_name": "person_name", "email": "email"}}`, or keyed by tool name.
178
+
179
+ It knows a value is real from three places, strongest first: your legend; every value of a field
180
+ whose kind you declared (a name in a vendor row that reappears in a web snippet is caught for
181
+ certain); and what a pattern finds (`email`, `phone`, `ip`). A finding gives the kind, the tool, the
182
+ record and a 3-character prefix, never the value.
183
+
184
+ Read `ok` as "no **detected** leaks with these detectors", not as proof: a name that is in no list
185
+ and matches no pattern is invisible to it. That is what `review_sample` is for. When you find a
186
+ miss, add it to the task's `entities` or `redact` values and run again; the raw file is real data,
187
+ keep it with the legend.
188
+
130
189
  ## Wiring
131
190
 
132
- - **Agent SDK** — `guarded = run.wrap({ lookup_email: lookupEmail })` and register the wrapped
133
- function wherever the SDK takes a tool implementation; `run.tools()` materializes the file's
134
- `[tools.*]` (a keyed web search, a connector operation) already wrapped;
135
- `verdict = run.end(report)` runs `run_end`.
191
+ - **Agent SDK** — open the file (`from gatewaysdk.hooks import HookRun; run = HookRun("gateway-hooks.toml")`
192
+ / `import { HookRun } from "@withgateway/sdk/hooks"; const run = await HookRun.fromConfig("gateway-hooks.toml")`),
193
+ then wrap the tools: Python `guarded = run.wrap({"lookup_email": lookup_email})`, TypeScript
194
+ `guarded = run.wrapTools({ lookup_email: lookupEmail })`; register the wrapped function wherever
195
+ the SDK takes a tool implementation. `run.tools()` materializes the file's `[tools.*]` (a keyed
196
+ web search, a connector operation) already wrapped; `verdict = run.end(report)` runs `run_end`.
136
197
  - **Claude Code / coding agents** — `.claude/settings.json`: `PreToolUse`, `PostToolUse` and
137
198
  `Stop` entries whose command is `gateway hooks run`. State (run key + seeds) persists under
138
199
  `--state`, `$GATEWAY_HOOKS_STATE`, else `.gateway/hooks/<session id>`. A coding agent shows the
@@ -208,7 +208,10 @@ gateway worlds describe <WORLD>; gateway worlds data query <WORLD> <entity> --li
208
208
  **The drive.** Every command here takes `drive:<path>` where it takes a local file: `gateway files
209
209
  upload rows.json --to seeds/rows.json` then `gateway worlds data import <WORLD> drive:seeds/rows.json`
210
210
  is the route from a laptop (or an agent's `request_file_upload`) to a world with nothing staged
211
- locally; the report keeps the drive path under `inputs`. `--to <path>` on `calls pull`, `extract`
211
+ locally: the platform reads the drive file itself (`.json`, `.jsonl`, `.ndjson`, optionally `.gz`;
212
+ convert CSV first) and stages it as one chunk — nothing comes down to your machine and nothing is
213
+ re-uploaded, so `drive:` needs a world slug and no `--dry-run`. The report keeps the drive path under
214
+ `inputs` and the manifest records the digest, size and rows the platform derived. `--to <path>` on `calls pull`, `extract`
212
215
  and `session export` lands the output in the drive (`--from drive:<path>` on `calls pull` and
213
216
  `ingest` reads it back). `gateway traces export drive:<path> --observations --type TOOL` puts a
214
217
  trace export there for the same route. `gateway files --help` lists the drive commands.
@@ -132,7 +132,7 @@ definitions and handler bodies are hand-written:
132
132
 
133
133
  ```bash
134
134
  gateway worlds schema entity add ./x crm_accounts --fields "id:string,name:string,tier:string?" --key id
135
- gateway worlds schema entity add ./x crm_contacts --ref CrmContact --key id --relationship account_id->crm_accounts
135
+ gateway worlds schema entity add ./x crm_contacts --ref CrmContact --key id --relationship 'account_id->crm_accounts' # quote it: a bare > is a shell redirect
136
136
  gateway worlds connector route add ./x list_accounts --path /v1/accounts --param page --entity crm_accounts --results data
137
137
  gateway worlds connector route add ./x get_account --path /v1/accounts/{id} --entity crm_accounts --filter id=id --single
138
138
  gateway worlds connector route add ./x create_account -X POST --path /v1/accounts --entity crm_accounts --action create --body name=name
@@ -140,8 +140,8 @@ gateway worlds connector route add ./x create_account -X POST --path /v1/account
140
140
 
141
141
  Each refuses (and writes nothing) on a duplicate name, an undeclared entity or `$defs`
142
142
  reference, a key that is not a required string/integer field, or a filter on an unknown
143
- argument. A custom world's first `route add` creates `connector.toml`; `schema init` prints `kind: "schema"`, `api` and
144
- the `next` commands in order (custom: entity add → route add → task add → compile → check → import → serve → `worlds create`).
143
+ argument. A create route mints its single-field primary key (`uuid` for strings, `int` for integers) unless `--body` maps it;
144
+ `--generate field=uuid|int|now` and `--default field=value` fill the rest. A custom world's first `route add` creates `connector.toml`; `schema init` prints `kind: "schema"`, `api` and the `next` commands in order (custom: entity add → route add → task add → compile → check → import → serve → `worlds create`).
145
145
  Manifest `kind` is `schema` (older trees: `mock-tools`). Handlers: `def name(ctx, args, query, body)`; `ctx` reads and writes the store and keeps the clock.
146
146
 
147
147
  MCP: `get_connector_authoring_skill({})` returns the authoring skill and conventions.
@@ -446,10 +446,13 @@ gateway bench fork x --into x-experiment # a new container sh
446
446
  gateway bench diff <base-version-id> <head-version-id> # per-file diff
447
447
  gateway bench propose ./x --title t -m why # a change proposal instead of a push
448
448
  gateway bench log x
449
+ gateway worlds sandbox open x@main # no local checkout: edit the source tree server-side,
450
+ gateway worlds sandbox write|exec|check|commit|close <id> # nothing on disk; `bench sandbox` is the same set
449
451
  ```
450
452
 
451
453
  `worlds run --path` on a clean `bench pull` checkout files runs pinned to the base version;
452
454
  a drifted directory runs unfiled. Push a new version when the change is meant to stick.
455
+ Sandbox twins (`open_world_sandbox` … `close_world_sandbox`): `references/cli-mcp-map.md`.
453
456
 
454
457
  MCP: `env_diff({ containerId })`, `env_propose({ containerId, ... })`,
455
458
  `env_list_proposals`, `env_get_proposal`, `env_apply_proposal`, `env_dismiss_proposal`,
@@ -462,9 +465,8 @@ tasks): `references/ui-map.md`.
462
465
 
463
466
  ## CLI to MCP map
464
467
 
465
- Every CLI step with its MCP twin, the steps that have none, and how the platform (console)
466
- agent reaches the same surface (`run_gateway_cli` for the local rows, `spawn_subagent` for
467
- one world per child): `references/cli-mcp-map.md`.
468
+ Every CLI step with its MCP twin, the steps that have none, and how the platform (console) agent
469
+ reaches the same surface (`run_gateway_cli`, `spawn_subagent`): `references/cli-mcp-map.md`.
468
470
 
469
471
  ## Gotchas
470
472
 
@@ -489,12 +491,10 @@ one world per child): `references/cli-mcp-map.md`.
489
491
  ## Related
490
492
 
491
493
  - `benchmarks-getting-started`: a benchmark is a world with tasks — init, check, push, run (models × tasks × rollouts), results.
492
- - `world-data-ingestion`: rows in, calls out.
493
- - `connector-schema-authoring`: the contract, fidelity shapes, conventions, definition of done.
494
+ - `world-data-ingestion`: rows in, calls out. `connector-schema-authoring`: the contract, fidelity shapes, definition of done.
494
495
  - `documentation/features/connector-worlds.md`: `connector.toml` in full, capture, conform, serving.
495
496
  - `documentation/features/workspace-connectors.md`: the connector object, sharing, MCP surface.
496
497
  - `documentation/features/world-schema.md`: the `gateway-world/1` contract and store API.
497
- - Outside the repository, `connector-worlds.md` and `world-schema.md` ship with the npm package
498
- beside the authoring skill: `gateway worlds skill connector-schema-authoring --install .` lands
499
- them under `.claude/skills/connector-schema-authoring/`.
498
+ - Outside the repository, `connector-worlds.md` and `world-schema.md` ship with the npm package beside
499
+ the authoring skill: `gateway worlds skill connector-schema-authoring --install .` installs them.
500
500
  - Consumer pages: `gateway_documentation/pages/worlds/{getting-started,data,custom-connections,slack}.mdx`.
@@ -27,16 +27,32 @@ Reference for `worlds-getting-started`: every CLI step and its MCP twin, then ho
27
27
  | `files connectors list\|sync <id>` | `list_file_connectors` / `sync_file_connector` (`GET\|POST /api/public/files/connectors[/sync]`) |
28
28
  | `traces list\|get\|export [--observations]` | `get_trace`, `get_observations` (`GET /api/public/traces[/{id}]`, `/observations`) |
29
29
  | `traces sessions list\|get [--io]` | `get_session` (`GET /api/public/sessions[/{id}?includeIO=true]`) |
30
- | `worlds data … drive:<path>` / `--to <path>` (any data command) | `get_file` + `import_world_data` (the CLI downloads, then runs the command as on a local file) |
30
+ | `worlds data import <slug> drive:<path>` | `POST /api/public/worlds/{slug}/data/batches/{id}/chunks` with `{ordinal, drive}` (the platform stages the drive file itself; nothing is downloaded or re-uploaded) |
31
+ | `worlds data … drive:<path>` / `--to <path>` (other data commands) | `get_file` + the command (the CLI downloads, then runs the command as on a local file) |
31
32
  | `environment run` / `worlds run` | `run_evaluation` |
32
33
  | `environment link-task-set` / `create-task` | `create_task_set`, `create_task`, `link_task_set` |
33
34
  | `bench diff` / `propose` | `env_diff`, `env_propose` |
35
+ | `worlds\|bench sandbox open\|status\|read\|write\|exec\|check\|commit\|close` | `open_world_sandbox`, `get_world_sandbox`, `read_world_sandbox_file`, `write_world_sandbox_file` (base64), `exec_in_world_sandbox`, `check_world_sandbox`, `commit_world_sandbox`, `close_world_sandbox` (`/api/public/benchmark-containers/sandboxes[/{id}[/file\|exec\|check\|commit]]`; the source tree only — no data snapshot, nothing on the caller's disk; `commit` lands on the recorded base) |
34
36
  | `worlds skill <name>` / `worlds connector skill` | `get_connector_authoring_skill`, `guide` |
35
37
  | `worlds connector capture --remote` / poll + download | `start_connector_capture`, `get_connector_capture` (platform agent only) |
36
38
 
37
39
  Steps with no MCP tool: opening a session (`worlds session open`) — use the CLI or the public
38
40
  sessions route.
39
41
 
42
+ ## Size and time limits (one source: `packages/shared/src/world/limits.ts`)
43
+
44
+ Every door enforces the same number and names it in the refusal; the CLI checks the file
45
+ before it makes a request.
46
+
47
+ | Door | Limit | Over it |
48
+ |---|---|---|
49
+ | `sandbox write`, `write_world_sandbox_file`, `env_write_file`, `propose_task_fix` | 256 KB per file | refused, naming the limit; rows → `worlds data import <slug> <file>`; a big source file → `gateway files upload`, then fetch it inside the sandbox with `sandbox exec`; the whole tree → `bench push` |
50
+ | `sandbox exec --timeout`, `exec_in_world_sandbox timeoutMs` | default 60 s, at most 600 s | refused, not clamped |
51
+ | Sandbox tree at `check`/`commit`, `bench push` unpacked | 128 MB of source | refused, naming the limit; move data to `worlds data import`, delete scratch files |
52
+ | `bench push` / `worlds create --from` compressed bundle | 64 MB (16 MB from the browser) | refused, naming the limit |
53
+ | A single manifest / metadata file in a bundle | 1 MB | refused |
54
+ | `sandbox write` HTTP body | 1 MB (framework cap; only reachable by a body no schema would accept) | 413 |
55
+
40
56
  ## The platform agent
41
57
 
42
58
  The platform (console) agent has every MCP tool in this table as an agent tool of the
@@ -44,11 +44,7 @@ from datetime import datetime, timezone
44
44
  from pathlib import Path
45
45
  from typing import Any, Iterable
46
46
 
47
- try:
48
- import tomllib
49
- except ModuleNotFoundError: # pragma: no cover - python 3.10
50
- import tomli as tomllib
51
-
47
+ from .._toml import tomllib
52
48
  from .compiler import Contract
53
49
  from .store import SQLiteStore
54
50
  from .validation import ContractError, canonical, fail, read_json
@@ -47,10 +47,12 @@ from dataclasses import dataclass, field
47
47
  from pathlib import Path
48
48
  from typing import Any
49
49
 
50
+ from .._toml import tomllib
50
51
  from .. import _bundle
51
52
  from .._materialize import materialize_template
52
53
  from .._world import World
53
54
  from ..errors import WorldStartError
55
+ from ..world import World as WorldDetector
54
56
  from . import snapshot as snapshots
55
57
  from .api import ApiSettings, WorldState, load_api_settings
56
58
  from .api import load_handlers as load_api_handlers
@@ -58,6 +60,7 @@ from .compiler import ContractError, bundle_mode, check_bundle, fail
58
60
  from .connector import SPEC, Spec, load_spec
59
61
  from .data import seed_batch
60
62
  from .dependencies import Dependency, parse_dependencies
63
+ from .grading import world_state
61
64
  from .tasks import TaskSpec
62
65
 
63
66
  Handler = Callable[..., Any]
@@ -84,6 +87,12 @@ class ToolsTask:
84
87
  #: verifiers; ``None`` (a task that names no worlds) grades with all of every
85
88
  #: dependency's, the way it grades with all of the root's.
86
89
  worlds: dict[str, tuple[str, ...]] | None = None
90
+ #: An authored ``scenarios/<id>/`` task: its seed overlay replaces whole
91
+ #: entities of the template at open and reset (None: the template alone).
92
+ scenario: dict[str, list] | None = None
93
+ #: The scenario's ``checks/verify.py``, when it grades by one instead of
94
+ #: the world's SQL verifiers (the runtime's :class:`TaskGrader` runs it).
95
+ verify: Path | None = None
87
96
 
88
97
  def selected(self, all_verifiers: list[str]) -> list[str]:
89
98
  """The verifiers this task grades with: its own list, else the
@@ -106,19 +115,12 @@ INLINE_TASK_ID = "session"
106
115
 
107
116
 
108
117
  def inline_task(spec: TaskSpec) -> ToolsTask:
109
- """A session task (:mod:`tasks`) as a mock-tools task: the spec's
110
- instruction as the prompt, graded by every verifier the bundle ships. The
111
- spec's grader vocabulary (assertions, python, rubric) is an api world's —
112
- a mock-tools world's checks are its ``verifiers/*.sql``, and a task cannot
113
- bring others — so a task that carries a grader is refused rather than
114
- graded by something the world never had. The seed is applied by
115
- :class:`ToolsWorld` once the template copy exists."""
116
- if spec.grader is not None:
117
- fail(
118
- "task",
119
- "a mock-tools world grades with its bundled verifiers; give the task no grader, or open an api world",
120
- "/task/grader",
121
- )
118
+ """A session task (:mod:`tasks`) as a schema-world task: the spec's
119
+ instruction as the prompt. Without a grader of its own the task is graded
120
+ by every verifier the bundle ships (:meth:`ToolsWorld.grade`); with one
121
+ (assertions, python, rubric) the host grades it the way it grades an api
122
+ world's session task, over :meth:`ToolsWorld.grader_state`. The seed is
123
+ applied by :class:`ToolsWorld` once the template copy exists."""
122
124
  return ToolsTask(id=spec.name or INLINE_TASK_ID, prompt=spec.instruction, metadata=dict(spec.metadata))
123
125
 
124
126
 
@@ -218,7 +220,9 @@ def served_with_dependencies(version: ToolsVersion, dependencies: dict[str, Depe
218
220
 
219
221
 
220
222
  def is_tools_bundle(bundle: Path) -> bool:
221
- return (bundle / "gateway-env.toml").is_file() and not (bundle / "gateway-api-world.toml").is_file()
223
+ """The tree runs on the tools engine (a schema or legacy tools world stored
224
+ as itself; an api bundle does not)."""
225
+ return WorldDetector.from_dir(bundle).tools_engine
222
226
 
223
227
 
224
228
  def verifier_names(bundle: Path) -> list[str]:
@@ -263,14 +267,37 @@ def _tasks_from_rows(rows: list) -> list[ToolsTask]:
263
267
  return out
264
268
 
265
269
 
270
+ def _scenario_task(name: str, scenario: Path) -> ToolsTask:
271
+ """A task from an authored scenario dir (what an api bundle carried and
272
+ ``worlds migrate`` keeps): ``instruction.md`` or the toml's ``instruction``
273
+ is the prompt, ``seed/*.json`` the overlay, ``checks/verify.py`` the grader."""
274
+ table = tomllib.loads((scenario / "scenario.toml").read_text(encoding="utf-8"))
275
+ instruction = scenario / "instruction.md"
276
+ prompt = instruction.read_text(encoding="utf-8") if instruction.is_file() else str(table.get("instruction") or "")
277
+ verify = scenario / "checks" / "verify.py"
278
+ return ToolsTask(
279
+ id=name,
280
+ prompt=prompt.strip() or f"Drive the world into the state the scenario '{name}' checks.",
281
+ metadata={"scenario_id": name},
282
+ scenario=snapshots.scenario_overlay(scenario) or None,
283
+ verify=verify if verify.is_file() else None,
284
+ )
285
+
286
+
287
+ def scenario_tasks(bundle: Path) -> list[ToolsTask]:
288
+ """One task per ``scenarios/<name>/scenario.toml`` of the tree."""
289
+ return [_scenario_task(name, path) for name, path in snapshots.scenario_dirs(bundle).items()]
290
+
291
+
266
292
  def load_tasks(
267
293
  bundle: Path, manifest: _bundle.EnvManifest, verifiers: list[str], *, required: bool = True
268
294
  ) -> dict[str, ToolsTask]:
269
295
  """The bundle's tasks, in the runner's precedence: ``[[tasks]]`` in the
270
296
  manifest, else ``tasks.jsonl`` / ``tasks.json``, else one task per
271
- verifier. A bundle with neither has nothing to grade and is refused when
272
- ``required`` (the host opens it to grade); a local session that only calls
273
- tools opens it with none."""
297
+ verifier; plus one per authored ``scenarios/<name>/`` the declared tasks
298
+ do not already name. A bundle with none has nothing to grade and is
299
+ refused when ``required`` (the host opens it to grade); a local session
300
+ that only calls tools opens it with none."""
274
301
  declared = manifest.raw.get("tasks")
275
302
  tasks = _tasks_from_rows(declared) if isinstance(declared, list) else []
276
303
  if not tasks:
@@ -296,16 +323,18 @@ def load_tasks(
296
323
  )
297
324
  for name in verifiers
298
325
  ]
299
- if not tasks and required:
300
- fail(
301
- "host",
302
- "mock-tools bundle declares no tasks and ships no verifiers/*.sql: nothing to grade",
303
- )
304
326
  by_id: dict[str, ToolsTask] = {}
305
327
  for task in tasks:
306
328
  if task.id in by_id:
307
329
  fail("host", f"task id '{task.id}' is declared twice")
308
330
  by_id[task.id] = task
331
+ for task in scenario_tasks(bundle):
332
+ by_id.setdefault(task.id, task)
333
+ if not by_id and required:
334
+ fail(
335
+ "host",
336
+ "mock-tools bundle declares no tasks and ships no verifiers/*.sql: nothing to grade",
337
+ )
309
338
  return by_id
310
339
 
311
340
 
@@ -459,7 +488,10 @@ class ToolsWorld:
459
488
 
460
489
  def _apply_task_seed(self) -> None:
461
490
  """Start the session in its task's state: the template with the task's
462
- rows on top. A task without a seed starts on the template alone."""
491
+ rows on top — a scenario task's overlay replacing whole entities, then
492
+ the session task's seed. A task without either starts on the template."""
493
+ if self.task.scenario is not None:
494
+ self.seed({"rows": self.task.scenario, "mode": "replace"})
463
495
  if self._task_seed is not None:
464
496
  self.seed({"rows": self._task_seed["rows"], "mode": self._task_seed["mode"]})
465
497
 
@@ -606,6 +638,10 @@ class ToolsWorld:
606
638
  counts[f"{prefix}.{entity}" if prefix else entity] = len(store.list(entity))
607
639
  return {"seeded": seeded, "mode": mode, "entity_counts": counts}
608
640
 
641
+ @property
642
+ def session_dir(self) -> Path:
643
+ return self._dir
644
+
609
645
  def state(self) -> dict:
610
646
  """The root's rows by entity, and under ``worlds`` each dependency's."""
611
647
  out: dict[str, Any] = {"entities": _entities_of(self.world)}
@@ -613,6 +649,18 @@ class ToolsWorld:
613
649
  out["worlds"] = {alias: {"entities": _entities_of(world)} for alias, world in self.worlds.items()}
614
650
  return out
615
651
 
652
+ def entity_names(self) -> list[str]:
653
+ """Every entity name :meth:`grader_state` reports, rows or not."""
654
+ names = list(self.world.contract.entities) if self.world.contract is not None else []
655
+ for alias, world in self.worlds.items():
656
+ if world.contract is not None:
657
+ names.extend(f"{alias}.{entity}" for entity in world.contract.entities)
658
+ return names
659
+
660
+ def grader_state(self) -> dict:
661
+ """The rows as a session task's grader reads them (:func:`world_state`)."""
662
+ return world_state(self.world, self.worlds)
663
+
616
664
  def close(self) -> None:
617
665
  world = getattr(self, "world", None)
618
666
  if world is not None:
@@ -32,11 +32,7 @@ from dataclasses import dataclass
32
32
  from pathlib import Path
33
33
  from typing import Any
34
34
 
35
- try:
36
- import tomllib
37
- except ModuleNotFoundError: # pragma: no cover - python < 3.11
38
- import tomli as tomllib # type: ignore[no-redef]
39
-
35
+ from .._toml import tomllib
40
36
  from .validation import fail
41
37
 
42
38
  STATIC_SOURCE = Path(__file__).with_name("ui_static")