@withgateway/sdk 0.3.0 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/README.md +6 -1
  2. package/dist/cli.js +3971 -2338
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +3950 -2318
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +214 -41
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +206 -35
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/{index-DAb-FCMg.d.ts → index-DUHmS74p.d.ts} +3 -13
  13. package/dist/{index-D6oi1TWE.d.mts → index-mJhG3vZU.d.mts} +3 -13
  14. package/dist/index.d.mts +4 -4
  15. package/dist/index.d.ts +4 -4
  16. package/dist/index.js +1456 -564
  17. package/dist/index.js.map +1 -1
  18. package/dist/index.mjs +1433 -557
  19. package/dist/index.mjs.map +1 -1
  20. package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
  21. package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
  22. package/dist/traces/index.d.mts +1 -1
  23. package/dist/traces/index.d.ts +1 -1
  24. package/dist/traces/index.js +211 -38
  25. package/dist/traces/index.js.map +1 -1
  26. package/dist/traces/index.mjs +203 -32
  27. package/dist/traces/index.mjs.map +1 -1
  28. package/dist/tracing/index.d.mts +2 -2
  29. package/dist/tracing/index.d.ts +2 -2
  30. package/dist/tracing/index.js +96 -21
  31. package/dist/tracing/index.js.map +1 -1
  32. package/dist/tracing/index.mjs +96 -21
  33. package/dist/tracing/index.mjs.map +1 -1
  34. package/dist/worlds/index.d.mts +300 -123
  35. package/dist/worlds/index.d.ts +300 -123
  36. package/dist/worlds/index.js +1385 -508
  37. package/dist/worlds/index.js.map +1 -1
  38. package/dist/worlds/index.mjs +1377 -501
  39. package/dist/worlds/index.mjs.map +1 -1
  40. package/package.json +1 -1
  41. package/runtime/gatewaysdk/environment/_bundle.py +2 -27
  42. package/runtime/gatewaysdk/environment/_materialize.py +49 -69
  43. package/runtime/gatewaysdk/environment/_world.py +122 -197
  44. package/runtime/gatewaysdk/environment/backing.py +137 -0
  45. package/runtime/gatewaysdk/environment/core.py +1 -1
  46. package/runtime/gatewaysdk/environment/legacy_sqlite.py +301 -0
  47. package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
  48. package/runtime/gatewaysdk/environment/runtime.py +8 -13
  49. package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
  50. package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
  51. package/runtime/gatewaysdk/environment/schema/api.py +266 -515
  52. package/runtime/gatewaysdk/environment/schema/batch.py +42 -37
  53. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
  54. package/runtime/gatewaysdk/environment/schema/call_log.py +291 -0
  55. package/runtime/gatewaysdk/environment/schema/call_redaction.py +344 -0
  56. package/runtime/gatewaysdk/environment/schema/cast.py +31 -4
  57. package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
  58. package/runtime/gatewaysdk/environment/schema/conform.py +8 -12
  59. package/runtime/gatewaysdk/environment/schema/connector.py +27 -62
  60. package/runtime/gatewaysdk/environment/schema/data.py +50 -207
  61. package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
  62. package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
  63. package/runtime/gatewaysdk/environment/schema/database/base.py +995 -0
  64. package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
  65. package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
  66. package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
  67. package/runtime/gatewaysdk/environment/schema/database/guard.py +208 -0
  68. package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
  69. package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
  70. package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
  71. package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
  72. package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
  73. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +367 -0
  74. package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
  75. package/runtime/gatewaysdk/environment/schema/envelope.py +169 -0
  76. package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
  77. package/runtime/gatewaysdk/environment/schema/grading.py +45 -30
  78. package/runtime/gatewaysdk/environment/schema/handler_errors.py +142 -12
  79. package/runtime/gatewaysdk/environment/schema/host.py +158 -250
  80. package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
  81. package/runtime/gatewaysdk/environment/schema/local.py +44 -71
  82. package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
  83. package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
  84. package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
  85. package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
  86. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
  87. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
  88. package/runtime/gatewaysdk/environment/schema/platform.py +124 -178
  89. package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
  90. package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
  91. package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
  92. package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
  93. package/runtime/gatewaysdk/environment/schema/runtime.py +77 -160
  94. package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
  95. package/runtime/gatewaysdk/environment/schema/session_files.py +166 -0
  96. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +2 -2
  97. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +22 -14
  98. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +29 -3
  99. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +16 -16
  100. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +18 -18
  101. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
  102. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +15 -0
  103. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
  104. package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
  105. package/runtime/gatewaysdk/environment/schema/snapshot.py +385 -129
  106. package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
  107. package/runtime/gatewaysdk/environment/schema/tasks.py +140 -55
  108. package/runtime/gatewaysdk/environment/schema/tools_world.py +110 -87
  109. package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
  110. package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
  111. package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
  112. package/runtime/gatewaysdk/environment/schema/validation.py +166 -40
  113. package/runtime/gatewaysdk/environment/schema/workers.py +142 -57
  114. package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
  115. package/runtime/gatewaysdk/environment/schema/world_tests.py +71 -39
  116. package/runtime/gatewaysdk/environment/score.py +57 -0
  117. package/runtime/gatewaysdk/environment/secret_values.py +36 -0
  118. package/runtime/gatewaysdk/environment/world.py +42 -3
  119. package/runtime/gatewaysdk/environment/world_session_contract.py +32 -5
  120. package/skills/benchmarks-getting-started/SKILL.md +2 -2
  121. package/skills/connector-schema-authoring/SKILL.md +22 -14
  122. package/skills/connector-schema-authoring/connector-worlds.md +26 -10
  123. package/skills/connector-schema-authoring/world-schema.md +111 -51
  124. package/skills/world-data-ingestion/ROWS.md +29 -3
  125. package/skills/world-data-ingestion/SKILL.md +16 -16
  126. package/skills/worlds-getting-started/SKILL.md +18 -18
  127. package/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
  128. package/skills/worlds-getting-started/references/gotchas.md +15 -0
  129. package/skills/worlds-getting-started/references/openapi-import.md +50 -21
  130. package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
@@ -41,7 +41,7 @@ registration, a platform builder, live ingestion, tools, or a registry framework
41
41
  ## The rules that never bend
42
42
 
43
43
  - **The schema is the deliverable.** A world with a faithful model and no tools is complete. Tools are added when wanted, by whoever wants them.
44
- - **Every write goes through the schema.** Handlers use `gateway_env.store`; ingestion uses `store.apply_batch` or `store.ingest`; nothing else writes. Out-of-contract writes are rejected with the reason.
44
+ - **Every write goes through the schema.** Handlers use `gateway_env.store`; ingestion uses `store.apply_batch` or `store.ingest`; nothing else writes. Out-of-contract writes are rejected with the reason (one known gap: never name an ingest session's `TEMP` staging table after a world table; see `world-schema.md`, Raw ingestion).
45
45
  - **Check the right boundary.** Compile/check validate compiled artifacts; materialization and store operations validate persisted data. Publication checks actual bundle bytes, handler declarations and initial state through the trusted runtime. Fidelity evidence and Environment Actions remain advisory.
46
46
  - **Fidelity shape is decided by the vendor, not by a house rule.** A **base** connector is prebaked once (Slack). A **tenant-shaped** connector's model is mostly customer-defined, so describe at bind time is required (Salesforce, NetSuite, Workday). A **mixed** connector is a base plus an optional overlay (Okta, Entra, Google, LDAP, Jamf).
47
47
  - **Surface parity is advisory.** If the world declares it mirrors an MCP surface, produce the parity report and show it. Never block on it, never shrink or bend the model to satisfy it.
@@ -55,7 +55,7 @@ registration, a platform builder, live ingestion, tools, or a registry framework
55
55
  | Command or path | Status |
56
56
  | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
57
57
  | `gateway worlds schema init PATH --connector <slug> [--name NAME] [--with-example-tool]` | exists (#219); `--connector custom` is the blank starting point for an arbitrary connector: its `$id` is derived from `--name` (`urn:gateway:connector:<name>`), it ships the `records` entity and a starter verifier but no tools unless `--with-example-tool` asks for the `list_records`/`get_record`/`create_record` example, and it writes no `data/records.json` (Brex retrospective, 2026-09-14) |
58
- | `gateway worlds schema init PATH --from-openapi SOURCE [--plan-only] [--include GLOB\|TAG]… [--exclude GLOB\|TAG]…` | exists (2026-09-22); imports a Swagger 2.0 / OpenAPI 3.x document (JSON or YAML; a local path, `drive:<path>` read platform-side, or an http(s) URL) into `schema/world.json` + `connector.toml` + `schema/provenance.json`: keyed schemas the selected operations return become entities, every path+method a route (passthrough when the response is not an entity), the plan is printed first and `--plan-only` writes nothing; refusals (no `paths`, unknown `$ref`, cyclic entity root, undeclared path parameter) carry a JSON pointer. Rules: `worlds-getting-started` → `references/openapi-import.md` |
58
+ | `gateway worlds schema init PATH --from-openapi SOURCE [--plan-only] [--include GLOB\|TAG]… [--exclude GLOB\|TAG]…` | exists (2026-09-22); imports a Swagger 2.0 / OpenAPI 3.x document (JSON or YAML; a local path, `drive:<path>` read platform-side, or an http(s) URL) into `schema/world.json` + `connector.toml` + `schema/provenance.json`: the item schemas the selected operations' responses carry, read through the vendor's envelope, become entities (never the wrappers; keyed by a declared id, else the field the read route looks a row up by, nested as a JSON Pointer when it sits inside the row), every path+method a route that answers in the vendor's envelope (`envelope = "message.items"` + `wrapper`) and pages as the document says (passthrough when the response is not an entity), the plan is printed first and `--plan-only` writes nothing; a YAML document needs PyYAML in the Python that runs the runtime and is refused naming the fix without it; refusals (no `paths`, unknown `$ref`, cyclic entity root, undeclared path parameter) carry a JSON pointer. Rules: `worlds-getting-started` → `references/openapi-import.md` |
59
59
  | `gateway worlds connector validate\|capture\|ingest PATH` | exists; a `connector.toml` in the world describes base URL, auth scheme, pacing, redaction and operations; `capture` records paced, redacted pages (errors included) under `captures/` with the secret never written; `ingest` seeds `data/initial.json` through the contract in a scratch copy first; `ingest --dry-run [--report refusals.jsonl]` lists every violation of every capture (the `data import --report` records plus `capture`), exit 2, nothing written |
60
60
  | `gateway worlds schema reconcile PATH --from captures [--write]` | exists (runtime, offline CLI and `gateway worlds schema reconcile`); the contract against the captured rows: relaxes `required` for fields absent in captures, adds undocumented keys as optional properties typed from the values, drops formats the captured values fail; a report of every change, `schema/world.json` rewritten and recompiled only with `--write` |
61
61
  | `gateway worlds connector conform PATH --openapi vendor.yaml` | exists; serves the world in-process, calls every declared operation with arguments from its own seed, and checks status + body against the vendor's OpenAPI response schemas; non-zero exit on any mismatch. Run it before calling a world 1:1; its FAIL list is the work list (unseeded entities → 404s, creates the contract rejects → 4xx, honest 501s). Reads OpenAPI 3.0 `nullable`, reference cycles and `discriminator` unions; `[operations.api] sample_headers` / `sample_body` shape a request the route demands (`Idempotency-Key`, a body) |
@@ -64,7 +64,8 @@ registration, a platform builder, live ingestion, tools, or a registry framework
64
64
  | `gateway worlds schema compile \| check \| describe PATH` | exists (#216); `compile` also writes `ui/static` (index.html, app.js, contract.json) when connector.toml declares `[ui]` (every shipped template does): the page over the routes an agent drives with `--surface ui,browser`; `[ui] generated = false` keeps a hand-written page |
65
65
  | `gateway worlds schema ui init PATH` | exists (TS CLI + `python -m … ui-init`); declares `[ui]` in connector.toml when missing (before the first `[[operations]]`) and compiles, which generates the page; refuses a world with no connector.toml (a page shows routes). `worlds serve` serves it on its own port at `/` (and at `/__ui/`) and answers its `api_path` for the pin; `generated = false` + any frontend built to static files into `ui/static` is served the same way |
66
66
  | `gateway worlds schema entity add PATH NAME --ref D \| --fields "a:string,b:integer?" \| --schema JSON --key F [--unique F,F] [--relationship F->entity]` | exists (TS CLI, local); declares one entity in `schema/world.json` without hand-editing the document: keeps `$defs`, package metadata and other entities byte-for-byte in meaning, refuses duplicate names, non-required or non-string/integer keys, unknown `$defs` and undeclared relationship targets, and writes nothing on refusal |
67
- | `gateway worlds connector route add PATH NAME --path /v1/x/{id} [-X POST] [--param q] [--entity E] [--results r] [--action create] [--filter id=id] [--body f=p] [--handler fn]` | exists (TS CLI, local); appends one `[[operations]]` (+ `[operations.api]`) to `connector.toml` leaving existing bytes untouched; validates name, method, path, entity against `schema/world.json`, filter keys against `{args}`/params, and the handler/action pairing |
67
+ | `gateway worlds schema entity remove PATH NAME` | exists (TS CLI, local); takes one entity out of `schema/world.json` with the `$defs` only it used; refused (exit 2, nothing written) while a `connector.toml` operation, a `verifiers/*.sql`, rows in `data/initial.json`, a `[dependencies.<alias>.entities]` map or another entity's relationship uses it, each named; keeps at least one entity and re-points the untouched `state_present.sql` placeholder |
68
+ | `gateway worlds connector route add PATH NAME --path /v1/x/{id} [-X POST] [--param q] [--entity E] [--results r] [--action create] [--filter id=id] [--body f=p] [--handler fn]` | exists (TS CLI, local); appends one `[[operations]]` (+ `[operations.api]`) to `connector.toml` leaving existing bytes untouched; validates name, method, path, entity against `schema/world.json`, filter keys against `{args}`/params, and the handler/action pairing; a new `connector.toml` starts with `[auth] scheme = "bearer"` (the hosted session token) and `--action handler --handler fn` writes a 501 stub into `api/handlers.py` unless `fn` is defined |
68
69
  | `gateway worlds connector publish DIR [--slug S] [-m MSG] [--tag T] [--workspace-visible] [--project-scope]` | exists; saves a template directory, or the world it was authored in, as a workspace connector (Context Hub `connector` repo, org-owned, versioned); the platform compiles the template before accepting it and answers 400 with the reason; a re-publish is a new commit and paths no longer present are removed; `data/`, `captures/`, `db/` and generated files stay behind |
69
70
  | `gateway worlds connector list [--hub] [--query Q]` / `pull SLUG [--out DIR] [--version V]` / `clone SLUG-OR-ID [--name N] [--slug S]` | exists; `list` shows this org's connectors (vendor, auth, entities, operations), `--hub` adds every connector another org published; `pull` writes the template files; `clone` copies a published connector into this org with provenance — a live reference is never made, so the publisher's later edits need a re-clone |
70
71
  | `gateway worlds schema init PATH --connector workspace:<slug>` | exists; initializes from the platform's copy through `init_bundle_from_files` (needs `GATEWAY_*` keys); the offline `python -m gatewaysdk.environment.schema init` refuses `workspace:` and names this command |
@@ -103,9 +104,9 @@ deliverable can finish with synthetic fixtures and explicitly skipped live valid
103
104
  - [ ] Author `schema/world.json` and reusable sibling definitions from pinned vendor sources, following the type-mapping notes in [CONNECTORS.md](CONNECTORS.md). `gateway worlds schema entity add` declares an entity with its key, uniques and relationships in one command so the declaration shape is never wrong; hand-author only the payload definitions (`--schema` or `$defs`). Record source URLs, versions/retrieval dates, and hashes of the actual retrieved bytes in `schema/provenance.json`.
104
105
  - [ ] Entities have the vendor's keys, relationships, enums, and nullability, and follow [CONVENTIONS.md](CONVENTIONS.md): one reserved prefix per template, plural snake_case names, vendor casing on fields, shared `base.json` primitives. Reserved prefixes and the `payload` name are avoided.
105
106
  - [ ] Enforce supported required fields, types, enums, key uniqueness, and reference integrity. Document vendor rules the dialect cannot express, normalization choices, and constraints the future mapper must enforce; structural validation is not vendor behavior.
106
- - [ ] Real vendor documents compile as they come (Brex retrospective, 2026-09-14): `format: int64`/`double` on numbers is ignored, not refused (string formats are still validated); a schema-valued `additionalProperties` types a map (`Record<string, string>`); a type nested in itself (`ExpandableExpense` holding an `ExpandableExpense`) compiles, with the nested occurrence validated as its bare type and stored as JSON, never projected to a column. A nested key or relationship field is a JSON Pointer, `/owner/user_id`, never `owner.user_id`; the compiler's refusal says so.
107
- - [ ] Every property verifier SQL will name has a column: scalars and compositions whose non-null branches are scalar (`anyOf [{$ref Status}, {type: null}]`, the nullable-enum shape) project; a composition that may hold an object or array stays JSON-only. A verifier that names a column the contract never projected fails the grade with SQLite's message and the statement; it never scores 0 quietly (double-quoted unknown names no longer fall back to string literals).
108
- - [ ] `gateway worlds schema compile PATH` then `check PATH` pass. Never edit generated files; SDK templates contain authored inputs and generate bundle artifacts at `init` time. Changing the root `$id` after `init` is fine: `compile` re-stamps an empty `data/initial.json` onto the new domain; a journal that holds rows is refused with the fix named (re-import its rows, or restore the schema it was written against).
107
+ - [ ] Real vendor documents compile as they come (Brex retrospective, 2026-09-14): `format: int64`/`double` on numbers is ignored, not refused (string formats are still validated); a schema-valued `additionalProperties` types a map (`Record<string, string>`); a type nested in itself (`ExpandableExpense` holding an `ExpandableExpense`) compiles, with the nested occurrence validated as its bare type and stored as JSON, never projected to a column. A contract whose inlined files would pass the 4 MB publication cap (Google Docs' recursive body) compiles each object/array definition once under `$defs` and references it instead; nothing to do in the template. A nested key or relationship field is a JSON Pointer, `/owner/user_id`, never `owner.user_id`; the compiler's refusal says so. A primary key may be nested too (`/protocolSection/identificationModule/nctId`) when every object on its path is required and non-null; lookups (`get`, `update`, `delete`) then pass it flat under the pointer itself. Field names keep the vendor's hyphens (`reference-count`); dots and slashes never belong to a name. A verifier's SQL double-quotes such a column (`"reference-count"`): unquoted, SQLite reads a subtraction.
108
+ - [ ] Every property verifier SQL will name has a column: scalars and compositions whose non-null branches are scalar (`anyOf [{$ref Status}, {type: null}]`, the nullable-enum shape) project; a composition that may hold an object or array stays JSON-only. A verifier that names a column the contract never projected fails the grade with the database engine's message and the statement; it never scores 0 quietly (double-quoted unknown names no longer fall back to string literals).
109
+ - [ ] `gateway worlds schema compile PATH` then `check PATH` pass. Never edit generated files; SDK templates contain authored inputs and generate bundle artifacts at `init` time. Changing the root `$id` after `init` is fine: `compile` re-stamps an empty `data/initial.json` onto the new domain; a journal that holds rows follows a change that only adds to its domain (a new entity, a new field that is not required) and is refused otherwise, with the fix named (re-import its rows, or restore the schema it was written against).
109
110
  - [ ] Shared templates and committed fixtures hold synthetic data only. Tenant captures stay in their owning project; initial batches and refresh journals retain explicit capture/synthetic provenance.
110
111
 
111
112
  ### 2. Tenant overlay (required for tenant-shaped, optional for mixed, absent for base)
@@ -132,7 +133,8 @@ deliverable can finish with synthetic fixtures and explicitly skipped live valid
132
133
  ### 5. Tools (only if the world ships tools)
133
134
 
134
135
  - [ ] Every tool in `x-gateway.tools` has exactly one handler in `tools/*.py` (the loader refuses a mismatch); generated defaults are fine.
135
- - [ ] Handler reads use `gateway_env.store.get` / `list`; writes use `insert` / `update` / `delete`. Read-only SQL uses `gateway_env.db.one/all`; raw SQL writes are denied. The host's `SQLiteStore.query` is not exposed by the handler store proxy.
136
+ - [ ] Routes are never tools. A call by a route's name is refused (`not_a_tool`, naming the session's api URL), and a task's `tools` lists declared tools only. A tool may share an operation's name: a call by that name runs the tool, and the route answers only over HTTP at its path.
137
+ - [ ] Handler reads use `gateway_env.store.get` / `list`; writes use `insert` / `update` / `delete`. Read-only SQL uses `gateway_env.db.one/all`; raw SQL writes are denied. The host's `WorldDatabase.query` is not exposed by the handler store proxy.
136
138
  - [ ] Handlers never read `benchmark_runtime_context` or `benchmark_conversation_events`.
137
139
  - [ ] A user attaching custom tools to this world needs nothing from this checklist beyond the store API; document any conventions the model expects (key formats, enum values).
138
140
 
@@ -142,7 +144,9 @@ Tool handlers (`tools/*.py`) reach the world's rows through `from gateway_env im
142
144
  exactly these methods; `gateway worlds schema check .` refuses any other `store.<name>` at file:line
143
145
  with this list, and a hosted call into such a handler answers 400 `{error: "handler", detail, handler}`
144
146
  (an `AttributeError`/`TypeError`/`NameError` is the author's bug; anything else a handler raises is a
145
- 500 `{error: "engine", detail, handler}`; never a bare exception string).
147
+ 500 `{error: "engine", detail, handler}`; never a bare exception string). A route's handler answers the
148
+ same way on every door, and an answer JSON cannot carry (a `datetime`, a set, NaN) is its 400; a route
149
+ answering bytes names its `Content-Type` (a vendor's `404 text/plain`), and the bytes go out as sent.
146
150
 
147
151
  - `get(entity: str, key: dict) -> dict | None` — one row by primary key, or `None`.
148
152
  - `list(entity: str) -> list[dict]` — every row, ordered by key.
@@ -234,7 +238,7 @@ Full procedure: `.llm_artifacts/plans/connector-implementation-orchestration.md`
234
238
  - Treating descriptive resources as schema authority. `init` preserves flat supporting `.json`, `.md` and `.toml` files; only referenced schemas become compiler inputs. Slack capability defaults expand into self-contained entries; see [world-schema.md](../../../documentation/features/world-schema.md#connector-templates-and-init).
235
239
  - Converting Slack message timestamps to floats or RFC3339. Preserve exact `(workspace_id, channel_id, ts)` keys; see the [Slack model guide](../../../gatewaysdk/sdk/src/gatewaysdk/environment/schema/connectors/slack/MODEL.md) for normalization and limits.
236
240
  - Writing a nested field as `owner.user_id` in `unique` or `relationships`. Nested paths are JSON Pointers: `/owner/user_id`.
237
- - Reading a verifier's 0 as "the agent failed". If the SQL names a column that does not exist the grade is now an error carrying SQLite's message; an older world that still scores 0 on every task has verifier SQL naming an unprojected property — recompile, the nullable-enum shape projects now.
241
+ - Reading a verifier's 0 as "the agent failed". If the SQL names a column that does not exist the grade is now an error carrying the database engine's message; an older world that still scores 0 on every task has verifier SQL naming an unprojected property — recompile, the nullable-enum shape projects now.
238
242
 
239
243
  ## Where this skill is discoverable
240
244
 
@@ -388,7 +392,10 @@ them:
388
392
  - `[api] envelope = "results"` wraps rows as `{"results": [...], "hits": N, "cursor": ""}`
389
393
  (empty means a bare array); `hits_key = ""` leaves the count out and
390
394
  `cursor_absent_when_done = true` (world-wide, or per route under `[operations.api]`)
391
- leaves the cursor key out of the last page — Brex's `{items, next_cursor?}`.
395
+ leaves the cursor key out of the last page — Brex's `{items, next_cursor?}`. A dotted
396
+ `envelope = "message.items"` nests the page (`{message: {items, total-results, next-cursor}}`),
397
+ and `[operations.api] wrapper = '{"status": "ok", "message-type": "work-list"}'` lays it onto
398
+ the rest of the vendor's envelope; a single row is wrapped only by its route's own `envelope`.
392
399
  - `unauthorized` / `not_found` / `invalid` are JSON strings replayed on 401 / 404 / 400, so
393
400
  the vendor's own error bodies come back; `""` (or `'null'`) means no body at all. A route
394
401
  whose missing row the vendor answers its own way sets `not_found` and `not_found_status`
@@ -404,10 +411,11 @@ them:
404
411
  credential for it, the way it does for a bearer of the session token.
405
412
 
406
413
  `gateway worlds serve PATH --port 8080 [--api-key K]` runs it locally; the runner's admin
407
- endpoints (`/healthz`, `/__world/seed`, `/__world/reset`, `/__world/state`) sit beside the
408
- vendor routes. `gateway bench push PATH` publishes the world: for a world carrying
414
+ endpoints (`/__world/seed`, `/__world/reset`, `/__world/state`) sit beside the vendor routes
415
+ and answer only the run's `X-World-Admin-Token` (`gateway worlds test` hands a world's pytest
416
+ files theirs as `WORLD_ADMIN_TOKEN`; by hand, use `/__session/*`); `/healthz` answers everyone,
417
+ and a World Host session's doors answer none of the others. `gateway bench push PATH` publishes the world: for a world carrying
409
418
  `connector.toml` it packages the api-kind bundle the platform hosts on the way (root
410
419
  legacy-shaped on purpose, since the api runner refuses a schema bundle there; `server/`
411
420
  needs nothing installed). Open a session with `--surface api` and hand the client that
412
- URL; the vendor's auth scheme applies (`WORLD_API_KEY` on the host pins the credential,
413
- otherwise any non-empty one).
421
+ URL; the vendor's auth scheme applies, with the session's own api token as the credential.
@@ -81,13 +81,23 @@ gateway worlds session open spycloud-world -t default --surface api # prints a
81
81
  A hosted api session runs on a fresh Fargate pod. The first open of a version takes about
82
82
  fifteen minutes (pod scheduling plus pulling the runner sidecar), during which the session
83
83
  reads `PROVISIONING` and the URL answers 503; that is a cold start, not a fault, until
84
- about twenty minutes have passed. `session open` waits ten minutes by default, so on a cold
85
- version pass `--no-wait` and poll `gateway worlds session status <sessionId>` until `ready`.
86
-
87
- The world's API honours the vendor's auth scheme: with `WORLD_API_KEY` set it accepts exactly that
88
- credential, otherwise any non-empty one, and answers 401 with the body configured under
84
+ about twenty minutes have passed. `session open` waits `--timeout` seconds (default 600) and,
85
+ when that runs out, prints the session with `ready: false` and exits 1 while it keeps opening;
86
+ on a cold version pass `--no-wait` and poll `gateway worlds session status <sessionId>` until
87
+ `ready`.
88
+
89
+ The world's API honours the vendor's auth scheme: a hosted session accepts its own api token (as a
90
+ bearer or in the vendor's header), `worlds serve` the credential `--api-key` pins (default `conform`),
91
+ and `WORLD_API_KEY` applies only to a legacy api bundle. Anything else answers 401 with the body configured under
89
92
  `[api].unauthorized` (`""` or `'null'` for a bare status with no body, the way some vendors answer).
90
93
 
94
+ The runner's own routes (`/__world/seed|reset|state|calls`) answer only `X-World-Admin-Token`. A session
95
+ pod and a Batch run mint that token themselves. A twin you host yourself (`hostedUrl`) gets it from
96
+ the world's secret `WORLD_ADMIN_TOKEN` (`gateway bench secrets set --env <slug> --key WORLD_ADMIN_TOKEN`,
97
+ the value on stdin), which a run or a session against the twin carries: start the twin's
98
+ `server/main.py` with the same `WORLD_ADMIN_TOKEN` in its environment, or every seed and grade is
99
+ refused with 403 `admin_token_required`.
100
+
91
101
  **Data.** Captures are one way in. Any agent can put rows into the world without knowing its
92
102
  layout: `gateway worlds data show ./spycloud-world` prints the kind and entity counts,
93
103
  `gateway worlds data import ./spycloud-world rows.json [--entity NAME] [--mode append|replace]`
@@ -120,15 +130,21 @@ least one `[[operations]]` entry.
120
130
  | `[capture]` | `requests_per_minute`, `timeout_seconds`, `max_pages`, `cursor_param`, `cursor_field`, `cursor_header` | Pacing between pages; providers ask us to be sparing. The next cursor is read from a body field or a response header (Torchlight sends `Cursor:`), never both, and is sent back as `cursor_param`; a cursor that is an absolute URL on the connection's host (DRF's `next`) is followed by its query. |
121
131
  | `[redact]` | `hash`, `drop`, `round`, `preserve` | Applied before a body reaches disk. `preserve` hashes shape-preservingly (same length, digits stay digits, letters keep their case, separators stay; slug-salted, deterministic) so an account number still meets the field's format — `hash` made it a 71-char `sha256:` string the contract refused (Brex). `hash` writes `sha256:<hex>` salted with the slug so joins survive, element by element for arrays so the shape the contract expects survives too; `drop` removes; `round = { lat = 2, lon = 2 }` keeps numbers numeric but coarsens them. A bare name matches at any depth; a dotted path (`results.*.email`, `*` for list items) matches that place only, so a catalog's `assets` count map keeps its `email` key. Field names keep the vendor's casing. |
122
132
  | `[api]` | `envelope`, `hits_key`, `cursor_key`, `page`, `limit_param`, `offset_param`, `cursor_param`, `page_size`, `previous_key`, `page_links`, `page_headers`, `cursor_absent_when_done`, `unauthorized`, `not_found`, `invalid`, `*_status`, `error_content_type`, `token_path`, `handlers`, `before` | `envelope = "results"` wraps rows as `{results, hits, cursor}` under the vendor's key names; empty means a bare array. `hits_key = ""` leaves the count out and `cursor_absent_when_done = true` leaves the cursor key out of the last page (instead of `""`), together the `{items, next_cursor?}` shape Brex-style vendors answer and clients test with `"next_cursor" in body`. `page` is `none`, `offset` or `cursor`, with the vendor's parameter names; `previous_key` adds the previous page and `page_links = true` renders next/previous as absolute URLs of the served host, the way Django REST Framework answers; `page_headers = { cursor = "Cursor", total = "Total-Count" }` sends the same values as response headers for vendors that page that way (Torchlight). Error bodies are JSON strings replayed verbatim with the vendor's statuses and content type (`application/problem+json` for RFC 7807 vendors); `""` (or `'null'`) means no body at all, for vendors that answer a bare 401. An oauth2 world mints bearers on `token_path` (default the spec's `token_url`) and honours only those. `handlers` names the world's Python module (default `api/handlers.py` when present); `before` names a gate in it that runs before every operation, declarative ones included, and returns `None` to let the request through or an answer (quota, allow-list, key state) to stop it. |
123
- | `[[operations]]` | `name`, `method`, `path`, `params`, `entity`, `results` | `path` may carry `{args}`; a `params` name may end in `[]` (`expand[]`, `status[]`) for a key the vendor repeats, which capture then sends repeated and the served route filters on any value of; `results` is the dotted path to the row array (empty when the body is the row). An operation without `entity` never seeds and answers 501 `not_modelled`. An `entity` (or an `explode` target) that `schema/world.json` does not declare is refused by `connector validate`, `schema compile`, `schema check` and the platform's describe (`gateway worlds describe`, local or platform) alike — `operation X lands on entity Y, which schema/world.json does not declare (declared: …)` at `/operations/<i>/entity` — and nothing is written; hand-edited TOML meets the same gate `route add` applies. |
133
+ | `[[operations]]` | `name`, `method`, `path`, `params`, `entity`, `results` | `path` may carry `{args}`; a `params` name keeps the vendor's spelling, dots included (`query.bibliographic`, `filter.overallStatus`), and may end in `[]` (`expand[]`, `status[]`) for a key the vendor repeats, which capture then sends repeated and the served route filters on any value of; `results` is the dotted path to the row array (empty when the body is the row). An operation without `entity` never seeds and answers 501 `not_modelled`. Routes are never tools: an agent calls them over HTTP (the api url and token `session open --surface api` prints), and a tool exists only where `x-gateway.tools` declares one with a handler. An `entity` (or an `explode` target) that `schema/world.json` does not declare is refused by `connector validate`, `schema compile`, `schema check` and the platform's describe (`gateway worlds describe`, local or platform) alike — `operation X lands on entity Y, which schema/world.json does not declare (declared: …)` at `/operations/<i>/entity` — and nothing is written; hand-edited TOML meets the same gate `route add` applies. |
124
134
  | `[operations.ingest]` | `drop`, `explode`, `carry_args` | `drop` removes vendor fields the entity does not carry; each `[[operations.ingest.explode]]` (`field`, `entity`, `carry`, `take`) turns a nested array into child rows; `carry_args = { account_id = "id" }` stamps the capture's path argument or query parameter onto every row before projection (a statement only knows its account from the URL it was fetched under). `connector ingest --dry-run [--report FILE]` lists every violation of every capture and writes nothing. |
125
- | `[operations.api]` | `filter`, `match`, `action`, `status`, `single`, `envelope`, `page`, `body`, `defaults`, `generate`, `handler`, `sample`, `sample_headers`, `sample_body`, `not_found`, `not_found_status`, `cursor_absent_when_done` | `filter` maps a path argument or declared query parameter to the row field it selects on (case-insensitive, any depth; an absent parameter does not narrow, one sent more than once matches rows carrying any of its values); `match = { name = "contains" }` switches an argument from equality to substring or `prefix`, and a list field matches when any element does. `action` is `read` (default), `create`, `update`, `delete` or `handler`: writes persist through the contract, `body` maps row fields (dotted for nested) to request-body paths, `defaults` fixes constants, `generate` fills `uuid`, `int` or `now`; `update` and `delete` find their row through `filter` and answer the vendor's 404 otherwise. `status` is the 2xx the vendor answers (201 for create, 204 for delete by default), `single` returns the row itself instead of an envelope, `envelope` and `page` override the world-wide setting per route. `handler` names a function `def name(ctx, args, query, body)` in the handlers module that returns a body, `(status, body)` or `(status, body, headers)`; `ctx` reads and writes the store (`rows`, `get`, `find`, `insert`, `update`, `delete`), keeps time (`now`, `elapsed`, `ladder` for status machines, frozen by `GATEWAY_WORLD_CLOCK`), mints ids (`uuid`) and shapes answers (`envelope`, `not_found`, `invalid`) and sees the request's `headers` and `query_all` (every value of every parameter, `{"expand[]": ["a", "b"]}`; the `query` argument keeps the first value only). `not_found` / `not_found_status` answer this route's missing row the vendor's own way (a 403 with its own message for one resource) and fall back to `[api]`; `cursor_absent_when_done` overrides `[api]` for this route. `sample = { ip = "203.0.113.5" }` gives the conformance run a value a validating route accepts when no seeded row can supply one; `sample_headers = { "Idempotency-Key" = "conform-1" }` and `sample_body = { reason = "conform" }` give it the headers and body a route demands (read by `connector conform` only; the served API never sends requests). |
135
+ | `[operations.api]` | `filter`, `match`, `action`, `status`, `single`, `envelope`, `wrapper`, `page`, `body`, `defaults`, `generate`, `handler`, `sample`, `sample_headers`, `sample_body`, `not_found`, `not_found_status`, `cursor_absent_when_done` | `filter` maps a path argument or declared query parameter to the row field it selects on (case-insensitive, any depth; an absent parameter does not narrow, one sent more than once matches rows carrying any of its values); `match = { name = "contains" }` switches an argument from equality to substring or `prefix`, and a list field matches when any element does. `action` is `read` (default), `create`, `update`, `delete` or `handler`: writes persist through the contract, `body` maps row fields (dotted for nested) to request-body paths, `defaults` fixes constants, `generate` fills `uuid`, `int` or `now`; `update` and `delete` find their row through `filter` and answer the vendor's 404 otherwise; they, and a `single` read with a path argument, refuse (`finds its row by nothing`, 400) when neither `filter` nor a path argument naming a field selects a row, rather than act on some row; a singleton served under a scoped path answers through `action = "handler"`. `status` is the 2xx the vendor answers (201 for create, 204 for delete by default), `single` returns the row itself instead of an envelope, `envelope` and `page` override the world-wide setting per route. A dotted `envelope` nests the answer the way the vendor does (`message.items` answers `{message: {items, total-results, next-cursor}}`), and a single row is wrapped only by its own route's `envelope` (`message` answers `{message: row}`). `wrapper` is the rest of the vendor's envelope as a JSON object string (`'{"status": "ok", "message-type": "work-list"}'`), the answer laid onto it key by key; `schema init --from-openapi` writes it from the document. `handler` names a function `def name(ctx, args, query, body)` in the handlers module that returns a body, `(status, body)` or `(status, body, headers)`; `ctx` reads and writes the store (`rows`, `get`, `find`, `insert`, `update`, `delete`), keeps time (`now`, `elapsed`, `ladder` for status machines, frozen by `GATEWAY_WORLD_CLOCK`), mints ids (`uuid`) and shapes answers (`envelope`, `not_found`, `invalid`) and sees the request's `headers` and `query_all` (every value of every parameter, `{"expand[]": ["a", "b"]}`; the `query` argument keeps the first value only). `not_found` / `not_found_status` answer this route's missing row the vendor's own way (a 403 with its own message for one resource) and fall back to `[api]`; `cursor_absent_when_done` overrides `[api]` for this route. `sample = { ip = "203.0.113.5" }` gives the conformance run a value a validating route accepts when no seeded row can supply one; `sample_headers = { "Idempotency-Key" = "conform-1" }` and `sample_body = { reason = "conform" }` give it the headers and body a route demands (read by `connector conform` only; the served API never sends requests). |
126
136
 
127
137
  ### Scaffolding a route
128
138
 
129
139
  `gateway worlds connector route add <world> <name>` appends one `[[operations]]` block to
130
140
  `connector.toml` without touching the bytes already there (comments and formatting survive;
131
- the TOML is parsed only to validate). Flags map onto the spec's keys:
141
+ the TOML is parsed only to validate). A world's first route creates `connector.toml` with
142
+ `[auth] scheme = "bearer"` and `secret_env = "<SLUG>_API_KEY"`: the api token a hosted session
143
+ hands the agent is a bearer, and `worlds serve` answers 401 without one and accepts the bearer
144
+ `--api-key` pins (default `conform`). `--action handler --handler fn` writes a stub
145
+ `def fn(ctx, args, query, body)` answering 501 into `api/handlers.py` unless `fn` is already
146
+ defined there; the output's `handler: {file, function, written}` says which. Flags map onto
147
+ the spec's keys:
132
148
 
133
149
  | Flag | Key |
134
150
  | ------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------- |
@@ -175,8 +191,8 @@ does all of that from the same `connector.toml`:
175
191
  (function allow-list, quota, blocked IP) hold on plain reads and writes as much as on handlers;
176
192
  handlers and the gate may answer with headers too (`SpyCloud-Error`, `Retry-After`).
177
193
  - An oauth2 world answers the vendor's token route before any authorization, so a client that
178
- performs the grant first works unchanged; with `WORLD_API_KEY` pinned, the client secret must
179
- be that key.
194
+ performs the grant first works unchanged; a hosted session checks the client secret against its
195
+ own api token, `worlds serve` against the `--api-key` pin.
180
196
 
181
197
  The handlers ship inside the api bundle beside the world, so the hosted container runs them
182
198
  with nothing installed. The abstraction is the platform's; the handlers are the customer's.
@@ -4,15 +4,16 @@
4
4
 
5
5
  ## Scope and runtime support
6
6
 
7
- `gateway-world/1` makes SQLite mock-tools state an executable contract. One
7
+ `gateway-world/1` makes a mock-tools world's database state an executable
8
+ contract, stored by the `WorldDatabase` engine the manifest names (SQLite today). One
8
9
  authored JSON Schema document declares payloads, entities, keys, relationships,
9
10
  and tool shapes. A stdlib Python compiler generates artifacts and a lock;
10
11
  store/world transactions enforce data and tool contracts.
11
12
 
12
13
  The bounded JSON Schema 2020-12 compiler enforces contracts; connector handlers
13
14
  implement vendor behavior. Slack now has a finite executable model and a
14
- structured capture-to-batch mapper. Automatic migrations, tenant overlays and
15
- non-SQLite backends remain outside this contract.
15
+ structured capture-to-batch mapper. Automatic migrations and tenant overlays
16
+ remain outside this contract; SQLite is the only engine shipped today.
16
17
  Ingestion code may write raw SQL through a [schema-gated
17
18
  session](#raw-ingestion-schema-gated-sql), and connector
18
19
  [templates](#connector-templates-and-init) initialize new worlds.
@@ -55,7 +56,7 @@ serve = ["list_accounts"]
55
56
  | Supporting scope, provenance, fidelity and capability metadata | Descriptive resources; not payload schemas or proof of executed checks. |
56
57
  | `data/initial.json` | Authored or mapper-produced initial mutation batch. |
57
58
  | `tools/*.py`, `verifiers/*.sql` | Authored behavior and read-only grading. |
58
- | `db/schema.sql` | Generated SQLite domain DDL. |
59
+ | `db/schema.sql` | Generated storage DDL for the world's engine (SQLite today). |
59
60
  | `schema/runtime.json` | Generated resolved contract; not authored. |
60
61
  | `schema/README.md` | Generated entity and tool documentation. |
61
62
  | `tool-schemas.json` | Generated `tools` map with `input_schema`, `output_schema`, and `description`. |
@@ -140,6 +141,14 @@ string/integer fields, unknown `$defs`, and any file that is not a `gateway-worl
140
141
  document; a refusal leaves the file untouched. Source:
141
142
  `gatewaysdk/sdk-ts/src/cli/runtime/scaffold.ts`.
142
143
 
144
+ `gateway worlds schema entity remove ./x records` takes one entity out again, with the `$defs`
145
+ only it used (the custom scaffold's example `records` entity once yours are declared). It is
146
+ refused, naming each reference and writing nothing, while a `connector.toml` operation answers
147
+ from it, a `verifiers/*.sql` reads it, `data/initial.json` holds rows of it, a
148
+ `[dependencies.<alias>.entities]` map names it or another entity relates to it; the world keeps
149
+ at least one entity, and the untouched `state_present.sql` placeholder is re-pointed at one that
150
+ stays. Source: `gatewaysdk/sdk-ts/src/cli/runtime/scaffoldEntityRemove.ts`.
151
+
143
152
  ## Supported payload language
144
153
 
145
154
  | Construct | `gateway-world/1` semantics |
@@ -152,7 +161,7 @@ document; a refusal leaves the file untouched. Source:
152
161
  | Strings | `minLength`, `maxLength`; no regex/pattern constraints. |
153
162
  | Numbers | `minimum`, `maximum`, numeric `exclusiveMinimum`/`exclusiveMaximum`. Every numeric value must be finite and within ±(2^53−1), including nonintegral numbers. A `format` on an integer or number (`int64`, `int32`, `double`, as vendor OpenAPI documents write them) is dropped, not refused; it asserts nothing. |
154
163
  | Timestamps | Only `format: "date-time"`: RFC3339 profile with a real date, seconds, and explicit `Z` or numeric offset; fractional seconds and lowercase `t`/`z` accepted; leap seconds rejected. |
155
- | Reuse | `$defs`, `$ref` to `#/$defs/Name` or `base.json#/$defs/Name` (including valid JSON Pointer paths beneath `$defs`). A definition may reference itself, directly or through a property, item or composition branch (Brex's `ExpandableExpense` holds an `ExpandableExpense`): the first occurrence is inlined in full, the nested occurrence keeps only the target's bare `type` (annotated with a `$comment` naming the target), so it is validated as that type and stored as JSON. A cycle of bare references or compositions, with no direct `type` to keep, is refused. |
164
+ | Reuse | `$defs`, `$ref` to `#/$defs/Name` or `base.json#/$defs/Name` (including valid JSON Pointer paths beneath `$defs`). A definition may reference itself, directly or through a property, item or composition branch (Brex's `ExpandableExpense` holds an `ExpandableExpense`): the first occurrence is inlined in full, the nested occurrence keeps only the target's bare `type` (annotated with a `$comment` naming the target), so it is validated as that type and stored as JSON. A cycle of bare references or compositions, with no direct `type` to keep, is refused. Compile inlines every reference while each generated file fits the 4 MB publication cap. When it would not (Google Docs: one body type reached along hundreds of paths, 24 MB inlined), each object or array definition is emitted once under the root's `$defs` as `{"$ref": "#/$defs/<file>.<Name>", "type": ...}`, a nested occurrence of a type in itself is a real reference (validated in full), scalar definitions stay inlined, and `schema/README.md` documents each definition once under Definitions. |
156
165
  | Composition | `allOf`, `anyOf`, `oneOf`, with nonempty branch lists. These validate values; they do not merge entity declarations or implement inheritance. |
157
166
  | Documentation | `title`, `description`, `$comment`, `examples`, `deprecated`, `readOnly`, `writeOnly` have no mutation/access-control semantics. `default` must validate but is never inserted. |
158
167
 
@@ -185,20 +194,25 @@ and inheritance/overrides are rejected. Reuse definitions by `$ref` and declare
185
194
  each entity once in the root catalog.
186
195
 
187
196
  Entity names use `[A-Za-z][A-Za-z0-9_]*`. Payload property names use
188
- `[A-Za-z_][A-Za-z0-9_]*`, so safe vendor fields such as `_links` and `_etag`
189
- are retained. `_key`, `payload`, and the `sqlite_`, `gateway_`, and
190
- `benchmark_` storage prefixes remain reserved case-insensitively. Case-fold
191
- collisions are rejected. Domain prefixes such as `crm_` avoid collisions when
192
- authoring one combined catalog.
193
-
194
- Primary keys are ordered, nonempty lists of distinct, direct, required, non-null
195
- string/integer properties. Unique keys may use direct field names or RFC 6901
196
- JSON Pointers such as `/profile/email`; pointer escapes `~0` and `~1` are
197
- decoded. Their scalar leaves may be optional or nullable: missing/null values
198
- do not conflict, while equal present non-null values do. Composite keys work.
199
- Composition and duplicate key declarations reject. Lookup objects contain
200
- exactly the direct primary-key fields. Integral keys share identity (`1` equals
201
- `1.0`) without rewriting payloads.
197
+ `[A-Za-z_][A-Za-z0-9_-]*`, so vendor fields such as `_links`, `_etag` and
198
+ `reference-count` keep the vendor's spelling. A dot or a slash is never part of a
199
+ name: nested fields are addressed by JSON Pointer. `_key`, `payload`, and the
200
+ `sqlite_`, `gateway_`, and `benchmark_` storage prefixes remain reserved
201
+ case-insensitively. Case-fold collisions are rejected. Domain prefixes such as
202
+ `crm_` avoid collisions when authoring one combined catalog.
203
+
204
+ Primary keys are ordered, nonempty lists of distinct, required, non-null
205
+ string/integer fields. A field is a direct property name or an RFC 6901 JSON
206
+ Pointer to a nested scalar, such as `/protocolSection/identificationModule/nctId`;
207
+ every object along the pointer must be required and non-null, so every row
208
+ carries its key. Unique keys may use direct field names or JSON Pointers such as
209
+ `/profile/email`; pointer escapes `~0` and `~1` are decoded. Their scalar leaves
210
+ may be optional or nullable: missing/null values do not conflict, while equal
211
+ present non-null values do. Composite keys work. Composition and duplicate key
212
+ declarations reject. Lookup objects contain exactly the primary-key fields, each
213
+ under its own name (the pointer itself for a nested field:
214
+ `{"/protocolSection/identificationModule/nctId": "NCT00000001"}`). Integral keys
215
+ share identity (`1` equals `1.0`) without rewriting payloads.
202
216
 
203
217
  Relationships declare ordered `fields`, a target entity, and
204
218
  `on_delete: "restrict"`, targeting its primary key with equal arity
@@ -224,15 +238,16 @@ Unresolved targets reject without fabricating placeholders. Mappers must choose
224
238
  valid records or model external identifiers as ordinary fields without a relationship.
225
239
  `ArtifactReference.logicalSourceId` identifies evidence, not universal entity keys.
226
240
 
227
- ## SQLite representation and supported store API
241
+ ## Storage representation and supported store API
228
242
 
229
- Tables hold canonical `payload` JSON, a private encoded `_key`, and generated
243
+ The representation below is the SQLite engine's (the one shipped today). Tables
244
+ hold canonical `payload` JSON, a private encoded `_key`, and generated
230
245
  scalar projections for verifiers and UNIQUE/FK constraints. A property projects
231
246
  when it can only hold scalars: a scalar `type` (nullable or not), or a
232
247
  composition whose non-null branches are all scalar, such as the nullable enum
233
248
  reference `anyOf [{$ref Status}, {type: null}]`. Objects/arrays, and any
234
249
  composition that may hold one, remain JSON only. SQL exposes booleans as SQLite
235
- values; store reads restore JSON booleans. Query errors keep SQLite's message
250
+ values; store reads restore JSON booleans. Query errors keep the engine's message
236
251
  (`no such column: status`), and the store disables SQLite's double-quoted
237
252
  string-literal fallback, so a verifier naming a column that does not exist
238
253
  fails the grade with the statement instead of comparing two literals and
@@ -241,11 +256,14 @@ Missing and null both project to SQL NULL but remain distinct in `payload`;
241
256
  grade that distinction with JSON presence/type queries. Validation preserves
242
257
  false, zero, empty values, null, and absence without coercion/default insertion.
243
258
 
244
- Import `SQLiteStore` from `gatewaysdk.environment.schema`:
259
+ Import `WorldDatabase` from `gatewaysdk.environment.schema`. `WorldDatabase.create` / `open` pick
260
+ the engine the contract was compiled for (its lock's `target`; SQLite today, `SQLiteWorldDatabase` in
261
+ `gatewaysdk.environment.schema.database`). Every method below lives on the base, which owns the
262
+ contract, the audit and the guard, so a later engine answers the same way.
245
263
 
246
264
  | API | Behavior |
247
265
  | ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ |
248
- | `create(path, contract)` / `open(path, contract)` | Context-managed store; creation refuses an existing file; opening checks the exact lock, actual DDL, and state. |
266
+ | `create(path, contract)` / `open(path, contract)` | Context-managed store; creation refuses an existing file; opening checks the storage contract, actual DDL, and state. |
249
267
  | `get(entity, key)` / `list(entity)` | Returns decoded payloads; a missing key returns `None`; list order follows encoded primary keys. |
250
268
  | `insert(entity, value, source=...)` | Validates a complete record; returns its payload. |
251
269
  | `update(entity, key, patch, source=...)` | Shallow-merges the patch, validates the complete result, and forbids primary-key changes; returns the resulting payload. |
@@ -260,15 +278,15 @@ Import `SQLiteStore` from `gatewaysdk.environment.schema`:
260
278
  A tool handler (`tools/handlers.py`) does not see this class. `from gateway_env import store` is a
261
279
  facade over it admitting exactly `get, list, list_where, insert, update, delete, apply_batch,
262
280
  lineage, counts` — the one list in `environment/schema/store_contract.py` (`STORE_METHODS`, read off
263
- `SQLiteStore` so it cannot drift). `gateway worlds schema check` refuses any other `store.<name>`
281
+ `WorldDatabase` so it cannot drift). `gateway worlds schema check` refuses any other `store.<name>`
264
282
  statically at file:line with that list; a hosted call into a handler whose own code raises
265
283
  `AttributeError`/`TypeError`/`NameError` answers 400 `{error: "handler", detail, handler}`, any other
266
284
  handler failure 500 `{error: "engine", detail, handler}` — never a bare exception string. There is
267
285
  no `upsert`: `get`, then `insert` or `update`.
268
286
 
269
287
  Connections enable foreign keys, bind values, and quote identifiers. `World` copies the validated
270
- template for branches/reset; snapshots require the exact contract on reopen.
271
- Reset rejects a changed contract.
288
+ template for branches/reset; a snapshot reopens under any contract with the storage contract its
289
+ rows were stored under (below). A live `World`'s reset rejects any changed lock: its tools are bound.
272
290
 
273
291
  `gateway_env.store` is that facade, bound to the currently executing schema world.
274
292
  Lifecycle/transaction ownership stays with `World`. The legacy `db.one/all` and
@@ -339,7 +357,10 @@ Provider authorization, capture lifetime/fences, and schema compilation remain s
339
357
  Ingestion code, including code an agent writes in a sandbox, may write SQL
340
358
  straight into a world. The contract, not the API, decides what lands: the
341
359
  session commits only when every persisted row satisfies the schema, keys and
342
- relationships, and otherwise rolls back as a whole.
360
+ relationships, and otherwise rolls back as a whole. Known gap: a `TEMP` table named like a
361
+ domain or `_gateway_*` table shadows it for that check and for the batch and lineage rows, so
362
+ an invalid write can commit unrecorded (open bug, pinned in the guard matrix, to be fixed by
363
+ #474). Never name staging tables after world tables.
343
364
 
344
365
  ```python
345
366
  with world.store.ingest(
@@ -358,7 +379,7 @@ print(conn.result) # {"status": "applied", "batch": ..., "inserted": n, ...}
358
379
  | Inside a session | Rule |
359
380
  | -------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
360
381
  | Domain tables | `INSERT`, `UPDATE`, `DELETE` allowed; `gateway_key(entity, payload)` computes `_key` in SQL and returns NULL for a payload without its primary key, so the NOT NULL constraint rejects the row with the reason attached. |
361
- | `TEMP` tables, views, indexes | Allowed for staging; dropped when the session ends. |
382
+ | `TEMP` tables, views | Allowed for staging; dropped when the session ends. A `TEMP` index is refused today (open bug, pinned in the guard matrix). |
362
383
  | `conn.script(sql)` | Runs a multi-statement script one statement at a time. `executescript` is not exposed because it would commit early. |
363
384
  | Transaction control, `PRAGMA`, `ATTACH`, persistent DDL | Denied by the authorizer (`ingest_sql`). The store owns the transaction. |
364
385
  | `_gateway_*`, `benchmark_runtime_context`, `benchmark_conversation_events` | Neither readable nor writable. `benchmark_runtime_parameters` stays readable. |
@@ -373,7 +394,8 @@ carrying the code, schema/data paths, entity and encoded key. A rejected
373
394
  session leaves state, lineage and its `batch_id` untouched. A committed session
374
395
  is recorded as one batch (`kind: "ingest"`) with a lineage row per inserted,
375
396
  updated or deleted key under the session's `source`, so raw ingestion keeps the
376
- same provenance as `apply_batch`, attributed per key rather than per mutation.
397
+ same provenance as `apply_batch`, attributed per key rather than per mutation
398
+ (except under the `TEMP`-shadow gap above, where the session commits with no batch or lineage).
377
399
  Reusing a committed `batch_id` is an `idempotency_conflict`; raw sessions are
378
400
  not replayable. Run `python samples/schema-worlds/ingest_example.py` for a
379
401
  committed and a rejected session.
@@ -420,7 +442,9 @@ and a starter verifier but no tools unless `--with-example-tool`
420
442
  holds only the recipe. Batches interchange only when the domain id, version and
421
443
  hash match. Giving an authored world a new `$id` later is fine: `compile`
422
444
  re-stamps an empty journal (`mutations == []`) onto the new domain, whatever
423
- `$id` it named; a journal that holds rows is refused with the fix named
445
+ `$id` it named; a journal that holds rows follows a change that only adds to
446
+ the domain it was written under (a new entity, a new field that is not
447
+ required, another `domain_version`) and is refused otherwise, with the fix named
424
448
  (re-import its rows against the new contract, or restore the schema it was
425
449
  written against). The target is pure to compute:
426
450
  `from gatewaysdk.environment.schema.compiler import build; build(world).target`.
@@ -436,17 +460,41 @@ gateway worlds schema init ./gh --from-openapi https://example.com/github.json -
436
460
  gateway worlds schema init ./crm --from-openapi drive:specs/crm.json --plan-only
437
461
  ```
438
462
 
439
- Every object schema under `definitions` / `components.schemas` with an id-like key (`id`,
440
- `<name>Id`, `<name>_id`, a `readOnly` field an `x-` extension marks unique, else the first
441
- required scalar) becomes an entity
442
- `$def` plus `x-gateway.entities[<snake_plural>]` with `primary_key`; keyless nested objects stay
443
- inline `$defs`; `allOf` is merged, `oneOf`/`anyOf` kept as unions the validator accepts; names
444
- are snake_cased, bounded to 64 characters, and the original recorded in `schema/provenance.json`. Every path+method becomes
445
- an operation in `connector.toml`: `operationId` snake_cased and bounded to 64 characters (a
446
- duplicate gets a numeric suffix and a warning), path and query parameters declared, an array
447
- or `{data: [...]}` response an entity list route (declared query parameters that name entity
448
- fields filter it), an object response the entity's read/create/update/delete by method; a
449
- response that is no entity stays a passthrough operation (no `entity`, `501` when called).
463
+ Entities are the rows the selected operations answer and take, read through the vendor's
464
+ envelope. A response that is an array of objects, or an object that declares no key and holds
465
+ exactly one array of objects (possibly behind objects holding nothing else, as Crossref's
466
+ `message` → `items`), lists that item schema. A single-row response is the row when it is some
467
+ list's item, declares a key, or names the field the route's own `{segment}` reads it by; else,
468
+ through a wrapper holding exactly one object, that object. A wrapper is never an entity: its path
469
+ becomes the route's dotted `envelope` (`message.items`) and the rest of it the route's `wrapper`
470
+ template, so the served world answers in the vendor's envelope again. An item's key is a
471
+ declared one (`id`, `<name>Id`, `<name>_id`, a `readOnly` field an `x-` extension marks unique),
472
+ else the field its single-row route reads it by: a top-level field spelled the vendor's way
473
+ (`/works/{doi}` → `DOI`), or one nested in the row's objects (never through an array or a
474
+ nullable object), which becomes a JSON Pointer key (`/studies/{nctId}` →
475
+ `/protocolSection/identificationModule/nctId`) and makes every object on that path required.
476
+ An item only lists carry falls back to its first required scalar, with a warning that rows
477
+ sharing it collapse. An item no planned route reads or writes is not an entity, and a schema
478
+ that is no item stays an inline `$def`; a document whose operations name no schema offers
479
+ every keyed schema instead. Each entity is a `$def` plus `x-gateway.entities[<snake_plural>]`
480
+ with `primary_key`; `allOf` is merged, `oneOf`/`anyOf` kept as unions the validator accepts;
481
+ entity names are snake_cased, bounded to 64 characters, and the original recorded in
482
+ `schema/provenance.json`; property names and query parameters keep the vendor's spelling
483
+ (`reference-count`, `query.bibliographic`). Every path+method becomes an operation in
484
+ `connector.toml`: `operationId` snake_cased and bounded to 64 characters (a duplicate gets a
485
+ numeric suffix and a warning), path and query parameters declared, a list response an entity
486
+ list route (declared query parameters that name entity fields filter it), a single-row response
487
+ the entity's read, and a write its create/update/delete by method; a single read whose path
488
+ argument names no field of the row, and a response that carries no entity, stay passthrough
489
+ operations (no `entity`, `501` when called), the reason in the plan. List routes page as the
490
+ document says: a cursor parameter (`cursor`, `pageToken`) beside a next-cursor field in the
491
+ envelope (`next-cursor`, `nextPageToken`) pages by cursor, an `offset` parameter by offset, and
492
+ the count beside the rows (`total-results`, `totalCount`) is the hits key. `[api]` pages the way
493
+ most list routes do, a route paging otherwise names its own `page`, and page-number paging
494
+ (`page` + `per_page`) is named in a warning, its route answering every row. An import declares
495
+ no tools: routes are never tools, so an agent reaches an imported world's routes over HTTP
496
+ (`session open --surface api`), and an author who wants a tool declares it in `x-gateway.tools`
497
+ with a handler.
450
498
  `HEAD`/`OPTIONS`/`TRACE` (anything outside `GET POST PUT PATCH DELETE`), a path that repeats a
451
499
  placeholder (`/a/{id}/{id}`), and operations
452
500
  `--include` / `--exclude` leave out (a path glob or a tag,
@@ -457,7 +505,7 @@ cyclic entity root, a document without `paths`, a document over 64 MiB.
457
505
  The command prints the plan (`entities`, `operations`, `skipped`, `warnings`, the document's
458
506
  version, title and sha256) and, unless `--plan-only`, writes the tree through the same writer as
459
507
  a template `init`, so a rejected import leaves nothing behind, then lists the `next` commands:
460
- `schema compile`, `schema check`, `data import`, `serve` / `call`, `sandbox commit` or `worlds
508
+ `schema compile`, `schema check`, `data import`, `serve` and a `curl` of a list route, `sandbox commit` or `worlds
461
509
  create`. `schema/provenance.json` records the source label (credentials and query strings
462
510
  stripped), its sha256 and size, the importer version and each entity's and operation's source
463
511
  pointer. `--connector` and `--with-example-tool` do not combine with it. The document's bytes
@@ -465,8 +513,14 @@ are read where the command runs: a local path directly; `drive:<path>` from the
465
513
  with the caller's key, in memory; an `http(s)://` URL through a private-address guard (no
466
514
  loopback, RFC 1918, link-local or metadata host, every redirect re-checked, five at most) — the
467
515
  platform never fetches a caller-named URL, and the sandbox's `run_gateway_cli` accepts only a
468
- `/workspace` path. `gateway worlds connector conform <world> --openapi <same document>` then
469
- checks the served world back against the document it came from.
516
+ `/workspace` path. The import runs in this machine's schema runtime whenever it has one
517
+ (`GATEWAY_RUNTIME=local` demands it and sends nothing to the platform); only with none here, or
518
+ with `GATEWAY_RUNTIME=platform`, does the platform's `init-openapi` runtime command read the
519
+ staged bytes. A YAML document is read by PyYAML in the Python that runs the runtime: without it
520
+ the command is refused before anything is sent, naming the fix (install PyYAML there, point
521
+ `GATEWAY_PYTHON` at a Python that has it, or convert the document to JSON).
522
+ `gateway worlds connector conform <world> --openapi <same document>` then checks the served
523
+ world back against the document it came from.
470
524
 
471
525
  **Build from connector** for Slack creates a durable populated world job in
472
526
  Gateway. Other registered source kinds show their CLI twin under **Use the CLI instead**:
@@ -563,7 +617,7 @@ as a credential-free subprocess (`GATEWAY_SCHEMA_PYTHON`, `GATEWAY_SCHEMA_RUNTIM
563
617
  | ------------------------------ | ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
564
618
  | `validate`, `validate-archive` | publication | Verify SHA-256, regenerated artifacts and exact lock of an existing tree/archive. |
565
619
  | `init-template` | `create_connector_world`, `create_custom_world --template` | Scaffold a connector template into a tree. |
566
- | `init-openapi` | `worlds schema init --from-openapi` (both CLIs) | `{source, document (base64), destination, plan_only?, include?, exclude?, name?}` → the import plan and, when written, the compiled tree as `files[]` (at most 2000). The bridge never opens a caller-named path or URL; the CLI stages the bytes. |
620
+ | `init-openapi` | `worlds schema init --from-openapi` (both CLIs; the TypeScript CLI through `POST /api/public/worlds/runtime` only when no runtime is here) | `{source, document (base64), destination, plan_only?, include?, exclude?, name?}` → the import plan and, when written, the compiled tree as `files[]` (at most 2000). The bridge never opens a caller-named path or URL; the CLI stages the bytes. A YAML document needs PyYAML in the Python that reads it, and is refused naming the fix without it. |
567
621
  | `compile` | `create_custom_world`, `ingest_to_world` on schema worlds | Compile an authored tree (`compileWorldFiles`): returns the full entry map with generated artifacts + lock, the schema report and which files were added/retargeted. |
568
622
  | `apply-batch` | `ingest_records_to_world` | Apply `insert\|update\|delete` mutations as one batch (`applyWorldBatch`): validated row by row against the compiled schema, appended as `data/batches/<batchId>.json` with `source` provenance; replaying the same `batchId` is a no-op. |
569
623
  | `build-slack`, `refresh-slack` | connector worlds | Slack capture → batches (durable job). |
@@ -601,11 +655,17 @@ and mutation order remain meaningful. No broader equivalence is promised.
601
655
  | Domain | Root `$id`, `domain_version`, and compiled entity-contract hash. Batches must match all three; changed domains require regeneration and explicit state rebuilding. |
602
656
  | Dependencies | Every resolved document's `$id` and normalized authored hash are locked. Missing, changed, or incompatible dependencies require an explicit fix/regeneration. |
603
657
  | Mapping | Batch `mapping.id/version`, persisted with lineage; a new mapping does not change the domain or package contract. |
604
- | Package/tool behavior | `x-gateway.package.id/version`, tool-contract hash, and generated artifact hashes. Tool/package changes require regeneration; old state cannot open under a different lock. Authored handler bytes remain bound by package/bundle versioning, not compiled schema semantics. |
605
-
606
- Opening requires exact lock equality, including package/tool/input hashes; no
607
- minor-version shortcut or automatic migration exists. Retain the original bundle
608
- with old state or explicitly rebuild. Version labels do not prove vendor fidelity.
658
+ | Package/tool behavior | `x-gateway.package.id/version`, tool-contract hash, and generated artifact hashes. Tool/package changes require regeneration; stored state still opens (it is bound to the storage contract, not the lock). Authored handler bytes remain bound by package/bundle versioning, not compiled schema semantics. |
659
+
660
+ Opening state (a snapshot, an export, a session's store) requires the same **storage
661
+ contract** (`StorageContract`): the language, annotations, compiler and target, the domain's
662
+ `$id`, `domain_version` and compiled-entity hash (in either resolver form; an entity field's
663
+ description, title and examples are part of it), and the generated SQL, plus identical DDL. The
664
+ rest of the lock (package, tools, document and artifact hashes) may differ: none of it touches a
665
+ stored row, so a tool edit keeps every snapshot. A domain change has no minor-version shortcut
666
+ or automatic migration: retain the original bundle with old state or explicitly rebuild. Two
667
+ worlds of one domain open each other's state; which state a hosted version opens is the
668
+ platform's to scope. Version labels do not prove vendor fidelity.
609
669
 
610
670
  ## Offline example
611
671