@withgateway/sdk 0.2.6 → 0.2.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/README.md +24 -8
  2. package/dist/cli.js +15468 -11263
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +15233 -11028
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +291 -40
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +284 -37
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/hooks/index.d.mts +1 -1
  13. package/dist/hooks/index.d.ts +1 -1
  14. package/dist/hooks/index.js +203 -47
  15. package/dist/hooks/index.js.map +1 -1
  16. package/dist/hooks/index.mjs +198 -42
  17. package/dist/hooks/index.mjs.map +1 -1
  18. package/dist/{index-BaKu5ZSU.d.mts → index-C3UzD0G3.d.mts} +75 -1
  19. package/dist/{index-BaKu5ZSU.d.ts → index-C3UzD0G3.d.ts} +75 -1
  20. package/dist/{index-CLkMr6VG.d.ts → index-D-_4Ibcn.d.ts} +36 -4
  21. package/dist/{index-CdbmO4aX.d.mts → index-FfK4Nump.d.mts} +36 -4
  22. package/dist/index.d.mts +3 -3
  23. package/dist/index.d.ts +3 -3
  24. package/dist/index.js +2149 -615
  25. package/dist/index.js.map +1 -1
  26. package/dist/index.mjs +2053 -566
  27. package/dist/index.mjs.map +1 -1
  28. package/dist/traces/index.js +268 -31
  29. package/dist/traces/index.js.map +1 -1
  30. package/dist/traces/index.mjs +262 -28
  31. package/dist/traces/index.mjs.map +1 -1
  32. package/dist/tracing/index.js +19 -11
  33. package/dist/tracing/index.js.map +1 -1
  34. package/dist/tracing/index.mjs +19 -11
  35. package/dist/tracing/index.mjs.map +1 -1
  36. package/dist/worlds/index.d.mts +886 -117
  37. package/dist/worlds/index.d.ts +886 -117
  38. package/dist/worlds/index.js +1770 -452
  39. package/dist/worlds/index.js.map +1 -1
  40. package/dist/worlds/index.mjs +1752 -434
  41. package/dist/worlds/index.mjs.map +1 -1
  42. package/package.json +14 -10
  43. package/runtime/gatewaysdk/environment/_toml.py +3 -10
  44. package/runtime/gatewaysdk/environment/_world.py +95 -28
  45. package/runtime/gatewaysdk/environment/runtime.py +10 -6
  46. package/runtime/gatewaysdk/environment/schema/__main__.py +35 -0
  47. package/runtime/gatewaysdk/environment/schema/api.py +19 -23
  48. package/runtime/gatewaysdk/environment/schema/batch.py +68 -14
  49. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +4 -2
  50. package/runtime/gatewaysdk/environment/schema/cast.py +264 -0
  51. package/runtime/gatewaysdk/environment/schema/compiler.py +35 -1
  52. package/runtime/gatewaysdk/environment/schema/conform.py +28 -10
  53. package/runtime/gatewaysdk/environment/schema/connector.py +59 -14
  54. package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +38 -1
  55. package/runtime/gatewaysdk/environment/schema/connectors/google-drive/readers/merge.toml +3 -1
  56. package/runtime/gatewaysdk/environment/schema/credentials.py +20 -0
  57. package/runtime/gatewaysdk/environment/schema/csv_rows.py +117 -0
  58. package/runtime/gatewaysdk/environment/schema/cycles.py +121 -0
  59. package/runtime/gatewaysdk/environment/schema/data.py +21 -5
  60. package/runtime/gatewaysdk/environment/schema/grading.py +95 -13
  61. package/runtime/gatewaysdk/environment/schema/handler_errors.py +44 -0
  62. package/runtime/gatewaysdk/environment/schema/handler_scan.py +175 -0
  63. package/runtime/gatewaysdk/environment/schema/host.py +68 -115
  64. package/runtime/gatewaysdk/environment/schema/host_requests.py +78 -0
  65. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +12 -0
  66. package/runtime/gatewaysdk/environment/schema/ingest.py +5 -18
  67. package/runtime/gatewaysdk/environment/schema/local.py +7 -10
  68. package/runtime/gatewaysdk/environment/schema/openapi_document.py +322 -0
  69. package/runtime/gatewaysdk/environment/schema/openapi_import.py +382 -0
  70. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +441 -0
  71. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +629 -0
  72. package/runtime/gatewaysdk/environment/schema/openapi_source.py +210 -0
  73. package/runtime/gatewaysdk/environment/schema/platform.py +110 -299
  74. package/runtime/gatewaysdk/environment/schema/platform_import.py +182 -0
  75. package/runtime/gatewaysdk/environment/schema/platform_query.py +95 -0
  76. package/runtime/gatewaysdk/environment/schema/route_bindings.py +176 -0
  77. package/runtime/gatewaysdk/environment/schema/runtime.py +56 -7
  78. package/runtime/gatewaysdk/environment/schema/scaffold.py +64 -5
  79. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +70 -22
  80. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +54 -4
  81. package/runtime/gatewaysdk/environment/schema/skills/hooks-getting-started/SKILL.md +27 -2
  82. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +62 -0
  83. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +91 -54
  84. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +146 -146
  85. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +11 -7
  86. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +30 -0
  87. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/new-world-sandbox.md +43 -0
  88. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +90 -0
  89. package/runtime/gatewaysdk/environment/schema/store.py +15 -6
  90. package/runtime/gatewaysdk/environment/schema/store_contract.py +119 -0
  91. package/runtime/gatewaysdk/environment/schema/tasks.py +121 -32
  92. package/runtime/gatewaysdk/environment/schema/tools_world.py +32 -10
  93. package/runtime/gatewaysdk/environment/schema/tree_import.py +228 -0
  94. package/runtime/gatewaysdk/environment/schema/ui.py +18 -2
  95. package/runtime/gatewaysdk/environment/schema/validation.py +35 -0
  96. package/runtime/gatewaysdk/environment/schema/verbs.py +8 -2
  97. package/runtime/gatewaysdk/environment/world.py +42 -3
  98. package/runtime/gatewaysdk/environment/world_session_contract.py +57 -0
  99. package/skills/benchmarks-getting-started/SKILL.md +70 -22
  100. package/skills/connector-schema-authoring/SKILL.md +54 -4
  101. package/skills/connector-schema-authoring/connector-worlds.md +21 -7
  102. package/skills/connector-schema-authoring/world-schema.md +144 -83
  103. package/skills/hooks-getting-started/SKILL.md +27 -2
  104. package/skills/world-data-ingestion/ROWS.md +62 -0
  105. package/skills/world-data-ingestion/SKILL.md +91 -54
  106. package/skills/worlds-getting-started/SKILL.md +146 -146
  107. package/skills/worlds-getting-started/references/cli-mcp-map.md +11 -7
  108. package/skills/worlds-getting-started/references/gotchas.md +30 -0
  109. package/skills/worlds-getting-started/references/new-world-sandbox.md +43 -0
  110. package/skills/worlds-getting-started/references/openapi-import.md +90 -0
@@ -17,15 +17,15 @@ Ingestion code may write raw SQL through a [schema-gated
17
17
  session](#raw-ingestion-schema-gated-sql), and connector
18
18
  [templates](#connector-templates-and-init) initialize new worlds.
19
19
 
20
- | Path | Schema-backed behavior |
21
- | --- | --- |
22
- | SDK `gwenv.init_from_path` | Checks artifacts and materializes isolated schema-aware worlds. |
23
- | Direct benchmark `mock-tools` runner | Reuses SDK materialization and `World` enforcement. |
24
- | SDK/public and platform publication, including READY/dedup/fork/ref promotion | Verifies the stored SHA-256, generated artifacts and exact lock through the trusted compiler/materializer using actual archive bytes; uploaded Python is parsed, not imported during publication. |
25
- | Worker builder, bundle tester, and workbench | Materialize through the shared schema store; tester/workbench execution uses an isolated sandbox with an installed trusted runtime. No legacy seed-SQL fallback. |
26
- | Hosted `mock-tools` sessions | Use the shared `World` in a child process for validated calls, reset and grading; dependency worlds are unsupported. |
20
+ | Path | Schema-backed behavior |
21
+ | ------------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
22
+ | SDK `gwenv.init_from_path` | Checks artifacts and materializes isolated schema-aware worlds. |
23
+ | Direct benchmark `mock-tools` runner | Reuses SDK materialization and `World` enforcement. |
24
+ | SDK/public and platform publication, including READY/dedup/fork/ref promotion | Verifies the stored SHA-256, generated artifacts and exact lock through the trusted compiler/materializer using actual archive bytes; uploaded Python is parsed, not imported during publication. |
25
+ | Worker builder, bundle tester, and workbench | Materialize through the shared schema store; tester/workbench execution uses an isolated sandbox with an installed trusted runtime. No legacy seed-SQL fallback. |
26
+ | Hosted `mock-tools` sessions | Use the shared `World` in a child process for validated calls, reset and grading; dependency worlds are unsupported. |
27
27
  | Platform agent (`create_custom_world`, `ingest_records_to_world`, `ingest_to_world`) | Authors and fills arbitrary schema worlds through the trusted bridge (`platform.py compile` / `apply-batch`) and publishes each result as one immutable READY version via `worldPublicationService`; see [agent.md](./agent.md#platform-actions-worlds-tasks-rollouts-connectors-ingestion-self-schedules). |
28
- | SDK Harbor worlds and Harbor/API/verifiers engines | Retain their separate unsupported-mode guards; do not combine their bundle markers with schema mode. |
28
+ | SDK Harbor worlds and Harbor/API/verifiers engines | Retain their separate unsupported-mode guards; do not combine their bundle markers with schema mode. |
29
29
 
30
30
  Environment Actions remain advisory. See [Environment Builder](./environment-builder.md) and
31
31
  [Benchmark Container Hub](./benchmark-containers.md).
@@ -47,21 +47,21 @@ language = "gateway-world/1"
47
47
  serve = ["list_accounts"]
48
48
  ```
49
49
 
50
- | Path | Ownership |
51
- | --- | --- |
52
- | `gateway-env.toml` | Manifest; `[tools].serve` must match declared contracts. |
53
- | `schema/world.json` | Authored domain/tool authority. |
54
- | `schema/base.json`, referenced sibling `.json` files | Reusable payload definitions. |
55
- | Supporting scope, provenance, fidelity and capability metadata | Descriptive resources; not payload schemas or proof of executed checks. |
56
- | `data/initial.json` | Authored or mapper-produced initial mutation batch. |
57
- | `tools/*.py`, `verifiers/*.sql` | Authored behavior and read-only grading. |
58
- | `db/schema.sql` | Generated SQLite domain DDL. |
59
- | `schema/runtime.json` | Generated resolved contract; not authored. |
60
- | `schema/README.md` | Generated entity and tool documentation. |
61
- | `tool-schemas.json` | Generated `tools` map with `input_schema`, `output_schema`, and `description`. |
62
- | `schema/lock.json` | Generated input/output and compatibility lock. |
63
- | `connector.toml` | Optional. The vendor connection: base URL, auth, pacing, redaction, the operations the agent calls, and the served envelope and errors. Read by `gateway worlds connector`, `gateway worlds serve` and `gateway bench push`; see [connector-worlds.md](./connector-worlds.md). |
64
- | `captures/` | Written by `gateway worlds connector capture`: redacted pages per operation plus `manifest.json`. Vendor data; never part of a template. |
50
+ | Path | Ownership |
51
+ | -------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
52
+ | `gateway-env.toml` | Manifest; `[tools].serve` must match declared contracts. |
53
+ | `schema/world.json` | Authored domain/tool authority. |
54
+ | `schema/base.json`, referenced sibling `.json` files | Reusable payload definitions. |
55
+ | Supporting scope, provenance, fidelity and capability metadata | Descriptive resources; not payload schemas or proof of executed checks. |
56
+ | `data/initial.json` | Authored or mapper-produced initial mutation batch. |
57
+ | `tools/*.py`, `verifiers/*.sql` | Authored behavior and read-only grading. |
58
+ | `db/schema.sql` | Generated SQLite domain DDL. |
59
+ | `schema/runtime.json` | Generated resolved contract; not authored. |
60
+ | `schema/README.md` | Generated entity and tool documentation. |
61
+ | `tool-schemas.json` | Generated `tools` map with `input_schema`, `output_schema`, and `description`. |
62
+ | `schema/lock.json` | Generated input/output and compatibility lock. |
63
+ | `connector.toml` | Optional. The vendor connection: base URL, auth, pacing, redaction, the operations the agent calls, and the served envelope and errors. Read by `gateway worlds connector`, `gateway worlds serve` and `gateway bench push`; see [connector-worlds.md](./connector-worlds.md). |
64
+ | `captures/` | Written by `gateway worlds connector capture`: redacted pages per operation plus `manifest.json`. Vendor data; never part of a template. |
65
65
 
66
66
  `db/seed.sql` is forbidden even if empty. Schema mode uses fixed paths, so
67
67
  `db.schema`, `db.seed`, and `db.path` overrides are also rejected. Contract paths
@@ -78,25 +78,33 @@ The root's `x-gateway` requires:
78
78
  {
79
79
  "version": 1,
80
80
  "domain_version": "1.0.0",
81
- "package": {"id": "gateway-sample-crm", "version": "1.0.0"},
81
+ "package": { "id": "gateway-sample-crm", "version": "1.0.0" },
82
82
  "entities": {
83
83
  "crm_accounts": {
84
- "schema": {"$ref": "#/$defs/Account"},
84
+ "schema": { "$ref": "#/$defs/Account" },
85
85
  "primary_key": ["id"],
86
86
  "unique": [["domain"]]
87
87
  },
88
88
  "crm_opportunities": {
89
- "schema": {"$ref": "#/$defs/Opportunity"},
89
+ "schema": { "$ref": "#/$defs/Opportunity" },
90
90
  "primary_key": ["id"],
91
91
  "relationships": [
92
- {"fields": ["account_id"], "target": "crm_accounts", "on_delete": "restrict"}
92
+ {
93
+ "fields": ["account_id"],
94
+ "target": "crm_accounts",
95
+ "on_delete": "restrict"
96
+ }
93
97
  ]
94
98
  }
95
99
  },
96
100
  "tools": {
97
101
  "list_accounts": {
98
- "input": {"type": "object", "properties": {}, "additionalProperties": false},
99
- "output": {"type": "array", "items": {"$ref": "#/$defs/Account"}},
102
+ "input": {
103
+ "type": "object",
104
+ "properties": {},
105
+ "additionalProperties": false
106
+ },
107
+ "output": { "type": "array", "items": { "$ref": "#/$defs/Account" } },
100
108
  "description": "List fixture accounts."
101
109
  }
102
110
  }
@@ -134,19 +142,19 @@ document; a refusal leaves the file untouched. Source:
134
142
 
135
143
  ## Supported payload language
136
144
 
137
- | Construct | `gateway-world/1` semantics |
138
- | --- | --- |
139
- | Types | `object`, `array`, `string`, `integer`, `number`, `boolean`, `null`; an explicit type or composition is required for each payload schema. |
140
- | Nullability | A single type, or an array containing one type plus `null`; broader unions use `anyOf`. Optionality is independent of nullability. |
141
- | Objects | `properties`, `required`, `additionalProperties` as a boolean or a schema (a schema types a map: every undeclared member holds a value of it). Required names must be declared properties. Nested objects may allow extra properties; entity roots must be closed (`additionalProperties: false`). |
142
- | Arrays | One required `items` schema, `minItems`, `maxItems`, `uniqueItems`; no tuple schemas. |
143
- | Values | `enum`, `const`; booleans are distinct from numeric values. |
144
- | Strings | `minLength`, `maxLength`; no regex/pattern constraints. |
145
- | Numbers | `minimum`, `maximum`, numeric `exclusiveMinimum`/`exclusiveMaximum`. Every numeric value must be finite and within ±(2^53−1), including nonintegral numbers. A `format` on an integer or number (`int64`, `int32`, `double`, as vendor OpenAPI documents write them) is dropped, not refused; it asserts nothing. |
146
- | Timestamps | Only `format: "date-time"`: RFC3339 profile with a real date, seconds, and explicit `Z` or numeric offset; fractional seconds and lowercase `t`/`z` accepted; leap seconds rejected. |
147
- | Reuse | `$defs`, `$ref` to `#/$defs/Name` or `base.json#/$defs/Name` (including valid JSON Pointer paths beneath `$defs`). A definition may reference itself, directly or through a property, item or composition branch (Brex's `ExpandableExpense` holds an `ExpandableExpense`): the first occurrence is inlined in full, the nested occurrence keeps only the target's bare `type` (annotated with a `$comment` naming the target), so it is validated as that type and stored as JSON. A cycle of bare references or compositions, with no direct `type` to keep, is refused. |
148
- | Composition | `allOf`, `anyOf`, `oneOf`, with nonempty branch lists. These validate values; they do not merge entity declarations or implement inheritance. |
149
- | Documentation | `title`, `description`, `$comment`, `examples`, `deprecated`, `readOnly`, `writeOnly` have no mutation/access-control semantics. `default` must validate but is never inserted. |
145
+ | Construct | `gateway-world/1` semantics |
146
+ | ------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
147
+ | Types | `object`, `array`, `string`, `integer`, `number`, `boolean`, `null`; an explicit type or composition is required for each payload schema. |
148
+ | Nullability | A single type, or an array containing one type plus `null`; broader unions use `anyOf`. Optionality is independent of nullability. |
149
+ | Objects | `properties`, `required`, `additionalProperties` as a boolean or a schema (a schema types a map: every undeclared member holds a value of it). Required names must be declared properties. Nested objects may allow extra properties; entity roots must be closed (`additionalProperties: false`). |
150
+ | Arrays | One required `items` schema, `minItems`, `maxItems`, `uniqueItems`; no tuple schemas. |
151
+ | Values | `enum`, `const`; booleans are distinct from numeric values. |
152
+ | Strings | `minLength`, `maxLength`; no regex/pattern constraints. |
153
+ | Numbers | `minimum`, `maximum`, numeric `exclusiveMinimum`/`exclusiveMaximum`. Every numeric value must be finite and within ±(2^53−1), including nonintegral numbers. A `format` on an integer or number (`int64`, `int32`, `double`, as vendor OpenAPI documents write them) is dropped, not refused; it asserts nothing. |
154
+ | Timestamps | Only `format: "date-time"`: RFC3339 profile with a real date, seconds, and explicit `Z` or numeric offset; fractional seconds and lowercase `t`/`z` accepted; leap seconds rejected. |
155
+ | Reuse | `$defs`, `$ref` to `#/$defs/Name` or `base.json#/$defs/Name` (including valid JSON Pointer paths beneath `$defs`). A definition may reference itself, directly or through a property, item or composition branch (Brex's `ExpandableExpense` holds an `ExpandableExpense`): the first occurrence is inlined in full, the nested occurrence keeps only the target's bare `type` (annotated with a `$comment` naming the target), so it is validated as that type and stored as JSON. A cycle of bare references or compositions, with no direct `type` to keep, is refused. |
156
+ | Composition | `allOf`, `anyOf`, `oneOf`, with nonempty branch lists. These validate values; they do not merge entity declarations or implement inheritance. |
157
+ | Documentation | `title`, `description`, `$comment`, `examples`, `deprecated`, `readOnly`, `writeOnly` have no mutation/access-control semantics. `default` must validate but is never inserted. |
150
158
 
151
159
  Constraints require applicable explicit types (`minLength` requires string).
152
160
  Integers accept integral numbers such as `1.0` without rewriting payloads;
@@ -235,26 +243,34 @@ false, zero, empty values, null, and absence without coercion/default insertion.
235
243
 
236
244
  Import `SQLiteStore` from `gatewaysdk.environment.schema`:
237
245
 
238
- | API | Behavior |
239
- | --- | --- |
240
- | `create(path, contract)` / `open(path, contract)` | Context-managed store; creation refuses an existing file; opening checks the exact lock, actual DDL, and state. |
241
- | `get(entity, key)` / `list(entity)` | Returns decoded payloads; a missing key returns `None`; list order follows encoded primary keys. |
242
- | `insert(entity, value, source=...)` | Validates a complete record; returns its payload. |
243
- | `update(entity, key, patch, source=...)` | Shallow-merges the patch, validates the complete result, and forbids primary-key changes; returns the resulting payload. |
244
- | `delete(entity, key, source=...)` | Explicit deletion; missing rows and unresolved dependents reject. |
245
- | `apply_batch(batch)` | Atomic mutations plus durable provenance and retry handling. |
246
- | `transaction()` | Groups operations; nested operations use savepoints; outer completion audits state and commits. |
247
- | `lineage(entity, key)` | All recorded contributions for the key, including batch target/mapping metadata, mutation index, and source. |
248
- | `query(sql, params=())` | Read-only SQL over domain tables and `benchmark_runtime_parameters`. |
249
- | `ingest(source=..., mapping=..., batch_id=None, limit=50)` | Schema-gated raw SQL session; see [Raw ingestion](#raw-ingestion-schema-gated-sql). |
250
- | `audit()` / `snapshot(destination)` / `close()` | Validate state, back up committed state to a new file, release resources. Snapshot/close refuse an active transaction. |
246
+ | API | Behavior |
247
+ | ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ |
248
+ | `create(path, contract)` / `open(path, contract)` | Context-managed store; creation refuses an existing file; opening checks the exact lock, actual DDL, and state. |
249
+ | `get(entity, key)` / `list(entity)` | Returns decoded payloads; a missing key returns `None`; list order follows encoded primary keys. |
250
+ | `insert(entity, value, source=...)` | Validates a complete record; returns its payload. |
251
+ | `update(entity, key, patch, source=...)` | Shallow-merges the patch, validates the complete result, and forbids primary-key changes; returns the resulting payload. |
252
+ | `delete(entity, key, source=...)` | Explicit deletion; missing rows and unresolved dependents reject. |
253
+ | `apply_batch(batch)` | Atomic mutations plus durable provenance and retry handling. |
254
+ | `transaction()` | Groups operations; nested operations use savepoints; outer completion audits state and commits. |
255
+ | `lineage(entity, key)` | All recorded contributions for the key, including batch target/mapping metadata, mutation index, and source. |
256
+ | `query(sql, params=())` | Read-only SQL over domain tables and `benchmark_runtime_parameters`. |
257
+ | `ingest(source=..., mapping=..., batch_id=None, limit=50)` | Schema-gated raw SQL session; see [Raw ingestion](#raw-ingestion-schema-gated-sql). |
258
+ | `audit()` / `snapshot(destination)` / `close()` | Validate state, back up committed state to a new file, release resources. Snapshot/close refuse an active transaction. |
259
+
260
+ A tool handler (`tools/handlers.py`) does not see this class. `from gateway_env import store` is a
261
+ facade over it admitting exactly `get, list, list_where, insert, update, delete, apply_batch,
262
+ lineage, counts` — the one list in `environment/schema/store_contract.py` (`STORE_METHODS`, read off
263
+ `SQLiteStore` so it cannot drift). `gateway worlds schema check` refuses any other `store.<name>`
264
+ statically at file:line with that list; a hosted call into a handler whose own code raises
265
+ `AttributeError`/`TypeError`/`NameError` answers 400 `{error: "handler", detail, handler}`, any other
266
+ handler failure 500 `{error: "engine", detail, handler}` — never a bare exception string. There is
267
+ no `upsert`: `get`, then `insert` or `update`.
251
268
 
252
269
  Connections enable foreign keys, bind values, and quote identifiers. `World` copies the validated
253
270
  template for branches/reset; snapshots require the exact contract on reopen.
254
271
  Reset rejects a changed contract.
255
272
 
256
- `gateway_env.store` exposes `get`, `list`, `insert`, `update`, `delete`,
257
- `apply_batch`, and `lineage` bound to the currently executing schema world.
273
+ `gateway_env.store` is that facade, bound to the currently executing schema world.
258
274
  Lifecycle/transaction ownership stays with `World`. The legacy `db.one/all` and
259
275
  injected `conn.execute` remain read-only in schema mode. Raw SQL writes,
260
276
  `executescript`, direct commits, writable PRAGMAs, and mutating CTEs are denied.
@@ -339,15 +355,15 @@ with world.store.ingest(
339
355
  print(conn.result) # {"status": "applied", "batch": ..., "inserted": n, ...}
340
356
  ```
341
357
 
342
- | Inside a session | Rule |
343
- | --- | --- |
344
- | Domain tables | `INSERT`, `UPDATE`, `DELETE` allowed; `gateway_key(entity, payload)` computes `_key` in SQL and returns NULL for a payload without its primary key, so the NOT NULL constraint rejects the row with the reason attached. |
345
- | `TEMP` tables, views, indexes | Allowed for staging; dropped when the session ends. |
346
- | `conn.script(sql)` | Runs a multi-statement script one statement at a time. `executescript` is not exposed because it would commit early. |
347
- | Transaction control, `PRAGMA`, `ATTACH`, persistent DDL | Denied by the authorizer (`ingest_sql`). The store owns the transaction. |
348
- | `_gateway_*`, `benchmark_runtime_context`, `benchmark_conversation_events` | Neither readable nor writable. `benchmark_runtime_parameters` stays readable. |
349
- | Store APIs (`insert`, `apply_batch`, `transaction`, `snapshot`) | Refused while a session is open; a session cannot start inside a tool or batch transaction. |
350
- | `gateway_env.store` | Has no `ingest`; tool handlers keep the validated API only. |
358
+ | Inside a session | Rule |
359
+ | -------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
360
+ | Domain tables | `INSERT`, `UPDATE`, `DELETE` allowed; `gateway_key(entity, payload)` computes `_key` in SQL and returns NULL for a payload without its primary key, so the NOT NULL constraint rejects the row with the reason attached. |
361
+ | `TEMP` tables, views, indexes | Allowed for staging; dropped when the session ends. |
362
+ | `conn.script(sql)` | Runs a multi-statement script one statement at a time. `executescript` is not exposed because it would commit early. |
363
+ | Transaction control, `PRAGMA`, `ATTACH`, persistent DDL | Denied by the authorizer (`ingest_sql`). The store owns the transaction. |
364
+ | `_gateway_*`, `benchmark_runtime_context`, `benchmark_conversation_events` | Neither readable nor writable. `benchmark_runtime_parameters` stays readable. |
365
+ | Store APIs (`insert`, `apply_batch`, `transaction`, `snapshot`) | Refused while a session is open; a session cannot start inside a tool or batch transaction. |
366
+ | `gateway_env.store` | Has no `ingest`; tool handlers keep the validated API only. |
351
367
 
352
368
  Key generation, audits, and reopening use the same strict JSON parser as authored
353
369
  inputs: duplicate object members reject rather than silently selecting a value.
@@ -409,9 +425,53 @@ re-stamps an empty journal (`mutations == []`) onto the new domain, whatever
409
425
  written against). The target is pure to compute:
410
426
  `from gatewaysdk.environment.schema.compiler import build; build(world).target`.
411
427
 
428
+ ### From an OpenAPI document (`--from-openapi`)
429
+
430
+ A vendor's Swagger 2.0 / OpenAPI 3.0.x / 3.1 document (JSON or YAML) becomes a world without
431
+ hand-transcription:
432
+
433
+ ```sh
434
+ gateway worlds schema init ./petstore --from-openapi ./petstore.yaml
435
+ gateway worlds schema init ./gh --from-openapi https://example.com/github.json --include '/repos/{owner}/{repo}/issues*'
436
+ gateway worlds schema init ./crm --from-openapi drive:specs/crm.json --plan-only
437
+ ```
438
+
439
+ Every object schema under `definitions` / `components.schemas` with an id-like key (`id`,
440
+ `<name>Id`, `<name>_id`, a `readOnly` field an `x-` extension marks unique, else the first
441
+ required scalar) becomes an entity
442
+ `$def` plus `x-gateway.entities[<snake_plural>]` with `primary_key`; keyless nested objects stay
443
+ inline `$defs`; `allOf` is merged, `oneOf`/`anyOf` kept as unions the validator accepts; names
444
+ are snake_cased, bounded to 64 characters, and the original recorded in `schema/provenance.json`. Every path+method becomes
445
+ an operation in `connector.toml`: `operationId` snake_cased and bounded to 64 characters (a
446
+ duplicate gets a numeric suffix and a warning), path and query parameters declared, an array
447
+ or `{data: [...]}` response an entity list route (declared query parameters that name entity
448
+ fields filter it), an object response the entity's read/create/update/delete by method; a
449
+ response that is no entity stays a passthrough operation (no `entity`, `501` when called).
450
+ `HEAD`/`OPTIONS`/`TRACE` (anything outside `GET POST PUT PATCH DELETE`), a path that repeats a
451
+ placeholder (`/a/{id}/{id}`), and operations
452
+ `--include` / `--exclude` leave out (a path glob or a tag,
453
+ repeatable) are listed under `skipped` / `excluded` with the reason. Refusals carry a JSON
454
+ pointer into the document: an undeclared path parameter, an unknown or external `$ref`, a
455
+ cyclic entity root, a document without `paths`, a document over 64 MiB.
456
+
457
+ The command prints the plan (`entities`, `operations`, `skipped`, `warnings`, the document's
458
+ version, title and sha256) and, unless `--plan-only`, writes the tree through the same writer as
459
+ a template `init`, so a rejected import leaves nothing behind, then lists the `next` commands:
460
+ `schema compile`, `schema check`, `data import`, `serve` / `call`, `sandbox commit` or `worlds
461
+ create`. `schema/provenance.json` records the source label (credentials and query strings
462
+ stripped), its sha256 and size, the importer version and each entity's and operation's source
463
+ pointer. `--connector` and `--with-example-tool` do not combine with it. The document's bytes
464
+ are read where the command runs: a local path directly; `drive:<path>` from the project drive
465
+ with the caller's key, in memory; an `http(s)://` URL through a private-address guard (no
466
+ loopback, RFC 1918, link-local or metadata host, every redirect re-checked, five at most) — the
467
+ platform never fetches a caller-named URL, and the sandbox's `run_gateway_cli` accepts only a
468
+ `/workspace` path. `gateway worlds connector conform <world> --openapi <same document>` then
469
+ checks the served world back against the document it came from.
470
+
412
471
  **Build from connector** for Slack creates a durable populated world job in
413
- Gateway. Other registered source kinds and **Custom connector world** retain the
414
- SDK init, check, local-run and `gateway bench push` sequence. SDK template
472
+ Gateway. Other registered source kinds show their CLI twin under **Use the CLI instead**:
473
+ `gateway worlds schema init`, `schema check`, then `gateway worlds create <slug> --from <dir>`.
474
+ A blank (custom) contract starts from the Connectors page's **New connector** form. SDK template
415
475
  availability does not register a platform source kind or supply a live mapper;
416
476
  Teams remains schema-only. The SDK checks the contract before push, and the
417
477
  shared publication validator checks actual stored bundle bytes before the world
@@ -480,7 +540,7 @@ the version commit transaction. Capture is bounded and read-only; world tools
480
540
  mutate only the copied world. See [Environment Files](./environment-files.md#populated-slack-world-jobs)
481
541
  and [Slack capture permissions](../integrations/slack.md#structured-world-capture).
482
542
 
483
- The server needs Python 3.11+ and the trusted SDK schema runtime.
543
+ The server needs Python 3.12+ and the trusted SDK schema runtime.
484
544
  `GATEWAY_SCHEMA_PYTHON` defaults to `python3`; `GATEWAY_SCHEMA_RUNTIME_PATH`
485
545
  overrides runtime discovery. Production images install it at `/opt/gateway-schema`;
486
546
  local development can discover the checkout. Compiler children receive no
@@ -499,13 +559,14 @@ as a credential-free subprocess (`GATEWAY_SCHEMA_PYTHON`, `GATEWAY_SCHEMA_RUNTIM
499
559
  64 MiB bundle, 2000 files, 60 s) through
500
560
  `gateway/packages/shared/src/server/services/worldSchemaRuntime.ts`. Commands:
501
561
 
502
- | Command | Used by | Effect |
503
- | --- | --- | --- |
504
- | `validate`, `validate-archive` | publication | Verify SHA-256, regenerated artifacts and exact lock of an existing tree/archive. |
505
- | `init-template` | `create_connector_world`, `create_custom_world --template` | Scaffold a connector template into a tree. |
506
- | `compile` | `create_custom_world`, `ingest_to_world` on schema worlds | Compile an authored tree (`compileWorldFiles`): returns the full entry map with generated artifacts + lock, the schema report and which files were added/retargeted. |
507
- | `apply-batch` | `ingest_records_to_world` | Apply `insert\|update\|delete` mutations as one batch (`applyWorldBatch`): validated row by row against the compiled schema, appended as `data/batches/<batchId>.json` with `source` provenance; replaying the same `batchId` is a no-op. |
508
- | `build-slack`, `refresh-slack` | connector worlds | Slack capture → batches (durable job). |
562
+ | Command | Used by | Effect |
563
+ | ------------------------------ | ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
564
+ | `validate`, `validate-archive` | publication | Verify SHA-256, regenerated artifacts and exact lock of an existing tree/archive. |
565
+ | `init-template` | `create_connector_world`, `create_custom_world --template` | Scaffold a connector template into a tree. |
566
+ | `init-openapi` | `worlds schema init --from-openapi` (both CLIs) | `{source, document (base64), destination, plan_only?, include?, exclude?, name?}` → the import plan and, when written, the compiled tree as `files[]` (at most 2000). The bridge never opens a caller-named path or URL; the CLI stages the bytes. |
567
+ | `compile` | `create_custom_world`, `ingest_to_world` on schema worlds | Compile an authored tree (`compileWorldFiles`): returns the full entry map with generated artifacts + lock, the schema report and which files were added/retargeted. |
568
+ | `apply-batch` | `ingest_records_to_world` | Apply `insert\|update\|delete` mutations as one batch (`applyWorldBatch`): validated row by row against the compiled schema, appended as `data/batches/<batchId>.json` with `source` provenance; replaying the same `batchId` is a no-op. |
569
+ | `build-slack`, `refresh-slack` | connector worlds | Slack capture → batches (durable job). |
509
570
 
510
571
  Every agent write therefore passes through the schema: `ingest_to_world` recompiles a schema world before it becomes head, and `ingest_records_to_world` refuses invalid rows with the field path instead of publishing a partial batch.
511
572
 
@@ -533,13 +594,13 @@ reordering preserves locks; authored list order remains in input hashes, even
533
594
  where generated artifacts normalize required/type/constraint lists. Key-component
534
595
  and mutation order remain meaningful. No broader equivalence is promised.
535
596
 
536
- | Version axis | Stored identity and compatibility |
537
- | --- | --- |
538
- | Language/annotations | `gateway-world/1`, `x-gateway.version = 1`; unknown versions reject. |
539
- | Compiler/storage | `gateway-world-compiler/1`, `sqlite-json/1`; exact regenerated lock required. |
540
- | Domain | Root `$id`, `domain_version`, and compiled entity-contract hash. Batches must match all three; changed domains require regeneration and explicit state rebuilding. |
541
- | Dependencies | Every resolved document's `$id` and normalized authored hash are locked. Missing, changed, or incompatible dependencies require an explicit fix/regeneration. |
542
- | Mapping | Batch `mapping.id/version`, persisted with lineage; a new mapping does not change the domain or package contract. |
597
+ | Version axis | Stored identity and compatibility |
598
+ | --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
599
+ | Language/annotations | `gateway-world/1`, `x-gateway.version = 1`; unknown versions reject. |
600
+ | Compiler/storage | `gateway-world-compiler/1`, `sqlite-json/1`; exact regenerated lock required. |
601
+ | Domain | Root `$id`, `domain_version`, and compiled entity-contract hash. Batches must match all three; changed domains require regeneration and explicit state rebuilding. |
602
+ | Dependencies | Every resolved document's `$id` and normalized authored hash are locked. Missing, changed, or incompatible dependencies require an explicit fix/regeneration. |
603
+ | Mapping | Batch `mapping.id/version`, persisted with lineage; a new mapping does not change the domain or package contract. |
543
604
  | Package/tool behavior | `x-gateway.package.id/version`, tool-contract hash, and generated artifact hashes. Tool/package changes require regeneration; old state cannot open under a different lock. Authored handler bytes remain bound by package/bundle versioning, not compiled schema semantics. |
544
605
 
545
606
  Opening requires exact lock equality, including package/tool/input hashes; no
@@ -78,8 +78,31 @@ person so they stay consistent; `role: "public"` is never replaced. Every varian
78
78
  tracked (bare surname, handle spellings, `(at)` emails, phone formats, percent-encoding, a
79
79
  world's `sha256:` stored form), and in any case: `Alexr` or `ALEXR` for a seeded `alexr` is swapped
80
80
  where it stands alone, and the stand-in keeps the case it arrived in. The door swaps and never
81
- blocks. What slips through is counted (`leak_through_inbound`), and a real value in the outgoing
82
- report fails the run (`leaks_outbound`).
81
+ blocks. A leak is about a **case subject**, an entity the task seeded: what slips through is counted
82
+ (`leak_through_inbound`), and a subject's real value in the outgoing report fails the run
83
+ (`leaks_outbound`). Someone who only turned up in a result and got a stand-in on sight is a bystander:
84
+ swapped the same way, counted apart (`bystander_leak_through_inbound`, `bystander_leaks_outbound`),
85
+ never a failure.
86
+
87
+ ## Choose the stand-in yourself
88
+
89
+ When a case must look a particular way to the model, or the sources spell a real value in
90
+ ways the door cannot guess, declare it on the seed instead of relying on the pools:
91
+
92
+ ```json
93
+ {"type": "username", "value": "<real handle>", "alias": "kmarsh7", "variants": ["<the forum's misspelling>"]}
94
+ {"type": "person_name", "value": "<real name>", "alias": "Kim Marsh"}
95
+ ```
96
+
97
+ `alias` is the stand-in the model sees; every form the kind has (name order, `(at)` emails, phone
98
+ formats, case) is paired from it. `variants` are other spellings of the real value; each is paired
99
+ with the same stand-in, so a misspelling in a forum post or a phone written without its country
100
+ digit is swapped like the value itself. Leave both out and the door generates a persona.
101
+
102
+ When the case must not show a generated persona at all (a model that recognises the reserved IP
103
+ range or the made-up domains stops trusting the data), set `declared_only = true` under `[alias]`:
104
+ every private seed then needs its own `alias`, a seed without one is refused before the run, and
105
+ `mint_inbound` stays off.
83
106
 
84
107
  ## Identities that arrive, not seeded
85
108
 
@@ -175,6 +198,8 @@ gateway hooks audit runs/<run id>/ --json # the
175
198
  ```
176
199
 
177
200
  `fields.json` is `{"*": {"full_name": "person_name", "email": "email"}}`, or keyed by tool name.
201
+ A directory with neither `raw.jsonl` nor `shown.jsonl` (a `task run --out-dir`, say) never ran under
202
+ hooks: the audit says so and exits 2 rather than reading the run's own `run.json` as hooks state.
178
203
 
179
204
  It knows a value is real from three places, strongest first: your legend; every value of a field
180
205
  whose kind you declared (a name in a vendor row that reappears in a web snippet is caught for
@@ -15,6 +15,8 @@ Companion to [SKILL.md](SKILL.md). The contract is fixed in
15
15
  8. [MCP payloads](#mcp-payloads)
16
16
  9. [Chunks, batches and refusal records](#chunks-batches-and-refusal-records)
17
17
  10. [Extract adapters](#extract-adapters)
18
+ 11. [CSV and TSV cells](#csv-and-tsv-cells)
19
+ 12. [Data snapshots](#data-snapshots)
18
20
 
19
21
  ## Shapes
20
22
 
@@ -46,6 +48,16 @@ gateway worlds data import ./world users.json --entity users
46
48
  gateway worlds data import ./world users.jsonl --entity users
47
49
  ```
48
50
 
51
+ **CSV/TSV with `--entity NAME`.** A `.csv`/`.tsv` (or `.gz`) whose header row names the fields;
52
+ every cell is cast by the entity field's schema type (integer, number, boolean, array/object as
53
+ JSON, null for an empty nullable cell) — [CSV and TSV cells](#csv-and-tsv-cells). `--delimiter`
54
+ picks the separator. Not accepted from `drive:` (import it from the machine that holds it, or
55
+ `worlds sandbox import` it from a sandbox).
56
+
57
+ ```bash
58
+ gateway worlds data import ./world tickets.csv --entity hd_tickets
59
+ ```
60
+
49
61
  A file that mixes shapes is refused. Several files on one command are applied as one batch:
50
62
  all or nothing (`--per-chunk` on a platform import relaxes that to one chunk at a time). Files
51
63
  are streamed, so size is not a concern: pass `-` for stdin, `.gz` files are gunzipped.
@@ -241,3 +253,53 @@ rows add missing fields; `upsert`: last wins); the world's `[redact]` policy app
241
253
  by element, `round` rounded). Output is tagged JSONL; stderr gets
242
254
  `{"read": N, "rows": M, "unknownFields": {name: n}, "redacted": {"dropped": n, "hashed": n}}`.
243
255
  The fixtures under `gatewaysdk/fixtures/world-data/extract/` are the worked examples.
256
+
257
+ ## CSV and TSV cells
258
+
259
+ A `.csv`/`.tsv` (optionally `.gz`) is read as a stream — memory is bounded at any size — and
260
+ split by `--delimiter` (one character, `\t` for a tab; default `,`, tab for a `.tsv`). The header
261
+ row names the fields; `--entity` says which entity they fill. Every cell is cast by that field's
262
+ schema type, the same rule in the TypeScript CLI (`worldData/cast.ts`) and the Python package
263
+ (`schema/cast.py`), pinned by `gatewaysdk/sdk/tests/conformance/csv_cast.json`:
264
+
265
+ | Field type | Cell | Value |
266
+ | --- | --- | --- |
267
+ | `integer` | `42`, `-7` | the number; `4.5` or `abc` is refused |
268
+ | `number` | `12.5`, `1e3`, `3` | the number (always a float) |
269
+ | `boolean` | `true`/`false`/`1`/`0`, any case | the boolean; `yes` is refused |
270
+ | `array` / `object` | JSON text of that shape (quote it in the CSV: `"[""a"",""b""]"`) | the parsed value |
271
+ | `string` | anything | the text as given |
272
+ | `integer \| string` (union) | each member tried in order, `string` last | the first that fits |
273
+ | any, empty cell | `` | null when the type admits null; left out when the field is optional; `""` for a required string; refused otherwise |
274
+
275
+ A cell that fits none is a row-level refusal naming the field and the cell —
276
+ `/taxi_trip/3: fare = "n/a" is not a number` — with the usual pointer; nothing is written. A
277
+ column the schema does not name stays text and the gate refuses it as an unknown field. A CSV
278
+ on the drive (`drive:<path>`) is still refused: import it from the machine that holds it, or
279
+ through `worlds sandbox import` when it sits in a sandbox.
280
+
281
+ ```bash
282
+ gateway worlds data import ./world trips.csv --entity taxi_trip
283
+ gateway worlds data import ./world trips.tsv --entity taxi_trip # tab by name
284
+ gateway worlds data import acme export.txt.csv --entity rows --delimiter ';'
285
+ ```
286
+
287
+ ## Data snapshots
288
+
289
+ Why a served world starts in under a second.
290
+
291
+ A schema world's validated database is kept, not rebuilt. `gateway worlds schema compile`
292
+ builds `db/state.sqlite` (the default snapshot: `data/initial.json` applied through the
293
+ contract) and `scenarios/<name>/state.sqlite` (the default with the scenario's overlays),
294
+ each with a `state.json` sidecar `{contractHash, batchDigest, entityCounts, builtAt,
295
+ runtime, fileSha256}`; the same key is never rebuilt, `schema check` verifies the sidecars,
296
+ and a data batch the contract refuses leaves no snapshot behind. `gateway worlds serve` and
297
+ the api bundle open the snapshot (copy + `SQLiteStore.open`, no row replay) and reset by
298
+ copying it again; without one the JSON is replayed as before. A hosted pod gets
299
+ `WORLD_SNAPSHOT_URL` (presigned GET, `.gz` or `Content-Encoding: gzip` is gunzipped),
300
+ `WORLD_SNAPSHOT_SHA256` and `WORLD_SNAPSHOT_DIGEST`; the server downloads, verifies, writes
301
+ the sidecar and only then serves; a mismatch exits 2 naming the digest. `/__world/state`
302
+ reports `"snapshot": {batchDigest, contractHash}` (or null) and `/__world/seed` takes the
303
+ overlay form `{"base": digest, "rows": {...}, "mode": append|replace}` beside the legacy
304
+ whole-state body. Snapshot files are build outputs: gitignore `db/state.sqlite`,
305
+ `db/state.json`, `scenarios/*/state.*`.