@withgateway/sdk 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -6
- package/dist/cli.js +4372 -2265
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +4362 -2256
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +259 -58
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +249 -50
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
- package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
- package/dist/index.d.mts +5 -5
- package/dist/index.d.ts +5 -5
- package/dist/index.js +2240 -789
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2150 -739
- package/dist/index.mjs.map +1 -1
- package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
- package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
- package/dist/traces/index.d.mts +1 -1
- package/dist/traces/index.d.ts +1 -1
- package/dist/traces/index.js +253 -52
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +243 -44
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +2 -2
- package/dist/tracing/index.d.ts +2 -2
- package/dist/tracing/index.js +75 -13
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +75 -13
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +893 -482
- package/dist/worlds/index.d.ts +893 -482
- package/dist/worlds/index.js +1968 -669
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1960 -662
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +2 -27
- package/runtime/gatewaysdk/environment/_materialize.py +48 -88
- package/runtime/gatewaysdk/environment/_world.py +155 -237
- package/runtime/gatewaysdk/environment/backing.py +133 -0
- package/runtime/gatewaysdk/environment/core.py +2 -17
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
- package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
- package/runtime/gatewaysdk/environment/runtime.py +8 -13
- package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
- package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +347 -892
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
- package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
- package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +60 -216
- package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
- package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
- package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
- package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
- package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
- package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
- package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
- package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
- package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
- package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
- package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
- package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
- package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +58 -170
- package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
- package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
- package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
- package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
- package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
- package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
- package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
- package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
- package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
- package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
- package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
- package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
- package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
- package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
- package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
- package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
- package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
- package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
- package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
- package/runtime/gatewaysdk/environment/score.py +57 -0
- package/runtime/gatewaysdk/environment/secret_values.py +36 -0
- package/runtime/gatewaysdk/environment/world.py +43 -4
- package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/skills/connector-schema-authoring/SKILL.md +49 -16
- package/skills/connector-schema-authoring/connector-worlds.md +142 -11
- package/skills/connector-schema-authoring/world-schema.md +103 -51
- package/skills/world-data-ingestion/ROWS.md +39 -9
- package/skills/world-data-ingestion/SKILL.md +27 -27
- package/skills/worlds-getting-started/SKILL.md +33 -33
- package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
|
@@ -4,15 +4,16 @@
|
|
|
4
4
|
|
|
5
5
|
## Scope and runtime support
|
|
6
6
|
|
|
7
|
-
`gateway-world/1` makes
|
|
7
|
+
`gateway-world/1` makes a mock-tools world's database state an executable
|
|
8
|
+
contract, stored by the `WorldDatabase` engine the manifest names (SQLite today). One
|
|
8
9
|
authored JSON Schema document declares payloads, entities, keys, relationships,
|
|
9
10
|
and tool shapes. A stdlib Python compiler generates artifacts and a lock;
|
|
10
11
|
store/world transactions enforce data and tool contracts.
|
|
11
12
|
|
|
12
13
|
The bounded JSON Schema 2020-12 compiler enforces contracts; connector handlers
|
|
13
14
|
implement vendor behavior. Slack now has a finite executable model and a
|
|
14
|
-
structured capture-to-batch mapper. Automatic migrations
|
|
15
|
-
|
|
15
|
+
structured capture-to-batch mapper. Automatic migrations and tenant overlays
|
|
16
|
+
remain outside this contract; SQLite is the only engine shipped today.
|
|
16
17
|
Ingestion code may write raw SQL through a [schema-gated
|
|
17
18
|
session](#raw-ingestion-schema-gated-sql), and connector
|
|
18
19
|
[templates](#connector-templates-and-init) initialize new worlds.
|
|
@@ -55,7 +56,7 @@ serve = ["list_accounts"]
|
|
|
55
56
|
| Supporting scope, provenance, fidelity and capability metadata | Descriptive resources; not payload schemas or proof of executed checks. |
|
|
56
57
|
| `data/initial.json` | Authored or mapper-produced initial mutation batch. |
|
|
57
58
|
| `tools/*.py`, `verifiers/*.sql` | Authored behavior and read-only grading. |
|
|
58
|
-
| `db/schema.sql` | Generated SQLite
|
|
59
|
+
| `db/schema.sql` | Generated storage DDL for the world's engine (SQLite today). |
|
|
59
60
|
| `schema/runtime.json` | Generated resolved contract; not authored. |
|
|
60
61
|
| `schema/README.md` | Generated entity and tool documentation. |
|
|
61
62
|
| `tool-schemas.json` | Generated `tools` map with `input_schema`, `output_schema`, and `description`. |
|
|
@@ -160,7 +161,7 @@ stays. Source: `gatewaysdk/sdk-ts/src/cli/runtime/scaffoldEntityRemove.ts`.
|
|
|
160
161
|
| Strings | `minLength`, `maxLength`; no regex/pattern constraints. |
|
|
161
162
|
| Numbers | `minimum`, `maximum`, numeric `exclusiveMinimum`/`exclusiveMaximum`. Every numeric value must be finite and within ±(2^53−1), including nonintegral numbers. A `format` on an integer or number (`int64`, `int32`, `double`, as vendor OpenAPI documents write them) is dropped, not refused; it asserts nothing. |
|
|
162
163
|
| Timestamps | Only `format: "date-time"`: RFC3339 profile with a real date, seconds, and explicit `Z` or numeric offset; fractional seconds and lowercase `t`/`z` accepted; leap seconds rejected. |
|
|
163
|
-
| Reuse | `$defs`, `$ref` to `#/$defs/Name` or `base.json#/$defs/Name` (including valid JSON Pointer paths beneath `$defs`). A definition may reference itself, directly or through a property, item or composition branch (Brex's `ExpandableExpense` holds an `ExpandableExpense`): the first occurrence is inlined in full, the nested occurrence keeps only the target's bare `type` (annotated with a `$comment` naming the target), so it is validated as that type and stored as JSON. A cycle of bare references or compositions, with no direct `type` to keep, is refused. |
|
|
164
|
+
| Reuse | `$defs`, `$ref` to `#/$defs/Name` or `base.json#/$defs/Name` (including valid JSON Pointer paths beneath `$defs`). A definition may reference itself, directly or through a property, item or composition branch (Brex's `ExpandableExpense` holds an `ExpandableExpense`): the first occurrence is inlined in full, the nested occurrence keeps only the target's bare `type` (annotated with a `$comment` naming the target), so it is validated as that type and stored as JSON. A cycle of bare references or compositions, with no direct `type` to keep, is refused. Compile inlines every reference while each generated file fits the 4 MB publication cap. When it would not (Google Docs: one body type reached along hundreds of paths, 24 MB inlined), each object or array definition is emitted once under the root's `$defs` as `{"$ref": "#/$defs/<file>.<Name>", "type": ...}`, a nested occurrence of a type in itself is a real reference (validated in full), scalar definitions stay inlined, and `schema/README.md` documents each definition once under Definitions. |
|
|
164
165
|
| Composition | `allOf`, `anyOf`, `oneOf`, with nonempty branch lists. These validate values; they do not merge entity declarations or implement inheritance. |
|
|
165
166
|
| Documentation | `title`, `description`, `$comment`, `examples`, `deprecated`, `readOnly`, `writeOnly` have no mutation/access-control semantics. `default` must validate but is never inserted. |
|
|
166
167
|
|
|
@@ -193,20 +194,25 @@ and inheritance/overrides are rejected. Reuse definitions by `$ref` and declare
|
|
|
193
194
|
each entity once in the root catalog.
|
|
194
195
|
|
|
195
196
|
Entity names use `[A-Za-z][A-Za-z0-9_]*`. Payload property names use
|
|
196
|
-
`[A-Za-z_][A-Za-z0-9_]*`, so
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
197
|
+
`[A-Za-z_][A-Za-z0-9_-]*`, so vendor fields such as `_links`, `_etag` and
|
|
198
|
+
`reference-count` keep the vendor's spelling. A dot or a slash is never part of a
|
|
199
|
+
name: nested fields are addressed by JSON Pointer. `_key`, `payload`, and the
|
|
200
|
+
`sqlite_`, `gateway_`, and `benchmark_` storage prefixes remain reserved
|
|
201
|
+
case-insensitively. Case-fold collisions are rejected. Domain prefixes such as
|
|
202
|
+
`crm_` avoid collisions when authoring one combined catalog.
|
|
203
|
+
|
|
204
|
+
Primary keys are ordered, nonempty lists of distinct, required, non-null
|
|
205
|
+
string/integer fields. A field is a direct property name or an RFC 6901 JSON
|
|
206
|
+
Pointer to a nested scalar, such as `/protocolSection/identificationModule/nctId`;
|
|
207
|
+
every object along the pointer must be required and non-null, so every row
|
|
208
|
+
carries its key. Unique keys may use direct field names or JSON Pointers such as
|
|
209
|
+
`/profile/email`; pointer escapes `~0` and `~1` are decoded. Their scalar leaves
|
|
210
|
+
may be optional or nullable: missing/null values do not conflict, while equal
|
|
211
|
+
present non-null values do. Composite keys work. Composition and duplicate key
|
|
212
|
+
declarations reject. Lookup objects contain exactly the primary-key fields, each
|
|
213
|
+
under its own name (the pointer itself for a nested field:
|
|
214
|
+
`{"/protocolSection/identificationModule/nctId": "NCT00000001"}`). Integral keys
|
|
215
|
+
share identity (`1` equals `1.0`) without rewriting payloads.
|
|
210
216
|
|
|
211
217
|
Relationships declare ordered `fields`, a target entity, and
|
|
212
218
|
`on_delete: "restrict"`, targeting its primary key with equal arity
|
|
@@ -232,15 +238,16 @@ Unresolved targets reject without fabricating placeholders. Mappers must choose
|
|
|
232
238
|
valid records or model external identifiers as ordinary fields without a relationship.
|
|
233
239
|
`ArtifactReference.logicalSourceId` identifies evidence, not universal entity keys.
|
|
234
240
|
|
|
235
|
-
##
|
|
241
|
+
## Storage representation and supported store API
|
|
236
242
|
|
|
237
|
-
|
|
243
|
+
The representation below is the SQLite engine's (the one shipped today). Tables
|
|
244
|
+
hold canonical `payload` JSON, a private encoded `_key`, and generated
|
|
238
245
|
scalar projections for verifiers and UNIQUE/FK constraints. A property projects
|
|
239
246
|
when it can only hold scalars: a scalar `type` (nullable or not), or a
|
|
240
247
|
composition whose non-null branches are all scalar, such as the nullable enum
|
|
241
248
|
reference `anyOf [{$ref Status}, {type: null}]`. Objects/arrays, and any
|
|
242
249
|
composition that may hold one, remain JSON only. SQL exposes booleans as SQLite
|
|
243
|
-
values; store reads restore JSON booleans. Query errors keep
|
|
250
|
+
values; store reads restore JSON booleans. Query errors keep the engine's message
|
|
244
251
|
(`no such column: status`), and the store disables SQLite's double-quoted
|
|
245
252
|
string-literal fallback, so a verifier naming a column that does not exist
|
|
246
253
|
fails the grade with the statement instead of comparing two literals and
|
|
@@ -249,11 +256,14 @@ Missing and null both project to SQL NULL but remain distinct in `payload`;
|
|
|
249
256
|
grade that distinction with JSON presence/type queries. Validation preserves
|
|
250
257
|
false, zero, empty values, null, and absence without coercion/default insertion.
|
|
251
258
|
|
|
252
|
-
Import `
|
|
259
|
+
Import `WorldDatabase` from `gatewaysdk.environment.schema`. `WorldDatabase.create` / `open` pick
|
|
260
|
+
the engine the contract was compiled for (its lock's `target`; SQLite today, `SQLiteWorldDatabase` in
|
|
261
|
+
`gatewaysdk.environment.schema.database`). Every method below lives on the base, which owns the
|
|
262
|
+
contract, the audit and the guard, so a later engine answers the same way.
|
|
253
263
|
|
|
254
264
|
| API | Behavior |
|
|
255
265
|
| ---------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------ |
|
|
256
|
-
| `create(path, contract)` / `open(path, contract)` | Context-managed store; creation refuses an existing file; opening checks the
|
|
266
|
+
| `create(path, contract)` / `open(path, contract)` | Context-managed store; creation refuses an existing file; opening checks the storage contract, actual DDL, and state. |
|
|
257
267
|
| `get(entity, key)` / `list(entity)` | Returns decoded payloads; a missing key returns `None`; list order follows encoded primary keys. |
|
|
258
268
|
| `insert(entity, value, source=...)` | Validates a complete record; returns its payload. |
|
|
259
269
|
| `update(entity, key, patch, source=...)` | Shallow-merges the patch, validates the complete result, and forbids primary-key changes; returns the resulting payload. |
|
|
@@ -268,15 +278,15 @@ Import `SQLiteStore` from `gatewaysdk.environment.schema`:
|
|
|
268
278
|
A tool handler (`tools/handlers.py`) does not see this class. `from gateway_env import store` is a
|
|
269
279
|
facade over it admitting exactly `get, list, list_where, insert, update, delete, apply_batch,
|
|
270
280
|
lineage, counts` — the one list in `environment/schema/store_contract.py` (`STORE_METHODS`, read off
|
|
271
|
-
`
|
|
281
|
+
`WorldDatabase` so it cannot drift). `gateway worlds schema check` refuses any other `store.<name>`
|
|
272
282
|
statically at file:line with that list; a hosted call into a handler whose own code raises
|
|
273
283
|
`AttributeError`/`TypeError`/`NameError` answers 400 `{error: "handler", detail, handler}`, any other
|
|
274
284
|
handler failure 500 `{error: "engine", detail, handler}` — never a bare exception string. There is
|
|
275
285
|
no `upsert`: `get`, then `insert` or `update`.
|
|
276
286
|
|
|
277
287
|
Connections enable foreign keys, bind values, and quote identifiers. `World` copies the validated
|
|
278
|
-
template for branches/reset;
|
|
279
|
-
|
|
288
|
+
template for branches/reset; a snapshot reopens under any contract with the storage contract its
|
|
289
|
+
rows were stored under (below). A live `World`'s reset rejects any changed lock: its tools are bound.
|
|
280
290
|
|
|
281
291
|
`gateway_env.store` is that facade, bound to the currently executing schema world.
|
|
282
292
|
Lifecycle/transaction ownership stays with `World`. The legacy `db.one/all` and
|
|
@@ -347,7 +357,10 @@ Provider authorization, capture lifetime/fences, and schema compilation remain s
|
|
|
347
357
|
Ingestion code, including code an agent writes in a sandbox, may write SQL
|
|
348
358
|
straight into a world. The contract, not the API, decides what lands: the
|
|
349
359
|
session commits only when every persisted row satisfies the schema, keys and
|
|
350
|
-
relationships, and otherwise rolls back as a whole.
|
|
360
|
+
relationships, and otherwise rolls back as a whole. Known gap: a `TEMP` table named like a
|
|
361
|
+
domain or `_gateway_*` table shadows it for that check and for the batch and lineage rows, so
|
|
362
|
+
an invalid write can commit unrecorded (open bug, pinned in the guard matrix, to be fixed by
|
|
363
|
+
#474). Never name staging tables after world tables.
|
|
351
364
|
|
|
352
365
|
```python
|
|
353
366
|
with world.store.ingest(
|
|
@@ -366,7 +379,7 @@ print(conn.result) # {"status": "applied", "batch": ..., "inserted": n, ...}
|
|
|
366
379
|
| Inside a session | Rule |
|
|
367
380
|
| -------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
|
368
381
|
| Domain tables | `INSERT`, `UPDATE`, `DELETE` allowed; `gateway_key(entity, payload)` computes `_key` in SQL and returns NULL for a payload without its primary key, so the NOT NULL constraint rejects the row with the reason attached. |
|
|
369
|
-
| `TEMP` tables, views
|
|
382
|
+
| `TEMP` tables, views | Allowed for staging; dropped when the session ends. A `TEMP` index is refused today (open bug, pinned in the guard matrix). |
|
|
370
383
|
| `conn.script(sql)` | Runs a multi-statement script one statement at a time. `executescript` is not exposed because it would commit early. |
|
|
371
384
|
| Transaction control, `PRAGMA`, `ATTACH`, persistent DDL | Denied by the authorizer (`ingest_sql`). The store owns the transaction. |
|
|
372
385
|
| `_gateway_*`, `benchmark_runtime_context`, `benchmark_conversation_events` | Neither readable nor writable. `benchmark_runtime_parameters` stays readable. |
|
|
@@ -381,7 +394,8 @@ carrying the code, schema/data paths, entity and encoded key. A rejected
|
|
|
381
394
|
session leaves state, lineage and its `batch_id` untouched. A committed session
|
|
382
395
|
is recorded as one batch (`kind: "ingest"`) with a lineage row per inserted,
|
|
383
396
|
updated or deleted key under the session's `source`, so raw ingestion keeps the
|
|
384
|
-
same provenance as `apply_batch`, attributed per key rather than per mutation
|
|
397
|
+
same provenance as `apply_batch`, attributed per key rather than per mutation
|
|
398
|
+
(except under the `TEMP`-shadow gap above, where the session commits with no batch or lineage).
|
|
385
399
|
Reusing a committed `batch_id` is an `idempotency_conflict`; raw sessions are
|
|
386
400
|
not replayable. Run `python samples/schema-worlds/ingest_example.py` for a
|
|
387
401
|
committed and a rejected session.
|
|
@@ -428,7 +442,9 @@ and a starter verifier but no tools unless `--with-example-tool`
|
|
|
428
442
|
holds only the recipe. Batches interchange only when the domain id, version and
|
|
429
443
|
hash match. Giving an authored world a new `$id` later is fine: `compile`
|
|
430
444
|
re-stamps an empty journal (`mutations == []`) onto the new domain, whatever
|
|
431
|
-
`$id` it named; a journal that holds rows
|
|
445
|
+
`$id` it named; a journal that holds rows follows a change that only adds to
|
|
446
|
+
the domain it was written under (a new entity, a new field that is not
|
|
447
|
+
required, another `domain_version`) and is refused otherwise, with the fix named
|
|
432
448
|
(re-import its rows against the new contract, or restore the schema it was
|
|
433
449
|
written against). The target is pure to compute:
|
|
434
450
|
`from gatewaysdk.environment.schema.compiler import build; build(world).target`.
|
|
@@ -444,17 +460,41 @@ gateway worlds schema init ./gh --from-openapi https://example.com/github.json -
|
|
|
444
460
|
gateway worlds schema init ./crm --from-openapi drive:specs/crm.json --plan-only
|
|
445
461
|
```
|
|
446
462
|
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
463
|
+
Entities are the rows the selected operations answer and take, read through the vendor's
|
|
464
|
+
envelope. A response that is an array of objects, or an object that declares no key and holds
|
|
465
|
+
exactly one array of objects (possibly behind objects holding nothing else, as Crossref's
|
|
466
|
+
`message` → `items`), lists that item schema. A single-row response is the row when it is some
|
|
467
|
+
list's item, declares a key, or names the field the route's own `{segment}` reads it by; else,
|
|
468
|
+
through a wrapper holding exactly one object, that object. A wrapper is never an entity: its path
|
|
469
|
+
becomes the route's dotted `envelope` (`message.items`) and the rest of it the route's `wrapper`
|
|
470
|
+
template, so the served world answers in the vendor's envelope again. An item's key is a
|
|
471
|
+
declared one (`id`, `<name>Id`, `<name>_id`, a `readOnly` field an `x-` extension marks unique),
|
|
472
|
+
else the field its single-row route reads it by: a top-level field spelled the vendor's way
|
|
473
|
+
(`/works/{doi}` → `DOI`), or one nested in the row's objects (never through an array or a
|
|
474
|
+
nullable object), which becomes a JSON Pointer key (`/studies/{nctId}` →
|
|
475
|
+
`/protocolSection/identificationModule/nctId`) and makes every object on that path required.
|
|
476
|
+
An item only lists carry falls back to its first required scalar, with a warning that rows
|
|
477
|
+
sharing it collapse. An item no planned route reads or writes is not an entity, and a schema
|
|
478
|
+
that is no item stays an inline `$def`; a document whose operations name no schema offers
|
|
479
|
+
every keyed schema instead. Each entity is a `$def` plus `x-gateway.entities[<snake_plural>]`
|
|
480
|
+
with `primary_key`; `allOf` is merged, `oneOf`/`anyOf` kept as unions the validator accepts;
|
|
481
|
+
entity names are snake_cased, bounded to 64 characters, and the original recorded in
|
|
482
|
+
`schema/provenance.json`; property names and query parameters keep the vendor's spelling
|
|
483
|
+
(`reference-count`, `query.bibliographic`). Every path+method becomes an operation in
|
|
484
|
+
`connector.toml`: `operationId` snake_cased and bounded to 64 characters (a duplicate gets a
|
|
485
|
+
numeric suffix and a warning), path and query parameters declared, a list response an entity
|
|
486
|
+
list route (declared query parameters that name entity fields filter it), a single-row response
|
|
487
|
+
the entity's read, and a write its create/update/delete by method; a single read whose path
|
|
488
|
+
argument names no field of the row, and a response that carries no entity, stay passthrough
|
|
489
|
+
operations (no `entity`, `501` when called), the reason in the plan. List routes page as the
|
|
490
|
+
document says: a cursor parameter (`cursor`, `pageToken`) beside a next-cursor field in the
|
|
491
|
+
envelope (`next-cursor`, `nextPageToken`) pages by cursor, an `offset` parameter by offset, and
|
|
492
|
+
the count beside the rows (`total-results`, `totalCount`) is the hits key. `[api]` pages the way
|
|
493
|
+
most list routes do, a route paging otherwise names its own `page`, and page-number paging
|
|
494
|
+
(`page` + `per_page`) is named in a warning, its route answering every row. An import declares
|
|
495
|
+
no tools: routes are never tools, so an agent reaches an imported world's routes over HTTP
|
|
496
|
+
(`session open --surface api`), and an author who wants a tool declares it in `x-gateway.tools`
|
|
497
|
+
with a handler.
|
|
458
498
|
`HEAD`/`OPTIONS`/`TRACE` (anything outside `GET POST PUT PATCH DELETE`), a path that repeats a
|
|
459
499
|
placeholder (`/a/{id}/{id}`), and operations
|
|
460
500
|
`--include` / `--exclude` leave out (a path glob or a tag,
|
|
@@ -465,7 +505,7 @@ cyclic entity root, a document without `paths`, a document over 64 MiB.
|
|
|
465
505
|
The command prints the plan (`entities`, `operations`, `skipped`, `warnings`, the document's
|
|
466
506
|
version, title and sha256) and, unless `--plan-only`, writes the tree through the same writer as
|
|
467
507
|
a template `init`, so a rejected import leaves nothing behind, then lists the `next` commands:
|
|
468
|
-
`schema compile`, `schema check`, `data import`, `serve`
|
|
508
|
+
`schema compile`, `schema check`, `data import`, `serve` and a `curl` of a list route, `sandbox commit` or `worlds
|
|
469
509
|
create`. `schema/provenance.json` records the source label (credentials and query strings
|
|
470
510
|
stripped), its sha256 and size, the importer version and each entity's and operation's source
|
|
471
511
|
pointer. `--connector` and `--with-example-tool` do not combine with it. The document's bytes
|
|
@@ -473,8 +513,14 @@ are read where the command runs: a local path directly; `drive:<path>` from the
|
|
|
473
513
|
with the caller's key, in memory; an `http(s)://` URL through a private-address guard (no
|
|
474
514
|
loopback, RFC 1918, link-local or metadata host, every redirect re-checked, five at most) — the
|
|
475
515
|
platform never fetches a caller-named URL, and the sandbox's `run_gateway_cli` accepts only a
|
|
476
|
-
`/workspace` path.
|
|
477
|
-
|
|
516
|
+
`/workspace` path. The import runs in this machine's schema runtime whenever it has one
|
|
517
|
+
(`GATEWAY_RUNTIME=local` demands it and sends nothing to the platform); only with none here, or
|
|
518
|
+
with `GATEWAY_RUNTIME=platform`, does the platform's `init-openapi` runtime command read the
|
|
519
|
+
staged bytes. A YAML document is read by PyYAML in the Python that runs the runtime: without it
|
|
520
|
+
the command is refused before anything is sent, naming the fix (install PyYAML there, point
|
|
521
|
+
`GATEWAY_PYTHON` at a Python that has it, or convert the document to JSON).
|
|
522
|
+
`gateway worlds connector conform <world> --openapi <same document>` then checks the served
|
|
523
|
+
world back against the document it came from.
|
|
478
524
|
|
|
479
525
|
**Build from connector** for Slack creates a durable populated world job in
|
|
480
526
|
Gateway. Other registered source kinds show their CLI twin under **Use the CLI instead**:
|
|
@@ -571,7 +617,7 @@ as a credential-free subprocess (`GATEWAY_SCHEMA_PYTHON`, `GATEWAY_SCHEMA_RUNTIM
|
|
|
571
617
|
| ------------------------------ | ---------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
572
618
|
| `validate`, `validate-archive` | publication | Verify SHA-256, regenerated artifacts and exact lock of an existing tree/archive. |
|
|
573
619
|
| `init-template` | `create_connector_world`, `create_custom_world --template` | Scaffold a connector template into a tree. |
|
|
574
|
-
| `init-openapi` | `worlds schema init --from-openapi` (both CLIs)
|
|
620
|
+
| `init-openapi` | `worlds schema init --from-openapi` (both CLIs; the TypeScript CLI through `POST /api/public/worlds/runtime` only when no runtime is here) | `{source, document (base64), destination, plan_only?, include?, exclude?, name?}` → the import plan and, when written, the compiled tree as `files[]` (at most 2000). The bridge never opens a caller-named path or URL; the CLI stages the bytes. A YAML document needs PyYAML in the Python that reads it, and is refused naming the fix without it. |
|
|
575
621
|
| `compile` | `create_custom_world`, `ingest_to_world` on schema worlds | Compile an authored tree (`compileWorldFiles`): returns the full entry map with generated artifacts + lock, the schema report and which files were added/retargeted. |
|
|
576
622
|
| `apply-batch` | `ingest_records_to_world` | Apply `insert\|update\|delete` mutations as one batch (`applyWorldBatch`): validated row by row against the compiled schema, appended as `data/batches/<batchId>.json` with `source` provenance; replaying the same `batchId` is a no-op. |
|
|
577
623
|
| `build-slack`, `refresh-slack` | connector worlds | Slack capture → batches (durable job). |
|
|
@@ -609,11 +655,17 @@ and mutation order remain meaningful. No broader equivalence is promised.
|
|
|
609
655
|
| Domain | Root `$id`, `domain_version`, and compiled entity-contract hash. Batches must match all three; changed domains require regeneration and explicit state rebuilding. |
|
|
610
656
|
| Dependencies | Every resolved document's `$id` and normalized authored hash are locked. Missing, changed, or incompatible dependencies require an explicit fix/regeneration. |
|
|
611
657
|
| Mapping | Batch `mapping.id/version`, persisted with lineage; a new mapping does not change the domain or package contract. |
|
|
612
|
-
| Package/tool behavior | `x-gateway.package.id/version`, tool-contract hash, and generated artifact hashes. Tool/package changes require regeneration;
|
|
613
|
-
|
|
614
|
-
Opening
|
|
615
|
-
|
|
616
|
-
|
|
658
|
+
| Package/tool behavior | `x-gateway.package.id/version`, tool-contract hash, and generated artifact hashes. Tool/package changes require regeneration; stored state still opens (it is bound to the storage contract, not the lock). Authored handler bytes remain bound by package/bundle versioning, not compiled schema semantics. |
|
|
659
|
+
|
|
660
|
+
Opening state (a snapshot, an export, a session's store) requires the same **storage
|
|
661
|
+
contract** (`StorageContract`): the language, annotations, compiler and target, the domain's
|
|
662
|
+
`$id`, `domain_version` and compiled-entity hash (in either resolver form; an entity field's
|
|
663
|
+
description, title and examples are part of it), and the generated SQL, plus identical DDL. The
|
|
664
|
+
rest of the lock (package, tools, document and artifact hashes) may differ: none of it touches a
|
|
665
|
+
stored row, so a tool edit keeps every snapshot. A domain change has no minor-version shortcut
|
|
666
|
+
or automatic migration: retain the original bundle with old state or explicitly rebuild. Two
|
|
667
|
+
worlds of one domain open each other's state; which state a hosted version opens is the
|
|
668
|
+
platform's to scope. Version labels do not prove vendor fidelity.
|
|
617
669
|
|
|
618
670
|
## Offline example
|
|
619
671
|
|
|
@@ -264,17 +264,18 @@ schema type, the same rule in the TypeScript CLI (`worldData/cast.ts`) and the P
|
|
|
264
264
|
|
|
265
265
|
| Field type | Cell | Value |
|
|
266
266
|
| --- | --- | --- |
|
|
267
|
-
| `integer` | `42`, `-7` | the number; `4.5` or `abc`
|
|
267
|
+
| `integer` | `42`, `-7` | the number; `4.5` or `abc` stays text, refused by the contract |
|
|
268
268
|
| `number` | `12.5`, `1e3`, `3` | the number (always a float) |
|
|
269
|
-
| `boolean` | `true`/`false`/`1`/`0`, any case | the boolean; `yes`
|
|
269
|
+
| `boolean` | `true`/`false`/`1`/`0`, any case | the boolean; `yes` stays text, refused by the contract |
|
|
270
270
|
| `array` / `object` | JSON text of that shape (quote it in the CSV: `"[""a"",""b""]"`) | the parsed value |
|
|
271
271
|
| `string` | anything | the text as given |
|
|
272
272
|
| `integer \| string` (union) | each member tried in order, `string` last | the first that fits |
|
|
273
|
-
| any, empty cell |
|
|
273
|
+
| any, empty cell | (nothing) | null when the type admits null; `""` for a required string; left out otherwise (a required one is then refused by the contract) |
|
|
274
274
|
|
|
275
|
-
A cell that fits none
|
|
276
|
-
|
|
277
|
-
|
|
275
|
+
A cell that fits none stays its text and the contract refuses the row as it refuses a JSON row
|
|
276
|
+
— `taxi_trip {"id":3}.fare: Expected number`, with the usual pointer — so a file reports every
|
|
277
|
+
bad row at once (`--report` writes them all) and nothing is written. A column the schema does
|
|
278
|
+
not name stays text and the gate refuses it as an unknown field. A CSV
|
|
278
279
|
on the drive (`drive:<path>`) is still refused: import it from the machine that holds it, or
|
|
279
280
|
through `worlds sandbox import` when it sits in a sandbox.
|
|
280
281
|
|
|
@@ -292,9 +293,10 @@ A schema world's validated database is kept, not rebuilt. `gateway worlds schema
|
|
|
292
293
|
builds `db/state.sqlite` (the default snapshot: `data/initial.json` applied through the
|
|
293
294
|
contract) and `scenarios/<name>/state.sqlite` (the default with the scenario's overlays),
|
|
294
295
|
each with a `state.json` sidecar `{contractHash, batchDigest, entityCounts, builtAt,
|
|
295
|
-
runtime, fileSha256}`; the same
|
|
296
|
-
and a data batch the contract refuses leaves no
|
|
297
|
-
|
|
296
|
+
runtime, fileSha256}`; a snapshot that already holds the same data is never rebuilt,
|
|
297
|
+
`schema check` verifies the sidecars, and a data batch the contract refuses leaves no
|
|
298
|
+
snapshot behind. `gateway worlds serve` and
|
|
299
|
+
the api bundle open the snapshot (copy + `WorldDatabase.open`, no row replay) and reset by
|
|
298
300
|
copying it again; without one the JSON is replayed as before. A hosted pod gets
|
|
299
301
|
`WORLD_SNAPSHOT_URL` (presigned GET, `.gz` or `Content-Encoding: gzip` is gunzipped),
|
|
300
302
|
`WORLD_SNAPSHOT_SHA256` and `WORLD_SNAPSHOT_DIGEST`; the server downloads, verifies, writes
|
|
@@ -303,3 +305,31 @@ reports `"snapshot": {batchDigest, contractHash}` (or null) and `/__world/seed`
|
|
|
303
305
|
overlay form `{"base": digest, "rows": {...}, "mode": append|replace}` beside the legacy
|
|
304
306
|
whole-state body. Snapshot files are build outputs: gitignore `db/state.sqlite`,
|
|
305
307
|
`db/state.json`, `scenarios/*/state.*`.
|
|
308
|
+
|
|
309
|
+
**Which edits keep a snapshot.** Rows are bound to the domain they were stored under, not to
|
|
310
|
+
the whole contract: adding a tool, editing a handler, a route, a test, a documentation file or
|
|
311
|
+
the package version keeps every snapshot (local, hosted, pinned), with no re-import. A change
|
|
312
|
+
that only adds to the domain (a new entity, a new field that is not required, another
|
|
313
|
+
`domain_version`) carries the rows: compile migrates the snapshots the tree holds and keeps the
|
|
314
|
+
pin with the stored domain's `lock` and `entities`, `sandbox check` says the commit migrates the
|
|
315
|
+
hosted snapshot, and `sandbox commit` migrates it as a job (a `migration` batch: the CLI follows it,
|
|
316
|
+
MCP answers `{batchId}` for `get_data_batch`) and re-pins the tree (a CLI push of such a change is
|
|
317
|
+
refused, naming the sandbox). Importing exactly the batch behind the served snapshot again (the
|
|
318
|
+
same chunks in the same order, with `redact` off) is a no-op: the batch lands on the same version
|
|
319
|
+
and names the one it repeats (`unchangedFrom`). Any other change to an entity, a field (its description
|
|
320
|
+
or title too), a key or a relationship refuses it, naming each change:
|
|
321
|
+
|
|
322
|
+
```text
|
|
323
|
+
snapshot_pinned at /db/snapshot.json: The rows here were stored under the previous domain, and
|
|
324
|
+
this change requires field users.age. Stored rows carry over only when a change adds to their
|
|
325
|
+
domain (a new entity, a new field that is not required, or another domain_version): ...
|
|
326
|
+
```
|
|
327
|
+
|
|
328
|
+
Then keep those as they were, or remove `db/snapshot.json` and import every row again. A tree
|
|
329
|
+
pinned to a hosted snapshot (`db/snapshot.json`, what a data import publishes) compiles and checks
|
|
330
|
+
without the snapshot itself, as in a sandbox: compile keeps the pin, never rebuilds state from
|
|
331
|
+
the data files, and records in the pin (`lock`) the contract the snapshot was built under, which
|
|
332
|
+
is how the platform proves the next edit still fits it. If `schema/lock.json` was replaced by
|
|
333
|
+
artifacts compiled elsewhere before that, compile says the domain cannot be proven: restore
|
|
334
|
+
`schema/lock.json` from the version you started from and compile again. A `[redact]` change is
|
|
335
|
+
not part of it: the store never applies the policy; capture and `--redact apply` do.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: world-data-ingestion
|
|
3
|
-
description: Put rows into a world from any agent. Use when uploading data to a world, seeding a world, importing rows (a few or millions), adding records to a schema world or mock-tools world, pulling a connector's data through an active connection (`gateway worlds data pull --connection`), seeding a live session, exporting a session's calls, turning a vendor export or a real run into world data, turning an agent's real tool calls, a trace's TOOL observations or a saved ClickHouse result into rows (`gateway worlds data calls pull`, then `data ingest --transform` with your own code or `--map ingest.toml`, `captureToolCalls`), or asking why a served world starts from a snapshot. Covers the rows file contract (JSON object, JSONL, --entity), the tool-call record contract, the transform contract, the `ingest.toml` grammar and `map init`, append vs replace, primary-key upsert, `gateway worlds describe`, `gateway worlds data show|import|pull|check|extract|calls pull|ingest|status|batches|publish`, chunked resumable uploads, refusal reports, data snapshots (`db/state.sqlite`, `WORLD_SNAPSHOT_*`), `gateway worlds session seed|export`, the MCP twins import_world_data / pull_world_data / seed_world_session / export_world_session, the refusals, and the definition of done.
|
|
3
|
+
description: Put rows into a world from any agent. Use when uploading data to a world, seeding a world, importing rows (a few or millions), adding records to a schema world or mock-tools world, pulling a connector's data through an active connection (`gateway worlds data pull --connection`), seeding a live session, exporting a session's calls, turning a vendor export or a real run into world data, turning an agent's real tool calls, a trace's TOOL observations or a saved ClickHouse result into rows (`gateway worlds data calls pull`, then `data ingest --transform` with your own code or `--map ingest.toml`, `captureToolCalls`), or asking why a served world starts from a snapshot. Covers the rows file contract (JSON object, JSONL, --entity), the tool-call record contract, the transform contract, the `ingest.toml` grammar and `map init`, append vs replace, primary-key upsert, `gateway worlds describe`, `gateway worlds data show|import|sql|pull|check|extract|calls pull|ingest|status|batches|publish`, chunked resumable uploads, refusal reports, data snapshots (`db/state.sqlite`, `WORLD_SNAPSHOT_*`), `gateway worlds session seed|export`, the MCP twins import_world_data / pull_world_data / sql_world_data / seed_world_session / export_world_session, the refusals, and the definition of done.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# World data ingestion
|
|
@@ -53,10 +53,9 @@ A CSV/TSV is decided by its name and read as a stream (bounded memory at any siz
|
|
|
53
53
|
`--delimiter <char>` splits the cells (one character, `\t` for a tab; default `,`, tab for a
|
|
54
54
|
`.tsv`). The header row names the fields and **every cell is cast by the entity's schema
|
|
55
55
|
type**: `integer`/`number` parse as numbers, `boolean` takes true/false/1/0, `array`/`object`
|
|
56
|
-
parse as JSON, `string` stays text; an empty cell is null when the field admits null, left out
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
pinned by `tests/conformance/csv_cast.json`. Without `--entity` a CSV is refused.
|
|
56
|
+
parse as JSON, `string` stays text; an empty cell is null when the field admits null, `""` for a required string, left out otherwise. A cell that does not fit stays text and the contract refuses its row as it refuses a JSON row:
|
|
57
|
+
every bad row named by entity, key and field (`taxi_trip {"id":3}.fare: Expected number`), all counted, each in `--report`.
|
|
58
|
+
Same rule in both CLIs, pinned by `tests/conformance/csv_cast.json`. Without `--entity` a CSV is refused.
|
|
60
59
|
|
|
61
60
|
Rules:
|
|
62
61
|
|
|
@@ -71,8 +70,7 @@ Rules:
|
|
|
71
70
|
(`--atomic`, the default); `--per-chunk` keeps the good chunks of a large upload and lists
|
|
72
71
|
the refused ones.
|
|
73
72
|
- Inside one batch, two rows with the same key merge by `--merge`: `upsert` (default, the
|
|
74
|
-
later row replaces the earlier) or `fill` (the first row wins, later rows only add fields
|
|
75
|
-
it lacks).
|
|
73
|
+
later row replaces the earlier) or `fill` (the first row wins, later rows only add fields it lacks).
|
|
76
74
|
|
|
77
75
|
## Decision table by world kind
|
|
78
76
|
|
|
@@ -95,7 +93,7 @@ Every CLI step has its MCP twin on the next line. `<WORLD>` is a local world dir
|
|
|
95
93
|
platform world slug (the CLI pulls it to a temp dir, imports, and pushes a new version unless
|
|
96
94
|
`--no-push`). Auth and host: `GATEWAY_HOST`, `GATEWAY_PUBLIC_KEY`, `GATEWAY_SECRET_KEY`, or
|
|
97
95
|
`gateway auth login`. Third place: a CLI sandbox (`worlds sandbox open <slug>@main`) has `gateway` + `sqlite3` on PATH, so
|
|
98
|
-
`sandbox exec <id> 'gateway worlds data import . rows.json'` (`show`, `query`) runs on the uncommitted copy with nothing local; `check` then `commit` lands it. Same refusals as a directory (exit 2).
|
|
96
|
+
`sandbox exec <id> 'gateway worlds data import . rows.json'` (`show`, `query`, `sql`) runs on the uncommitted copy with nothing local; `check` then `commit` lands it. Same refusals as a directory (exit 2).
|
|
99
97
|
The sandbox holds **no platform credentials, by design**: never write keys into its tree (`.gateway-env`, `*.env` are held out of every
|
|
100
98
|
commit, with a warning); bulk rows reach the platform world through `worlds sandbox import <id> <path>` from where the keys are (2d).
|
|
101
99
|
|
|
@@ -108,8 +106,13 @@ gateway worlds data show <WORLD> # {world, kind, entity_count
|
|
|
108
106
|
gateway worlds data show <WORLD> --entity users # the rows of one entity
|
|
109
107
|
gateway worlds data query <WORLD> users --where plan=pro --limit 50 # the rows you mean; a repeated --where key is IN
|
|
110
108
|
gateway worlds data query <WORLD> users --where email=jo@acme.test --resolve # plaintext into a hashed field
|
|
109
|
+
gateway worlds data sql <WORLD> 'SELECT id, name FROM users LIMIT 5' # read-only SQL, any engine
|
|
111
110
|
```
|
|
112
111
|
|
|
112
|
+
`data sql` runs ONE read-only statement (joins, `GROUP BY`, counts) over a schema world's entity tables,
|
|
113
|
+
whatever its database engine: a write, `ATTACH`, `PRAGMA` or a second statement is refused; at most 1000 rows
|
|
114
|
+
(`--limit`). MCP twin: `sql_world_data({ slug, sql, limit? })`. A db/schema.sql world is a SQLite file: `sqlite3`.
|
|
115
|
+
|
|
113
116
|
`data query --resolve` turns each `--where` value into the form the store holds under the
|
|
114
117
|
`[redact]` policy before matching, and prints `resolved <field> -> <form>`; a dropped field
|
|
115
118
|
resolves to nothing, so zero rows there is the policy, not a missing import. MCP twin:
|
|
@@ -152,10 +155,10 @@ JSONL chunks of `--chunk-bytes` (64 MiB default) or `--chunk-rows N`, uploads th
|
|
|
152
155
|
`--parallel 4` at a time straight to the bucket, and completes the batch; the platform
|
|
153
156
|
validates every row against the contract, applies the batch onto the world's data snapshot
|
|
154
157
|
and, with `--publish now` (default), cuts a data-only version (no image rebuild) on your open edit session
|
|
155
|
-
(`editSessionId`; only your credential reads it
|
|
158
|
+
(`editSessionId`; only your credential's unpinned reads see it, and `slug@main`, runs and tasks keep the sealed `main`, until `worlds edit close <slug>` or 60 idle minutes seal it). `--wait`
|
|
156
159
|
(default) polls to the end and prints the batch: `{batchId, state, rows, refusalCount,
|
|
157
160
|
refusals[:200], entityCounts, versionId?, batchDigest?}`. `--no-wait` prints `{batchId,
|
|
158
|
-
state}` and `gateway worlds data status <batchId
|
|
161
|
+
state}` and `gateway worlds data status <batchId> [--report <file>]` reads it later. Exit codes: 0 applied or
|
|
159
162
|
published, 2 refused (the report is written when `--report` is given: JSONL of refusal
|
|
160
163
|
records, at most 10,000 then `{"truncated": true, "total": N}`), 1 error.
|
|
161
164
|
|
|
@@ -270,8 +273,7 @@ level}]}`; text before the JSON is skipped; `input`/`output` decoded as JSON or
|
|
|
270
273
|
|
|
271
274
|
A `gateway worlds session export` line (`{seq, kind, tool, args, result, error, completedAt}`)
|
|
272
275
|
is accepted as it is: `kind != "call"` (a grade, or a `--requests` line of kind `request`) and a failed call are skipped and counted, never
|
|
273
|
-
silently dropped. `captureToolCalls` (TS) / `capture_tool_calls` (Python) write the same
|
|
274
|
-
record at the source.
|
|
276
|
+
silently dropped. `captureToolCalls` (TS) / `capture_tool_calls` (Python) write the same record at the source.
|
|
275
277
|
|
|
276
278
|
**2. Transform.** `--transform <script>`: a `.py` file, or `.ts`/`.mjs`/`.js` for node, that
|
|
277
279
|
exports `transform(records) -> {entity: [rows]}` (node: `transform` or the default export),
|
|
@@ -282,7 +284,7 @@ what nests, what a write's `args` become, which call wins — in code you read t
|
|
|
282
284
|
write. The worked CRM example (nested contacts/deals exploded out of an account read, a
|
|
283
285
|
deal's nested account lifted, activity rows from `crm_log_activity` args keyed by the
|
|
284
286
|
observation id, a deletion turned into an event row, latest call wins) is on
|
|
285
|
-
|
|
287
|
+
https://docs.surfacearea.ai/worlds/data in both languages; the fixtures live in
|
|
286
288
|
`gatewaysdk/sdk/tests/environment/fixtures/crm/` (`records.jsonl`, `transform.py`,
|
|
287
289
|
`transform.ts`, `ingest.toml`, `rows.json`).
|
|
288
290
|
|
|
@@ -366,7 +368,7 @@ the console agent has `run_gateway_cli data ingest` on a sandboxed directory.
|
|
|
366
368
|
|
|
367
369
|
```bash
|
|
368
370
|
gateway worlds sandbox exec <id> 'curl -sSL https://example.test/trips.csv -o data/trips.csv' # or a script that writes rows
|
|
369
|
-
gateway worlds sandbox import <id> data/trips.csv --entity taxi_trip [--delimiter ...] [--mode append|replace] [--dry-run]
|
|
371
|
+
gateway worlds sandbox import <id> data/trips.csv --entity taxi_trip [--delimiter ...] [--mode append|replace] [--dry-run] [--report refusals.jsonl]
|
|
370
372
|
gateway worlds sandbox commit <id> -m "tools" # the SOURCE tree only; rows are not in a commit
|
|
371
373
|
```
|
|
372
374
|
|
|
@@ -375,31 +377,29 @@ sandbox (`POST /api/public/benchmark-containers/sandboxes/{id}/import`; the sand
|
|
|
375
377
|
chunks it to JSONL in a scratch dir the platform reads and removes) and streams it through the
|
|
376
378
|
same ledger as `data import <slug>`: contract, refusals with pointers, `--mode`, `--merge`,
|
|
377
379
|
`--redact`, `--per-chunk`, `-m`. Prints `{versionId, entityCounts, rowCount}`, or `{batchId,
|
|
378
|
-
state}` to poll with `data status`.
|
|
379
|
-
`
|
|
380
|
+
state}` to poll with `data status`. Refused rows exit 2: `{batchId, state: refused, refusalCount, refusals}` (all counted,
|
|
381
|
+
the first 200 inline); `--report` writes every refusal as JSONL, a `--per-chunk` import's and a polled batch's too. MCP:
|
|
382
|
+
`import_world_sandbox_data({ sandboxId, path, entity?, mode? })`. A new-world sandbox commits once first; `commit` never carries rows.
|
|
380
383
|
|
|
381
384
|
### 3. Verify on a live session
|
|
382
385
|
|
|
383
|
-
Open a session, seed it, hit it. Live seeding does not change the world's version; it
|
|
384
|
-
changes this session only.
|
|
386
|
+
Open a session, seed it, hit it. Live seeding does not change the world's version; it changes this session only.
|
|
385
387
|
|
|
386
388
|
```bash
|
|
387
389
|
gateway worlds session open <slug> -t <scenario> --surface api # prints sessionId and api.url
|
|
388
390
|
gateway worlds session seed <SESSION_ID> rows.json # kind:"seed" call, waits for the result
|
|
389
391
|
gateway worlds session seed <SESSION_ID> rows.json --redact refuse # same, refusing plaintext in a redacted field (apply hashes it first)
|
|
390
392
|
gateway worlds session seed <SESSION_ID> rows.json --mode replace # swap the named entities
|
|
391
|
-
curl -H "
|
|
393
|
+
curl -H "X-World-Session: $API_TOKEN" "$API_URL/<route>" # read it back (api.token from session open)
|
|
392
394
|
```
|
|
393
395
|
|
|
394
396
|
Prints `{sessionId, mode, result}`. MCP: `seed_world_session({ sessionId, rows, mode? })` waits
|
|
395
397
|
up to 60 seconds for the engine and otherwise returns a `callId` to poll. Same caps as the
|
|
396
|
-
import. Public API: `POST /api/public/world-sessions/{sessionId}/calls` with
|
|
397
|
-
`{ kind: "seed", rows, mode }`.
|
|
398
|
+
import. Public API: `POST /api/public/world-sessions/{sessionId}/calls` with `{ kind: "seed", rows, mode }`.
|
|
398
399
|
|
|
399
400
|
The tools surface is open by default; `--surface api` adds the vendor's HTTP routes. A fresh
|
|
400
401
|
version's first session takes about fifteen minutes to come up (`PROVISIONING`, 503 on the
|
|
401
|
-
URL); that is a cold pod, not a fault. Pass `--no-wait` and poll
|
|
402
|
-
`gateway worlds session status <SESSION_ID>`.
|
|
402
|
+
URL); that is a cold pod, not a fault. Pass `--no-wait` and poll `gateway worlds session status <SESSION_ID>`.
|
|
403
403
|
|
|
404
404
|
Which kinds can seed live: schema worlds (the store applies the batch through the contract),
|
|
405
405
|
api worlds (the runner merges the rows into the whole state and posts it to the world's seed
|
|
@@ -457,7 +457,7 @@ Sanitize before importing: hash or drop personal fields the world's `[redact]` p
|
|
|
457
457
|
| refused, kind `api` (or `harbor`), naming the file to edit | the world is a generated bundle or a task suite | edit the named source, `bench push` again |
|
|
458
458
|
| `version` missing from the result | a local directory was given | push with `gateway bench push <dir>` when ready |
|
|
459
459
|
| `... is over what a world's source tree carries as data (4 MiB / 100000 rows ...)`; `data/initial.json would be N MiB with these rows, over the 4 MiB publication reads a source file whole`; `exceed the inline cap ... without the bundled runtime` | a directory import above the tree cap; or no `python3`, so the rows travel whole (5,000 / 4 MiB) | `worlds sandbox import` from a sandbox, `data import <slug>` with keys (never keys into the sandbox); install python3 |
|
|
460
|
-
|
|
|
460
|
+
| `<entity> {"<key>":…}.<field>: Expected number` (integer, boolean, array, object) on a CSV row | a cell the field's schema type does not fit: it stays text and the contract refuses its row | fix the cell or the header; `--report` lists every row; nothing was written |
|
|
461
461
|
| exit 2, `state: refused` | the platform refused the batch; `--report` has every record | fix the rows named, `--resume` is not for this: run the import again |
|
|
462
462
|
| `nothing to resume` / `only an open or uploading batch resumes` | no incomplete manifest, or the batch already ran | run the import without `--resume` |
|
|
463
463
|
| `409 base_mismatch` from `/__world/seed` | the overlay named another snapshot digest than the pod is on | the runner posts the digest of the version's snapshot; a mismatch is a version mismatch, never reseed silently |
|
|
@@ -478,8 +478,8 @@ Sanitize before importing: hash or drop personal fields the world's `[redact]` p
|
|
|
478
478
|
|
|
479
479
|
## Data snapshots
|
|
480
480
|
|
|
481
|
-
Why a served world starts in under a second (`db/state.sqlite`, `WORLD_SNAPSHOT_*`,
|
|
482
|
-
|
|
481
|
+
Why a served world starts in under a second (`db/state.sqlite`, `WORLD_SNAPSHOT_*`, `/__world/seed`), and why a tool,
|
|
482
|
+
handler or route edit keeps the imported data while an entity, field or key edit does not: [ROWS.md — Data snapshots](ROWS.md#data-snapshots).
|
|
483
483
|
|
|
484
484
|
## Definition of done
|
|
485
485
|
|
|
@@ -497,4 +497,4 @@ Why a served world starts in under a second (`db/state.sqlite`, `WORLD_SNAPSHOT_
|
|
|
497
497
|
- `connector-schema-authoring`: the data model the rows are checked against.
|
|
498
498
|
- `documentation/features/world-data-ingestion.md`: what exists, the contract, code pointers.
|
|
499
499
|
- `documentation/features/connector-worlds.md`: captures, ingest, serving the vendor's API.
|
|
500
|
-
- Consumer page:
|
|
500
|
+
- Consumer page: https://docs.surfacearea.ai/worlds/data.
|