@withgateway/sdk 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -6
- package/dist/cli.js +4372 -2265
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +4362 -2256
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +259 -58
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +249 -50
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
- package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
- package/dist/index.d.mts +5 -5
- package/dist/index.d.ts +5 -5
- package/dist/index.js +2240 -789
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2150 -739
- package/dist/index.mjs.map +1 -1
- package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
- package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
- package/dist/traces/index.d.mts +1 -1
- package/dist/traces/index.d.ts +1 -1
- package/dist/traces/index.js +253 -52
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +243 -44
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +2 -2
- package/dist/tracing/index.d.ts +2 -2
- package/dist/tracing/index.js +75 -13
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +75 -13
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +893 -482
- package/dist/worlds/index.d.ts +893 -482
- package/dist/worlds/index.js +1968 -669
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1960 -662
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +2 -27
- package/runtime/gatewaysdk/environment/_materialize.py +48 -88
- package/runtime/gatewaysdk/environment/_world.py +155 -237
- package/runtime/gatewaysdk/environment/backing.py +133 -0
- package/runtime/gatewaysdk/environment/core.py +2 -17
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
- package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
- package/runtime/gatewaysdk/environment/runtime.py +8 -13
- package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
- package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +347 -892
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
- package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
- package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +60 -216
- package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
- package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
- package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
- package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
- package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
- package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
- package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
- package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
- package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
- package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
- package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
- package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
- package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +58 -170
- package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
- package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
- package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
- package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
- package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
- package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
- package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
- package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
- package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
- package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
- package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
- package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
- package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
- package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
- package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
- package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
- package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
- package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
- package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
- package/runtime/gatewaysdk/environment/score.py +57 -0
- package/runtime/gatewaysdk/environment/secret_values.py +36 -0
- package/runtime/gatewaysdk/environment/world.py +43 -4
- package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/skills/connector-schema-authoring/SKILL.md +49 -16
- package/skills/connector-schema-authoring/connector-worlds.md +142 -11
- package/skills/connector-schema-authoring/world-schema.md +103 -51
- package/skills/world-data-ingestion/ROWS.md +39 -9
- package/skills/world-data-ingestion/SKILL.md +27 -27
- package/skills/worlds-getting-started/SKILL.md +33 -33
- package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
|
@@ -41,7 +41,7 @@ registration, a platform builder, live ingestion, tools, or a registry framework
|
|
|
41
41
|
## The rules that never bend
|
|
42
42
|
|
|
43
43
|
- **The schema is the deliverable.** A world with a faithful model and no tools is complete. Tools are added when wanted, by whoever wants them.
|
|
44
|
-
- **Every write goes through the schema.** Handlers use `gateway_env.store`; ingestion uses `store.apply_batch` or `store.ingest`; nothing else writes. Out-of-contract writes are rejected with the reason.
|
|
44
|
+
- **Every write goes through the schema.** Handlers use `gateway_env.store`; ingestion uses `store.apply_batch` or `store.ingest`; nothing else writes. Out-of-contract writes are rejected with the reason (one known gap: never name an ingest session's `TEMP` staging table after a world table; see `world-schema.md`, Raw ingestion).
|
|
45
45
|
- **Check the right boundary.** Compile/check validate compiled artifacts; materialization and store operations validate persisted data. Publication checks actual bundle bytes, handler declarations and initial state through the trusted runtime. Fidelity evidence and Environment Actions remain advisory.
|
|
46
46
|
- **Fidelity shape is decided by the vendor, not by a house rule.** A **base** connector is prebaked once (Slack). A **tenant-shaped** connector's model is mostly customer-defined, so describe at bind time is required (Salesforce, NetSuite, Workday). A **mixed** connector is a base plus an optional overlay (Okta, Entra, Google, LDAP, Jamf).
|
|
47
47
|
- **Surface parity is advisory.** If the world declares it mirrors an MCP surface, produce the parity report and show it. Never block on it, never shrink or bend the model to satisfy it.
|
|
@@ -55,7 +55,7 @@ registration, a platform builder, live ingestion, tools, or a registry framework
|
|
|
55
55
|
| Command or path | Status |
|
|
56
56
|
| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
|
57
57
|
| `gateway worlds schema init PATH --connector <slug> [--name NAME] [--with-example-tool]` | exists (#219); `--connector custom` is the blank starting point for an arbitrary connector: its `$id` is derived from `--name` (`urn:gateway:connector:<name>`), it ships the `records` entity and a starter verifier but no tools unless `--with-example-tool` asks for the `list_records`/`get_record`/`create_record` example, and it writes no `data/records.json` (Brex retrospective, 2026-09-14) |
|
|
58
|
-
| `gateway worlds schema init PATH --from-openapi SOURCE [--plan-only] [--include GLOB\|TAG]… [--exclude GLOB\|TAG]…` | exists (2026-09-22); imports a Swagger 2.0 / OpenAPI 3.x document (JSON or YAML; a local path, `drive:<path>` read platform-side, or an http(s) URL) into `schema/world.json` + `connector.toml` + `schema/provenance.json`:
|
|
58
|
+
| `gateway worlds schema init PATH --from-openapi SOURCE [--plan-only] [--include GLOB\|TAG]… [--exclude GLOB\|TAG]…` | exists (2026-09-22); imports a Swagger 2.0 / OpenAPI 3.x document (JSON or YAML; a local path, `drive:<path>` read platform-side, or an http(s) URL) into `schema/world.json` + `connector.toml` + `schema/provenance.json`: the item schemas the selected operations' responses carry, read through the vendor's envelope, become entities (never the wrappers; keyed by a declared id, else the field the read route looks a row up by, nested as a JSON Pointer when it sits inside the row), every path+method a route that answers in the vendor's envelope (`envelope = "message.items"` + `wrapper`) and pages as the document says (passthrough when the response is not an entity), the plan is printed first and `--plan-only` writes nothing; a YAML document needs PyYAML in the Python that runs the runtime and is refused naming the fix without it; refusals (no `paths`, unknown `$ref`, cyclic entity root, undeclared path parameter) carry a JSON pointer. Rules: `worlds-getting-started` → `references/openapi-import.md` |
|
|
59
59
|
| `gateway worlds connector validate\|capture\|ingest PATH` | exists; a `connector.toml` in the world describes base URL, auth scheme, pacing, redaction and operations; `capture` records paced, redacted pages (errors included) under `captures/` with the secret never written; `ingest` seeds `data/initial.json` through the contract in a scratch copy first; `ingest --dry-run [--report refusals.jsonl]` lists every violation of every capture (the `data import --report` records plus `capture`), exit 2, nothing written |
|
|
60
60
|
| `gateway worlds schema reconcile PATH --from captures [--write]` | exists (runtime, offline CLI and `gateway worlds schema reconcile`); the contract against the captured rows: relaxes `required` for fields absent in captures, adds undocumented keys as optional properties typed from the values, drops formats the captured values fail; a report of every change, `schema/world.json` rewritten and recompiled only with `--write` |
|
|
61
61
|
| `gateway worlds connector conform PATH --openapi vendor.yaml` | exists; serves the world in-process, calls every declared operation with arguments from its own seed, and checks status + body against the vendor's OpenAPI response schemas; non-zero exit on any mismatch. Run it before calling a world 1:1; its FAIL list is the work list (unseeded entities → 404s, creates the contract rejects → 4xx, honest 501s). Reads OpenAPI 3.0 `nullable`, reference cycles and `discriminator` unions; `[operations.api] sample_headers` / `sample_body` shape a request the route demands (`Idempotency-Key`, a body) |
|
|
@@ -104,9 +104,9 @@ deliverable can finish with synthetic fixtures and explicitly skipped live valid
|
|
|
104
104
|
- [ ] Author `schema/world.json` and reusable sibling definitions from pinned vendor sources, following the type-mapping notes in [CONNECTORS.md](CONNECTORS.md). `gateway worlds schema entity add` declares an entity with its key, uniques and relationships in one command so the declaration shape is never wrong; hand-author only the payload definitions (`--schema` or `$defs`). Record source URLs, versions/retrieval dates, and hashes of the actual retrieved bytes in `schema/provenance.json`.
|
|
105
105
|
- [ ] Entities have the vendor's keys, relationships, enums, and nullability, and follow [CONVENTIONS.md](CONVENTIONS.md): one reserved prefix per template, plural snake_case names, vendor casing on fields, shared `base.json` primitives. Reserved prefixes and the `payload` name are avoided.
|
|
106
106
|
- [ ] Enforce supported required fields, types, enums, key uniqueness, and reference integrity. Document vendor rules the dialect cannot express, normalization choices, and constraints the future mapper must enforce; structural validation is not vendor behavior.
|
|
107
|
-
- [ ] Real vendor documents compile as they come (Brex retrospective, 2026-09-14): `format: int64`/`double` on numbers is ignored, not refused (string formats are still validated); a schema-valued `additionalProperties` types a map (`Record<string, string>`); a type nested in itself (`ExpandableExpense` holding an `ExpandableExpense`) compiles, with the nested occurrence validated as its bare type and stored as JSON, never projected to a column. A nested key or relationship field is a JSON Pointer, `/owner/user_id`, never `owner.user_id`; the compiler's refusal says so.
|
|
108
|
-
- [ ] Every property verifier SQL will name has a column: scalars and compositions whose non-null branches are scalar (`anyOf [{$ref Status}, {type: null}]`, the nullable-enum shape) project; a composition that may hold an object or array stays JSON-only. A verifier that names a column the contract never projected fails the grade with
|
|
109
|
-
- [ ] `gateway worlds schema compile PATH` then `check PATH` pass. Never edit generated files; SDK templates contain authored inputs and generate bundle artifacts at `init` time. Changing the root `$id` after `init` is fine: `compile` re-stamps an empty `data/initial.json` onto the new domain; a journal that holds rows is refused with the fix named (re-import its rows, or restore the schema it was written against).
|
|
107
|
+
- [ ] Real vendor documents compile as they come (Brex retrospective, 2026-09-14): `format: int64`/`double` on numbers is ignored, not refused (string formats are still validated); a schema-valued `additionalProperties` types a map (`Record<string, string>`); a type nested in itself (`ExpandableExpense` holding an `ExpandableExpense`) compiles, with the nested occurrence validated as its bare type and stored as JSON, never projected to a column. A contract whose inlined files would pass the 4 MB publication cap (Google Docs' recursive body) compiles each object/array definition once under `$defs` and references it instead; nothing to do in the template. A nested key or relationship field is a JSON Pointer, `/owner/user_id`, never `owner.user_id`; the compiler's refusal says so. A primary key may be nested too (`/protocolSection/identificationModule/nctId`) when every object on its path is required and non-null; lookups (`get`, `update`, `delete`) then pass it flat under the pointer itself. Field names keep the vendor's hyphens (`reference-count`); dots and slashes never belong to a name. A verifier's SQL double-quotes such a column (`"reference-count"`): unquoted, SQLite reads a subtraction.
|
|
108
|
+
- [ ] Every property verifier SQL will name has a column: scalars and compositions whose non-null branches are scalar (`anyOf [{$ref Status}, {type: null}]`, the nullable-enum shape) project; a composition that may hold an object or array stays JSON-only. A verifier that names a column the contract never projected fails the grade with the database engine's message and the statement; it never scores 0 quietly (double-quoted unknown names no longer fall back to string literals).
|
|
109
|
+
- [ ] `gateway worlds schema compile PATH` then `check PATH` pass. Never edit generated files; SDK templates contain authored inputs and generate bundle artifacts at `init` time. Changing the root `$id` after `init` is fine: `compile` re-stamps an empty `data/initial.json` onto the new domain; a journal that holds rows follows a change that only adds to its domain (a new entity, a new field that is not required) and is refused otherwise, with the fix named (re-import its rows, or restore the schema it was written against).
|
|
110
110
|
- [ ] Shared templates and committed fixtures hold synthetic data only. Tenant captures stay in their owning project; initial batches and refresh journals retain explicit capture/synthetic provenance.
|
|
111
111
|
|
|
112
112
|
### 2. Tenant overlay (required for tenant-shaped, optional for mixed, absent for base)
|
|
@@ -133,7 +133,8 @@ deliverable can finish with synthetic fixtures and explicitly skipped live valid
|
|
|
133
133
|
### 5. Tools (only if the world ships tools)
|
|
134
134
|
|
|
135
135
|
- [ ] Every tool in `x-gateway.tools` has exactly one handler in `tools/*.py` (the loader refuses a mismatch); generated defaults are fine.
|
|
136
|
-
- [ ]
|
|
136
|
+
- [ ] Routes are never tools. A call by a route's name is refused (`not_a_tool`, naming the session's api URL), and a task's `tools` lists declared tools only. A tool may share an operation's name: a call by that name runs the tool, and the route answers only over HTTP at its path.
|
|
137
|
+
- [ ] Handler reads use `gateway_env.store.get` / `list`; writes use `insert` / `update` / `delete`. Read-only SQL uses `gateway_env.db.one/all`; raw SQL writes are denied. The host's `WorldDatabase.query` is not exposed by the handler store proxy.
|
|
137
138
|
- [ ] Handlers never read `benchmark_runtime_context` or `benchmark_conversation_events`.
|
|
138
139
|
- [ ] A user attaching custom tools to this world needs nothing from this checklist beyond the store API; document any conventions the model expects (key formats, enum values).
|
|
139
140
|
|
|
@@ -143,7 +144,9 @@ Tool handlers (`tools/*.py`) reach the world's rows through `from gateway_env im
|
|
|
143
144
|
exactly these methods; `gateway worlds schema check .` refuses any other `store.<name>` at file:line
|
|
144
145
|
with this list, and a hosted call into such a handler answers 400 `{error: "handler", detail, handler}`
|
|
145
146
|
(an `AttributeError`/`TypeError`/`NameError` is the author's bug; anything else a handler raises is a
|
|
146
|
-
500 `{error: "engine", detail, handler}`; never a bare exception string).
|
|
147
|
+
500 `{error: "engine", detail, handler}`; never a bare exception string). A route's handler answers the
|
|
148
|
+
same way on every door, and an answer JSON cannot carry (a `datetime`, a set, NaN) is its 400; a route
|
|
149
|
+
answering bytes names its `Content-Type` (a vendor's `404 text/plain`), and the bytes go out as sent.
|
|
147
150
|
|
|
148
151
|
- `get(entity: str, key: dict) -> dict | None` — one row by primary key, or `None`.
|
|
149
152
|
- `list(entity: str) -> list[dict]` — every row, ordered by key.
|
|
@@ -235,7 +238,7 @@ Full procedure: `.llm_artifacts/plans/connector-implementation-orchestration.md`
|
|
|
235
238
|
- Treating descriptive resources as schema authority. `init` preserves flat supporting `.json`, `.md` and `.toml` files; only referenced schemas become compiler inputs. Slack capability defaults expand into self-contained entries; see [world-schema.md](../../../documentation/features/world-schema.md#connector-templates-and-init).
|
|
236
239
|
- Converting Slack message timestamps to floats or RFC3339. Preserve exact `(workspace_id, channel_id, ts)` keys; see the [Slack model guide](../../../gatewaysdk/sdk/src/gatewaysdk/environment/schema/connectors/slack/MODEL.md) for normalization and limits.
|
|
237
240
|
- Writing a nested field as `owner.user_id` in `unique` or `relationships`. Nested paths are JSON Pointers: `/owner/user_id`.
|
|
238
|
-
- Reading a verifier's 0 as "the agent failed". If the SQL names a column that does not exist the grade is now an error carrying
|
|
241
|
+
- Reading a verifier's 0 as "the agent failed". If the SQL names a column that does not exist the grade is now an error carrying the database engine's message; an older world that still scores 0 on every task has verifier SQL naming an unprojected property — recompile, the nullable-enum shape projects now.
|
|
239
242
|
|
|
240
243
|
## Where this skill is discoverable
|
|
241
244
|
|
|
@@ -270,8 +273,12 @@ These source checks do not imply live-provider validation or executed fidelity s
|
|
|
270
273
|
|
|
271
274
|
A world can describe the vendor it mirrors in `connector.toml` at its root: base URL,
|
|
272
275
|
auth scheme (`header`, `bearer`, `basic`, `query`, `oauth2`, `none`) with the secret named by an
|
|
273
|
-
environment variable (
|
|
274
|
-
|
|
276
|
+
environment variable (a `bearer`'s `prefixes` lists the `Authorization` words the vendor takes,
|
|
277
|
+
`["Bearer", "token"]` for GitHub, the first carried; a `basic` key is the Basic user with an empty
|
|
278
|
+
password, or with `credential = "password"` the password under any user, as Atlassian and
|
|
279
|
+
ServiceNow take it), the vendor's own login route (`[auth.login]`; an oauth2 `token_url` path may
|
|
280
|
+
name a segment, `/ccx/oauth2/{tenant}/token`) and presigned download routes (`[auth.downloads]`),
|
|
281
|
+
capture pacing, a redaction policy (`hash` or `drop` fields at any
|
|
275
282
|
depth), and the operations the agent actually calls (method, path with `{args}`, query
|
|
276
283
|
params, and the entity plus results path their rows land in). The same file is meant to
|
|
277
284
|
drive the platform's custom source later, so describe once.
|
|
@@ -313,6 +320,10 @@ Three refinements the first three vendors forced:
|
|
|
313
320
|
as well as a path argument; a parameter the caller leaves out does not narrow, and one
|
|
314
321
|
sent more than once (`status[]=a&status[]=b`) matches rows carrying any of its values.
|
|
315
322
|
Handlers read every value from `ctx.query_all`; the `query` argument keeps the first.
|
|
323
|
+
- **Uploads.** A multipart form reaches a handler as its fields (a file part reads
|
|
324
|
+
`{"filename", "content_type", "size"}`) with the bytes in `ctx.files[name].content`; a raw
|
|
325
|
+
body (`application/octet-stream`, `text/csv`) reads `body` as `{}` with the bytes in
|
|
326
|
+
`ctx.raw`. A declared route refuses an upload as the vendor's 400.
|
|
316
327
|
|
|
317
328
|
Four more the Brex retrospective forced (each cost a day of one-field-per-run fixing):
|
|
318
329
|
|
|
@@ -389,25 +400,47 @@ them:
|
|
|
389
400
|
- `[api] envelope = "results"` wraps rows as `{"results": [...], "hits": N, "cursor": ""}`
|
|
390
401
|
(empty means a bare array); `hits_key = ""` leaves the count out and
|
|
391
402
|
`cursor_absent_when_done = true` (world-wide, or per route under `[operations.api]`)
|
|
392
|
-
leaves the cursor key out of the last page — Brex's `{items, next_cursor?}`.
|
|
403
|
+
leaves the cursor key out of the last page — Brex's `{items, next_cursor?}`. A dotted
|
|
404
|
+
`envelope = "message.items"` nests the page (`{message: {items, total-results, next-cursor}}`),
|
|
405
|
+
and `[operations.api] wrapper = '{"status": "ok", "message-type": "work-list"}'` lays it onto
|
|
406
|
+
the rest of the vendor's envelope; a single row is wrapped only by its route's own `envelope`.
|
|
393
407
|
- `unauthorized` / `not_found` / `invalid` are JSON strings replayed on 401 / 404 / 400, so
|
|
394
408
|
the vendor's own error bodies come back; `""` (or `'null'`) means no body at all. A route
|
|
395
409
|
whose missing row the vendor answers its own way sets `not_found` and `not_found_status`
|
|
396
|
-
under `[operations.api]`, falling back to `[api]`.
|
|
410
|
+
under `[operations.api]`, falling back to `[api]`. A path the world does not serve answers
|
|
411
|
+
`not_found`; a path it serves, called with another method, answers `wrong_method` (405 with
|
|
412
|
+
`Allow`, no body unless declared; `wrong_method_status` for a vendor that says 404). A write
|
|
413
|
+
the contract refuses answers `invalid` exactly as declared; the refusal's rule goes to the
|
|
414
|
+
host's log, never the answer.
|
|
415
|
+
- `link_header = true` sends RFC 8288 `Link: <next>; rel="next"` (and `rel="prev"`) on list
|
|
416
|
+
answers, each URL the request's own (GitHub's paging); `cors = true` answers a browser's
|
|
417
|
+
preflight before any credential and names any origin (exposing `Link`) on every answer, for a vendor whose
|
|
418
|
+
API serves pages (`worlds serve` answers only this machine's pages). HEAD answers as GET
|
|
419
|
+
without a body on every world.
|
|
397
420
|
- `[operations.api] action` makes a route a `create`, `update`, `delete` or `handler`; handlers live in the world's `api/handlers.py` and carry computed results and workflow state (`ctx.ladder` on the clock). A world is read _and_ write: an agent that creates a search and gets 501 is not testing anything. Full reference: `documentation/features/connector-worlds.md`.
|
|
398
421
|
- `[operations.api] filter = { email = "attributes.email" }` maps a path argument to the
|
|
399
422
|
row field it selects on, matched case-insensitively at any depth. An operation without
|
|
400
423
|
an `entity` answers 501 `not_modelled` rather than inventing a response.
|
|
424
|
+
- `[auth.login]` mirrors a vendor that trades its key for a token (`path`, `credential = { json |
|
|
425
|
+
form | header = "<field>" }`, `required`, `reply = '{"data": {"token": "{token}"}}'`);
|
|
426
|
+
later calls carry the token the way `scheme` says, and only tokens minted for that session count.
|
|
427
|
+
`[auth.downloads] routes = ["download_x"]` lets those GET routes answer a link a handler signs with
|
|
428
|
+
`ctx.signed_url(path)` without the key. Tokens and links live as long as the session; set
|
|
429
|
+
`ttl_seconds` (login, downloads, oauth2) only when the eval needs expiry. `mode = "open"` under
|
|
430
|
+
`[auth]` opens sessions with no credential at all (a capability URL; `--auth` on the session's
|
|
431
|
+
open overrides; `scheme = "open"` is refused). Full grammar: connector-worlds.md, "The door".
|
|
401
432
|
- `[ui] static = "ui/static"` (+ `entry`, `api_path`, `generated`) is the page over the
|
|
402
433
|
routes: generated by `compile` from the contract unless `generated = false`, served by the
|
|
403
434
|
host on the session's ui origin (`?token=` once, then a cookie; `api_path` rewritten onto
|
|
404
435
|
the routes) and locally on its own port (and at `/__ui/`). A page holds no key; the host presents the world's
|
|
405
|
-
credential for it, the way it does for a
|
|
436
|
+
credential for it, the way it does for a caller holding the session token in `X-World-Session`.
|
|
406
437
|
|
|
407
438
|
`gateway worlds serve PATH --port 8080 [--api-key K]` runs it locally; the runner's admin
|
|
408
|
-
endpoints (`/
|
|
409
|
-
|
|
439
|
+
endpoints (`/__world/seed`, `/__world/reset`, `/__world/state`) sit beside the vendor routes
|
|
440
|
+
and answer only the run's `X-World-Admin-Token` (`gateway worlds test` hands a world's pytest
|
|
441
|
+
files theirs as `WORLD_ADMIN_TOKEN`; by hand, use `/__session/*`); `/healthz` answers everyone,
|
|
442
|
+
and a World Host session's api and tool doors answer none of them (there `/healthz` is the world's own path; its ui origin answers `/healthz` under `api_path` to the twin's own page). `gateway bench push PATH` publishes the world: for a world carrying
|
|
410
443
|
`connector.toml` it packages the api-kind bundle the platform hosts on the way (root
|
|
411
444
|
legacy-shaped on purpose, since the api runner refuses a schema bundle there; `server/`
|
|
412
445
|
needs nothing installed). Open a session with `--surface api` and hand the client that
|
|
413
|
-
URL; the vendor's auth
|
|
446
|
+
URL; the vendor's auth applies as `[auth]` declares it, with the session's own api token as the credential.
|
|
@@ -264,17 +264,18 @@ schema type, the same rule in the TypeScript CLI (`worldData/cast.ts`) and the P
|
|
|
264
264
|
|
|
265
265
|
| Field type | Cell | Value |
|
|
266
266
|
| --- | --- | --- |
|
|
267
|
-
| `integer` | `42`, `-7` | the number; `4.5` or `abc`
|
|
267
|
+
| `integer` | `42`, `-7` | the number; `4.5` or `abc` stays text, refused by the contract |
|
|
268
268
|
| `number` | `12.5`, `1e3`, `3` | the number (always a float) |
|
|
269
|
-
| `boolean` | `true`/`false`/`1`/`0`, any case | the boolean; `yes`
|
|
269
|
+
| `boolean` | `true`/`false`/`1`/`0`, any case | the boolean; `yes` stays text, refused by the contract |
|
|
270
270
|
| `array` / `object` | JSON text of that shape (quote it in the CSV: `"[""a"",""b""]"`) | the parsed value |
|
|
271
271
|
| `string` | anything | the text as given |
|
|
272
272
|
| `integer \| string` (union) | each member tried in order, `string` last | the first that fits |
|
|
273
|
-
| any, empty cell |
|
|
273
|
+
| any, empty cell | (nothing) | null when the type admits null; `""` for a required string; left out otherwise (a required one is then refused by the contract) |
|
|
274
274
|
|
|
275
|
-
A cell that fits none
|
|
276
|
-
|
|
277
|
-
|
|
275
|
+
A cell that fits none stays its text and the contract refuses the row as it refuses a JSON row
|
|
276
|
+
— `taxi_trip {"id":3}.fare: Expected number`, with the usual pointer — so a file reports every
|
|
277
|
+
bad row at once (`--report` writes them all) and nothing is written. A column the schema does
|
|
278
|
+
not name stays text and the gate refuses it as an unknown field. A CSV
|
|
278
279
|
on the drive (`drive:<path>`) is still refused: import it from the machine that holds it, or
|
|
279
280
|
through `worlds sandbox import` when it sits in a sandbox.
|
|
280
281
|
|
|
@@ -292,9 +293,10 @@ A schema world's validated database is kept, not rebuilt. `gateway worlds schema
|
|
|
292
293
|
builds `db/state.sqlite` (the default snapshot: `data/initial.json` applied through the
|
|
293
294
|
contract) and `scenarios/<name>/state.sqlite` (the default with the scenario's overlays),
|
|
294
295
|
each with a `state.json` sidecar `{contractHash, batchDigest, entityCounts, builtAt,
|
|
295
|
-
runtime, fileSha256}`; the same
|
|
296
|
-
and a data batch the contract refuses leaves no
|
|
297
|
-
|
|
296
|
+
runtime, fileSha256}`; a snapshot that already holds the same data is never rebuilt,
|
|
297
|
+
`schema check` verifies the sidecars, and a data batch the contract refuses leaves no
|
|
298
|
+
snapshot behind. `gateway worlds serve` and
|
|
299
|
+
the api bundle open the snapshot (copy + `WorldDatabase.open`, no row replay) and reset by
|
|
298
300
|
copying it again; without one the JSON is replayed as before. A hosted pod gets
|
|
299
301
|
`WORLD_SNAPSHOT_URL` (presigned GET, `.gz` or `Content-Encoding: gzip` is gunzipped),
|
|
300
302
|
`WORLD_SNAPSHOT_SHA256` and `WORLD_SNAPSHOT_DIGEST`; the server downloads, verifies, writes
|
|
@@ -303,3 +305,31 @@ reports `"snapshot": {batchDigest, contractHash}` (or null) and `/__world/seed`
|
|
|
303
305
|
overlay form `{"base": digest, "rows": {...}, "mode": append|replace}` beside the legacy
|
|
304
306
|
whole-state body. Snapshot files are build outputs: gitignore `db/state.sqlite`,
|
|
305
307
|
`db/state.json`, `scenarios/*/state.*`.
|
|
308
|
+
|
|
309
|
+
**Which edits keep a snapshot.** Rows are bound to the domain they were stored under, not to
|
|
310
|
+
the whole contract: adding a tool, editing a handler, a route, a test, a documentation file or
|
|
311
|
+
the package version keeps every snapshot (local, hosted, pinned), with no re-import. A change
|
|
312
|
+
that only adds to the domain (a new entity, a new field that is not required, another
|
|
313
|
+
`domain_version`) carries the rows: compile migrates the snapshots the tree holds and keeps the
|
|
314
|
+
pin with the stored domain's `lock` and `entities`, `sandbox check` says the commit migrates the
|
|
315
|
+
hosted snapshot, and `sandbox commit` migrates it as a job (a `migration` batch: the CLI follows it,
|
|
316
|
+
MCP answers `{batchId}` for `get_data_batch`) and re-pins the tree (a CLI push of such a change is
|
|
317
|
+
refused, naming the sandbox). Importing exactly the batch behind the served snapshot again (the
|
|
318
|
+
same chunks in the same order, with `redact` off) is a no-op: the batch lands on the same version
|
|
319
|
+
and names the one it repeats (`unchangedFrom`). Any other change to an entity, a field (its description
|
|
320
|
+
or title too), a key or a relationship refuses it, naming each change:
|
|
321
|
+
|
|
322
|
+
```text
|
|
323
|
+
snapshot_pinned at /db/snapshot.json: The rows here were stored under the previous domain, and
|
|
324
|
+
this change requires field users.age. Stored rows carry over only when a change adds to their
|
|
325
|
+
domain (a new entity, a new field that is not required, or another domain_version): ...
|
|
326
|
+
```
|
|
327
|
+
|
|
328
|
+
Then keep those as they were, or remove `db/snapshot.json` and import every row again. A tree
|
|
329
|
+
pinned to a hosted snapshot (`db/snapshot.json`, what a data import publishes) compiles and checks
|
|
330
|
+
without the snapshot itself, as in a sandbox: compile keeps the pin, never rebuilds state from
|
|
331
|
+
the data files, and records in the pin (`lock`) the contract the snapshot was built under, which
|
|
332
|
+
is how the platform proves the next edit still fits it. If `schema/lock.json` was replaced by
|
|
333
|
+
artifacts compiled elsewhere before that, compile says the domain cannot be proven: restore
|
|
334
|
+
`schema/lock.json` from the version you started from and compile again. A `[redact]` change is
|
|
335
|
+
not part of it: the store never applies the policy; capture and `--redact apply` do.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: world-data-ingestion
|
|
3
|
-
description: Put rows into a world from any agent. Use when uploading data to a world, seeding a world, importing rows (a few or millions), adding records to a schema world or mock-tools world, pulling a connector's data through an active connection (`gateway worlds data pull --connection`), seeding a live session, exporting a session's calls, turning a vendor export or a real run into world data, turning an agent's real tool calls, a trace's TOOL observations or a saved ClickHouse result into rows (`gateway worlds data calls pull`, then `data ingest --transform` with your own code or `--map ingest.toml`, `captureToolCalls`), or asking why a served world starts from a snapshot. Covers the rows file contract (JSON object, JSONL, --entity), the tool-call record contract, the transform contract, the `ingest.toml` grammar and `map init`, append vs replace, primary-key upsert, `gateway worlds describe`, `gateway worlds data show|import|pull|check|extract|calls pull|ingest|status|batches|publish`, chunked resumable uploads, refusal reports, data snapshots (`db/state.sqlite`, `WORLD_SNAPSHOT_*`), `gateway worlds session seed|export`, the MCP twins import_world_data / pull_world_data / seed_world_session / export_world_session, the refusals, and the definition of done.
|
|
3
|
+
description: Put rows into a world from any agent. Use when uploading data to a world, seeding a world, importing rows (a few or millions), adding records to a schema world or mock-tools world, pulling a connector's data through an active connection (`gateway worlds data pull --connection`), seeding a live session, exporting a session's calls, turning a vendor export or a real run into world data, turning an agent's real tool calls, a trace's TOOL observations or a saved ClickHouse result into rows (`gateway worlds data calls pull`, then `data ingest --transform` with your own code or `--map ingest.toml`, `captureToolCalls`), or asking why a served world starts from a snapshot. Covers the rows file contract (JSON object, JSONL, --entity), the tool-call record contract, the transform contract, the `ingest.toml` grammar and `map init`, append vs replace, primary-key upsert, `gateway worlds describe`, `gateway worlds data show|import|sql|pull|check|extract|calls pull|ingest|status|batches|publish`, chunked resumable uploads, refusal reports, data snapshots (`db/state.sqlite`, `WORLD_SNAPSHOT_*`), `gateway worlds session seed|export`, the MCP twins import_world_data / pull_world_data / sql_world_data / seed_world_session / export_world_session, the refusals, and the definition of done.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# World data ingestion
|
|
@@ -53,10 +53,9 @@ A CSV/TSV is decided by its name and read as a stream (bounded memory at any siz
|
|
|
53
53
|
`--delimiter <char>` splits the cells (one character, `\t` for a tab; default `,`, tab for a
|
|
54
54
|
`.tsv`). The header row names the fields and **every cell is cast by the entity's schema
|
|
55
55
|
type**: `integer`/`number` parse as numbers, `boolean` takes true/false/1/0, `array`/`object`
|
|
56
|
-
parse as JSON, `string` stays text; an empty cell is null when the field admits null, left out
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
pinned by `tests/conformance/csv_cast.json`. Without `--entity` a CSV is refused.
|
|
56
|
+
parse as JSON, `string` stays text; an empty cell is null when the field admits null, `""` for a required string, left out otherwise. A cell that does not fit stays text and the contract refuses its row as it refuses a JSON row:
|
|
57
|
+
every bad row named by entity, key and field (`taxi_trip {"id":3}.fare: Expected number`), all counted, each in `--report`.
|
|
58
|
+
Same rule in both CLIs, pinned by `tests/conformance/csv_cast.json`. Without `--entity` a CSV is refused.
|
|
60
59
|
|
|
61
60
|
Rules:
|
|
62
61
|
|
|
@@ -71,8 +70,7 @@ Rules:
|
|
|
71
70
|
(`--atomic`, the default); `--per-chunk` keeps the good chunks of a large upload and lists
|
|
72
71
|
the refused ones.
|
|
73
72
|
- Inside one batch, two rows with the same key merge by `--merge`: `upsert` (default, the
|
|
74
|
-
later row replaces the earlier) or `fill` (the first row wins, later rows only add fields
|
|
75
|
-
it lacks).
|
|
73
|
+
later row replaces the earlier) or `fill` (the first row wins, later rows only add fields it lacks).
|
|
76
74
|
|
|
77
75
|
## Decision table by world kind
|
|
78
76
|
|
|
@@ -95,7 +93,7 @@ Every CLI step has its MCP twin on the next line. `<WORLD>` is a local world dir
|
|
|
95
93
|
platform world slug (the CLI pulls it to a temp dir, imports, and pushes a new version unless
|
|
96
94
|
`--no-push`). Auth and host: `GATEWAY_HOST`, `GATEWAY_PUBLIC_KEY`, `GATEWAY_SECRET_KEY`, or
|
|
97
95
|
`gateway auth login`. Third place: a CLI sandbox (`worlds sandbox open <slug>@main`) has `gateway` + `sqlite3` on PATH, so
|
|
98
|
-
`sandbox exec <id> 'gateway worlds data import . rows.json'` (`show`, `query`) runs on the uncommitted copy with nothing local; `check` then `commit` lands it. Same refusals as a directory (exit 2).
|
|
96
|
+
`sandbox exec <id> 'gateway worlds data import . rows.json'` (`show`, `query`, `sql`) runs on the uncommitted copy with nothing local; `check` then `commit` lands it. Same refusals as a directory (exit 2).
|
|
99
97
|
The sandbox holds **no platform credentials, by design**: never write keys into its tree (`.gateway-env`, `*.env` are held out of every
|
|
100
98
|
commit, with a warning); bulk rows reach the platform world through `worlds sandbox import <id> <path>` from where the keys are (2d).
|
|
101
99
|
|
|
@@ -108,8 +106,13 @@ gateway worlds data show <WORLD> # {world, kind, entity_count
|
|
|
108
106
|
gateway worlds data show <WORLD> --entity users # the rows of one entity
|
|
109
107
|
gateway worlds data query <WORLD> users --where plan=pro --limit 50 # the rows you mean; a repeated --where key is IN
|
|
110
108
|
gateway worlds data query <WORLD> users --where email=jo@acme.test --resolve # plaintext into a hashed field
|
|
109
|
+
gateway worlds data sql <WORLD> 'SELECT id, name FROM users LIMIT 5' # read-only SQL, any engine
|
|
111
110
|
```
|
|
112
111
|
|
|
112
|
+
`data sql` runs ONE read-only statement (joins, `GROUP BY`, counts) over a schema world's entity tables,
|
|
113
|
+
whatever its database engine: a write, `ATTACH`, `PRAGMA` or a second statement is refused; at most 1000 rows
|
|
114
|
+
(`--limit`). MCP twin: `sql_world_data({ slug, sql, limit? })`. A db/schema.sql world is a SQLite file: `sqlite3`.
|
|
115
|
+
|
|
113
116
|
`data query --resolve` turns each `--where` value into the form the store holds under the
|
|
114
117
|
`[redact]` policy before matching, and prints `resolved <field> -> <form>`; a dropped field
|
|
115
118
|
resolves to nothing, so zero rows there is the policy, not a missing import. MCP twin:
|
|
@@ -152,10 +155,10 @@ JSONL chunks of `--chunk-bytes` (64 MiB default) or `--chunk-rows N`, uploads th
|
|
|
152
155
|
`--parallel 4` at a time straight to the bucket, and completes the batch; the platform
|
|
153
156
|
validates every row against the contract, applies the batch onto the world's data snapshot
|
|
154
157
|
and, with `--publish now` (default), cuts a data-only version (no image rebuild) on your open edit session
|
|
155
|
-
(`editSessionId`; only your credential reads it
|
|
158
|
+
(`editSessionId`; only your credential's unpinned reads see it, and `slug@main`, runs and tasks keep the sealed `main`, until `worlds edit close <slug>` or 60 idle minutes seal it). `--wait`
|
|
156
159
|
(default) polls to the end and prints the batch: `{batchId, state, rows, refusalCount,
|
|
157
160
|
refusals[:200], entityCounts, versionId?, batchDigest?}`. `--no-wait` prints `{batchId,
|
|
158
|
-
state}` and `gateway worlds data status <batchId
|
|
161
|
+
state}` and `gateway worlds data status <batchId> [--report <file>]` reads it later. Exit codes: 0 applied or
|
|
159
162
|
published, 2 refused (the report is written when `--report` is given: JSONL of refusal
|
|
160
163
|
records, at most 10,000 then `{"truncated": true, "total": N}`), 1 error.
|
|
161
164
|
|
|
@@ -270,8 +273,7 @@ level}]}`; text before the JSON is skipped; `input`/`output` decoded as JSON or
|
|
|
270
273
|
|
|
271
274
|
A `gateway worlds session export` line (`{seq, kind, tool, args, result, error, completedAt}`)
|
|
272
275
|
is accepted as it is: `kind != "call"` (a grade, or a `--requests` line of kind `request`) and a failed call are skipped and counted, never
|
|
273
|
-
silently dropped. `captureToolCalls` (TS) / `capture_tool_calls` (Python) write the same
|
|
274
|
-
record at the source.
|
|
276
|
+
silently dropped. `captureToolCalls` (TS) / `capture_tool_calls` (Python) write the same record at the source.
|
|
275
277
|
|
|
276
278
|
**2. Transform.** `--transform <script>`: a `.py` file, or `.ts`/`.mjs`/`.js` for node, that
|
|
277
279
|
exports `transform(records) -> {entity: [rows]}` (node: `transform` or the default export),
|
|
@@ -282,7 +284,7 @@ what nests, what a write's `args` become, which call wins — in code you read t
|
|
|
282
284
|
write. The worked CRM example (nested contacts/deals exploded out of an account read, a
|
|
283
285
|
deal's nested account lifted, activity rows from `crm_log_activity` args keyed by the
|
|
284
286
|
observation id, a deletion turned into an event row, latest call wins) is on
|
|
285
|
-
|
|
287
|
+
https://docs.surfacearea.ai/worlds/data in both languages; the fixtures live in
|
|
286
288
|
`gatewaysdk/sdk/tests/environment/fixtures/crm/` (`records.jsonl`, `transform.py`,
|
|
287
289
|
`transform.ts`, `ingest.toml`, `rows.json`).
|
|
288
290
|
|
|
@@ -366,7 +368,7 @@ the console agent has `run_gateway_cli data ingest` on a sandboxed directory.
|
|
|
366
368
|
|
|
367
369
|
```bash
|
|
368
370
|
gateway worlds sandbox exec <id> 'curl -sSL https://example.test/trips.csv -o data/trips.csv' # or a script that writes rows
|
|
369
|
-
gateway worlds sandbox import <id> data/trips.csv --entity taxi_trip [--delimiter ...] [--mode append|replace] [--dry-run]
|
|
371
|
+
gateway worlds sandbox import <id> data/trips.csv --entity taxi_trip [--delimiter ...] [--mode append|replace] [--dry-run] [--report refusals.jsonl]
|
|
370
372
|
gateway worlds sandbox commit <id> -m "tools" # the SOURCE tree only; rows are not in a commit
|
|
371
373
|
```
|
|
372
374
|
|
|
@@ -375,31 +377,29 @@ sandbox (`POST /api/public/benchmark-containers/sandboxes/{id}/import`; the sand
|
|
|
375
377
|
chunks it to JSONL in a scratch dir the platform reads and removes) and streams it through the
|
|
376
378
|
same ledger as `data import <slug>`: contract, refusals with pointers, `--mode`, `--merge`,
|
|
377
379
|
`--redact`, `--per-chunk`, `-m`. Prints `{versionId, entityCounts, rowCount}`, or `{batchId,
|
|
378
|
-
state}` to poll with `data status`.
|
|
379
|
-
`
|
|
380
|
+
state}` to poll with `data status`. Refused rows exit 2: `{batchId, state: refused, refusalCount, refusals}` (all counted,
|
|
381
|
+
the first 200 inline); `--report` writes every refusal as JSONL, a `--per-chunk` import's and a polled batch's too. MCP:
|
|
382
|
+
`import_world_sandbox_data({ sandboxId, path, entity?, mode? })`. A new-world sandbox commits once first; `commit` never carries rows.
|
|
380
383
|
|
|
381
384
|
### 3. Verify on a live session
|
|
382
385
|
|
|
383
|
-
Open a session, seed it, hit it. Live seeding does not change the world's version; it
|
|
384
|
-
changes this session only.
|
|
386
|
+
Open a session, seed it, hit it. Live seeding does not change the world's version; it changes this session only.
|
|
385
387
|
|
|
386
388
|
```bash
|
|
387
389
|
gateway worlds session open <slug> -t <scenario> --surface api # prints sessionId and api.url
|
|
388
390
|
gateway worlds session seed <SESSION_ID> rows.json # kind:"seed" call, waits for the result
|
|
389
391
|
gateway worlds session seed <SESSION_ID> rows.json --redact refuse # same, refusing plaintext in a redacted field (apply hashes it first)
|
|
390
392
|
gateway worlds session seed <SESSION_ID> rows.json --mode replace # swap the named entities
|
|
391
|
-
curl -H "
|
|
393
|
+
curl -H "X-World-Session: $API_TOKEN" "$API_URL/<route>" # read it back (api.token from session open)
|
|
392
394
|
```
|
|
393
395
|
|
|
394
396
|
Prints `{sessionId, mode, result}`. MCP: `seed_world_session({ sessionId, rows, mode? })` waits
|
|
395
397
|
up to 60 seconds for the engine and otherwise returns a `callId` to poll. Same caps as the
|
|
396
|
-
import. Public API: `POST /api/public/world-sessions/{sessionId}/calls` with
|
|
397
|
-
`{ kind: "seed", rows, mode }`.
|
|
398
|
+
import. Public API: `POST /api/public/world-sessions/{sessionId}/calls` with `{ kind: "seed", rows, mode }`.
|
|
398
399
|
|
|
399
400
|
The tools surface is open by default; `--surface api` adds the vendor's HTTP routes. A fresh
|
|
400
401
|
version's first session takes about fifteen minutes to come up (`PROVISIONING`, 503 on the
|
|
401
|
-
URL); that is a cold pod, not a fault. Pass `--no-wait` and poll
|
|
402
|
-
`gateway worlds session status <SESSION_ID>`.
|
|
402
|
+
URL); that is a cold pod, not a fault. Pass `--no-wait` and poll `gateway worlds session status <SESSION_ID>`.
|
|
403
403
|
|
|
404
404
|
Which kinds can seed live: schema worlds (the store applies the batch through the contract),
|
|
405
405
|
api worlds (the runner merges the rows into the whole state and posts it to the world's seed
|
|
@@ -457,7 +457,7 @@ Sanitize before importing: hash or drop personal fields the world's `[redact]` p
|
|
|
457
457
|
| refused, kind `api` (or `harbor`), naming the file to edit | the world is a generated bundle or a task suite | edit the named source, `bench push` again |
|
|
458
458
|
| `version` missing from the result | a local directory was given | push with `gateway bench push <dir>` when ready |
|
|
459
459
|
| `... is over what a world's source tree carries as data (4 MiB / 100000 rows ...)`; `data/initial.json would be N MiB with these rows, over the 4 MiB publication reads a source file whole`; `exceed the inline cap ... without the bundled runtime` | a directory import above the tree cap; or no `python3`, so the rows travel whole (5,000 / 4 MiB) | `worlds sandbox import` from a sandbox, `data import <slug>` with keys (never keys into the sandbox); install python3 |
|
|
460
|
-
|
|
|
460
|
+
| `<entity> {"<key>":…}.<field>: Expected number` (integer, boolean, array, object) on a CSV row | a cell the field's schema type does not fit: it stays text and the contract refuses its row | fix the cell or the header; `--report` lists every row; nothing was written |
|
|
461
461
|
| exit 2, `state: refused` | the platform refused the batch; `--report` has every record | fix the rows named, `--resume` is not for this: run the import again |
|
|
462
462
|
| `nothing to resume` / `only an open or uploading batch resumes` | no incomplete manifest, or the batch already ran | run the import without `--resume` |
|
|
463
463
|
| `409 base_mismatch` from `/__world/seed` | the overlay named another snapshot digest than the pod is on | the runner posts the digest of the version's snapshot; a mismatch is a version mismatch, never reseed silently |
|
|
@@ -478,8 +478,8 @@ Sanitize before importing: hash or drop personal fields the world's `[redact]` p
|
|
|
478
478
|
|
|
479
479
|
## Data snapshots
|
|
480
480
|
|
|
481
|
-
Why a served world starts in under a second (`db/state.sqlite`, `WORLD_SNAPSHOT_*`,
|
|
482
|
-
|
|
481
|
+
Why a served world starts in under a second (`db/state.sqlite`, `WORLD_SNAPSHOT_*`, `/__world/seed`), and why a tool,
|
|
482
|
+
handler or route edit keeps the imported data while an entity, field or key edit does not: [ROWS.md — Data snapshots](ROWS.md#data-snapshots).
|
|
483
483
|
|
|
484
484
|
## Definition of done
|
|
485
485
|
|
|
@@ -497,4 +497,4 @@ Why a served world starts in under a second (`db/state.sqlite`, `WORLD_SNAPSHOT_
|
|
|
497
497
|
- `connector-schema-authoring`: the data model the rows are checked against.
|
|
498
498
|
- `documentation/features/world-data-ingestion.md`: what exists, the contract, code pointers.
|
|
499
499
|
- `documentation/features/connector-worlds.md`: captures, ingest, serving the vendor's API.
|
|
500
|
-
- Consumer page:
|
|
500
|
+
- Consumer page: https://docs.surfacearea.ai/worlds/data.
|