@withgateway/sdk 0.3.3 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +2221 -962
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2212 -953
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +81 -19
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +79 -17
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +903 -221
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +868 -212
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js +78 -16
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +76 -14
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.js +7 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +7 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +692 -410
- package/dist/worlds/index.d.ts +692 -410
- package/dist/worlds/index.js +740 -197
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +736 -193
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_materialize.py +5 -25
- package/runtime/gatewaysdk/environment/_world.py +58 -65
- package/runtime/gatewaysdk/environment/backing.py +20 -24
- package/runtime/gatewaysdk/environment/core.py +1 -16
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +174 -471
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +230 -18
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +201 -19
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/conform.py +6 -27
- package/runtime/gatewaysdk/environment/schema/connector.py +89 -152
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +14 -13
- package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
- package/runtime/gatewaysdk/environment/schema/database/guard.py +43 -36
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
- package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +25 -4
- package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +31 -116
- package/runtime/gatewaysdk/environment/schema/platform.py +15 -17
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +129 -176
- package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +31 -6
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +13 -13
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +23 -23
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +35 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +149 -29
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
- package/runtime/gatewaysdk/environment/schema/tools_world.py +84 -203
- package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
- package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
- package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +9 -11
- package/runtime/gatewaysdk/environment/world.py +1 -1
- package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/skills/connector-schema-authoring/SKILL.md +31 -6
- package/skills/connector-schema-authoring/connector-worlds.md +129 -7
- package/skills/world-data-ingestion/ROWS.md +12 -8
- package/skills/world-data-ingestion/SKILL.md +13 -13
- package/skills/worlds-getting-started/SKILL.md +23 -23
- package/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
- package/skills/worlds-getting-started/references/gotchas.md +35 -2
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
|
@@ -79,9 +79,9 @@ from typing import Any
|
|
|
79
79
|
|
|
80
80
|
from .._toml import tomllib
|
|
81
81
|
from ..world_session_contract import ADMIN_TOKEN_ENV
|
|
82
|
-
from .
|
|
83
|
-
from .conform import PIN, _credentials
|
|
82
|
+
from .conform import PIN
|
|
84
83
|
from .connector import load_spec
|
|
84
|
+
from .door import Door
|
|
85
85
|
from .local import LocalSession, open_world
|
|
86
86
|
from .validation import fail
|
|
87
87
|
from .workers import INHERITED_ENV
|
|
@@ -352,12 +352,12 @@ def run(world: str | Path, *, scenario: str | None = None) -> Report:
|
|
|
352
352
|
)
|
|
353
353
|
spec = load_spec(root) if session.routes else None
|
|
354
354
|
prefix = urllib.parse.urlsplit(spec.base_url).path.rstrip("/") if spec else ""
|
|
355
|
-
|
|
355
|
+
door = session.app.door if session.app is not None else None
|
|
356
356
|
for file, case in cases:
|
|
357
357
|
if "tool" in case:
|
|
358
358
|
results.append(_run_tool_case(session, file, case))
|
|
359
359
|
else:
|
|
360
|
-
results.append(_run_case(base, prefix,
|
|
360
|
+
results.append(_run_case(base, prefix, door, file, case))
|
|
361
361
|
python = _run_python(root, python_files, session)
|
|
362
362
|
finally:
|
|
363
363
|
session.close()
|
|
@@ -395,7 +395,7 @@ def _run_tool_case(session: LocalSession, file: str, case: dict) -> CaseResult:
|
|
|
395
395
|
return result
|
|
396
396
|
|
|
397
397
|
|
|
398
|
-
def _run_case(base: str, prefix: str,
|
|
398
|
+
def _run_case(base: str, prefix: str, door: Door | None, file: str, case: dict) -> CaseResult:
|
|
399
399
|
request = case["request"]
|
|
400
400
|
expect = case["expect"]
|
|
401
401
|
name = str(case.get("name") or f"{request.get('method', 'GET')} {request.get('path', '/')}")
|
|
@@ -413,13 +413,11 @@ def _run_case(base: str, prefix: str, spec, credentials: dict[str, str], file: s
|
|
|
413
413
|
if prefix and not path.startswith(prefix + "/") and path != prefix and not path.startswith("/__"):
|
|
414
414
|
# the world's own /__ routes (session, ui, the ui's api_path) are never under the vendor's prefix
|
|
415
415
|
path = prefix + (path if path.startswith("/") else "/" + path)
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
416
|
+
presents = door is not None and request.get("auth", True)
|
|
417
|
+
credential_headers, credential_query = door.present() if presents else ({}, {})
|
|
418
|
+
query = {**dict(request.get("query") or {}), **credential_query}
|
|
419
419
|
url = base + path + (f"?{urllib.parse.urlencode(query)}" if query else "")
|
|
420
|
-
headers = {k: str(v) for k, v in (request.get("headers") or {}).items()}
|
|
421
|
-
if request.get("auth", True):
|
|
422
|
-
headers = {**credentials, **headers}
|
|
420
|
+
headers = {**credential_headers, **{k: str(v) for k, v in (request.get("headers") or {}).items()}}
|
|
423
421
|
body = request.get("body")
|
|
424
422
|
data = None
|
|
425
423
|
if body is not None:
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
(``_bundle.detect_kind``), the data commands (``data.world_kind``), the hub
|
|
5
5
|
client (``harbor_adapter.detect_env_kind``), the benchmark runner
|
|
6
6
|
(``engines/hub_bundle.detect_env_kind``), the platform bridge
|
|
7
|
-
(``platform.world_root``) and the World Host (``
|
|
7
|
+
(``platform.world_root``) and the World Host (``host_versions.world_tree``) all read
|
|
8
8
|
their answer from here. The TypeScript twin is
|
|
9
9
|
``packages/shared/src/world/`` (``WorldTree`` + ``World.fromTree``); the
|
|
10
10
|
corpus under ``conformance/worlds/`` pins both to the same answers.
|
|
@@ -26,6 +26,20 @@ CALL_BODY_LIMIT_LABEL = "4mb"
|
|
|
26
26
|
#: ``session status`` warns once an unpinned session has less than this left.
|
|
27
27
|
IDLE_WARNING_MINUTES = 5
|
|
28
28
|
|
|
29
|
+
#: The most entries one page of the world request ledger holds, as the platform lists it
|
|
30
|
+
#: and as a World Host drains a session's outbox.
|
|
31
|
+
REQUEST_LOG_PAGE_MAX = 200
|
|
32
|
+
|
|
33
|
+
#: How much of each body the request ledger keeps: the first this many bytes of its text.
|
|
34
|
+
REQUEST_BODY_CAPTURE_BYTES = 64 * 1024
|
|
35
|
+
|
|
36
|
+
#: What a World Host's heartbeat names to say it serves open sessions; the platform places an
|
|
37
|
+
#: open session only on a host that does.
|
|
38
|
+
OPEN_DOOR_FEATURE = "open-door"
|
|
39
|
+
|
|
40
|
+
#: The code a session the World Host did not serve answers its request ledger read with.
|
|
41
|
+
REQUESTS_NOT_RECORDED = "requests_not_recorded"
|
|
42
|
+
|
|
29
43
|
#: The task an exploratory session runs on when the caller names none, and
|
|
30
44
|
#: the ``task.source`` the platform reports for it.
|
|
31
45
|
EXPLORATORY_TASK: dict[str, str] = {"instruction": "exploratory session"}
|
|
@@ -93,7 +93,11 @@ ship `connector.toml` in the template directory. `init` copies it to the world r
|
|
|
93
93
|
The format is documented in `SKILL.md` under "Custom connections" and "Serving the vendor's API":
|
|
94
94
|
|
|
95
95
|
- `[connector]` slug and `base_url`; `[auth]` scheme (`header|bearer|basic|query|oauth2|none`),
|
|
96
|
-
`name`, and `secret_env` naming the environment variable, never a value
|
|
96
|
+
`name`, and `secret_env` naming the environment variable, never a value; `prefixes` when the
|
|
97
|
+
vendor takes other `Authorization` words than `Bearer`, `credential = "password"` when its Basic
|
|
98
|
+
key is the password; `[auth.login]` when the vendor trades its key for a token,
|
|
99
|
+
`[auth.downloads]` when it hands out links to files; `[api] unauthorized_status` and
|
|
100
|
+
`unauthorized` as the vendor refuses a bad key.
|
|
97
101
|
- `[capture]` pacing (`requests_per_minute` low; providers ask us to be sparing), cursor
|
|
98
102
|
pagination when the vendor has it (`cursor_field` in the body or `cursor_header`);
|
|
99
103
|
`[redact]` `hash`/`drop` for every PII field the vendor documents returning, `round` for
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
name: benchmarks-getting-started
|
|
3
|
-
description: Build, validate, dry-run, push, run and read a benchmark on the platform, for an agent (Claude Code, Cursor, Codex). A benchmark is either a world with tasks (tools + data, a schema when the vendor has one; a prompt and the SQL verifiers that grade the world's end state) or a container benchmark (a harbor task tree graded by its own tests/test.sh in a container, or a verifiers package graded by Python reward functions), run k times per task per model. Use when building a benchmark, an eval suite, a coding benchmark, a task set or a leaderboard for an agent; when you have a harbor task tree, task.toml files, a verifiers package, a Taskset, or tests that grade an agent; when choosing between a world benchmark and a container (benchmark runner) benchmark; when you need a dry run before pushing (locally in Docker, or hosted on the benchmark runner with `--hosted` when the machine has no Docker); when running k rollouts per task per model or reading per-task results. Every CLI step names its MCP twin (create_world, env_write_file, env_commit, create_environment_version, run_evaluation, get_task_performance, list_rollouts, get_rollout_samples) and where the result shows in the UI.
|
|
3
|
+
description: Build, validate, dry-run, push, run and read a benchmark on the platform, for an agent (Claude Code, Cursor, Codex). A benchmark is either a world with tasks (tools + data, a schema when the vendor has one; a prompt and the SQL verifiers that grade the world's end state and the requests it received) or a container benchmark (a harbor task tree graded by its own tests/test.sh in a container, or a verifiers package graded by Python reward functions), run k times per task per model. Use when building a benchmark, an eval suite, a coding benchmark, a task set or a leaderboard for an agent; when you have a harbor task tree, task.toml files, a verifiers package, a Taskset, or tests that grade an agent; when choosing between a world benchmark and a container (benchmark runner) benchmark; when you need a dry run before pushing (locally in Docker, or hosted on the benchmark runner with `--hosted` when the machine has no Docker); when running k rollouts per task per model or reading per-task results. Every CLI step names its MCP twin (create_world, env_write_file, env_commit, create_environment_version, run_evaluation, get_task_performance, list_rollouts, get_rollout_samples) and where the result shows in the UI.
|
|
4
4
|
---
|
|
5
5
|
|
|
6
6
|
# Benchmarks: getting started
|
|
@@ -355,13 +355,12 @@ gateway worlds schema task add ./crm-bench close_ticket --prompt "Close ticket T
|
|
|
355
355
|
gateway worlds schema task add ./crm-bench escalate --prompt "..." --verifier state_present --world slack=state_present
|
|
356
356
|
```
|
|
357
357
|
|
|
358
|
-
- `--prompt` is what the agent is told. `--verifier <name>` names `verifiers/<name>.sql`; it must
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
`
|
|
362
|
-
- A verifier
|
|
363
|
-
|
|
364
|
-
"crm_tickets" WHERE id = 'T-7' AND status = 'closed';`
|
|
358
|
+
- `--prompt` is what the agent is told. `--verifier <name>` names `verifiers/<name>.sql`; it must exist — `schema verifier add <dir> <name> --entity <entity> [--where "<sql condition>"]` scaffolds
|
|
359
|
+
it (without `--where`, the `state_present` placeholder). Each verifier becomes a key in `rewards`; `reward` is their mean.
|
|
360
|
+
- A verifier is one SQL statement returning one number in 0..1 over the world's tables (entity names from `schema/world.json`):
|
|
361
|
+
`SELECT CASE WHEN COUNT(*) > 0 THEN 1.0 ELSE 0.0 END FROM "crm_tickets" WHERE id = 'T-7' AND status = 'closed';`
|
|
362
|
+
- A verifier can also read every request the session's world received, in `gateway_requests` (and its summary `gateway_journal`):
|
|
363
|
+
`SELECT CASE WHEN COUNT(*) = 0 THEN 1.0 ELSE 0.0 END FROM gateway_requests WHERE method = 'DELETE';`. Only a World Host session records them (`dry-run` and `validate` read them as not recorded, and such a verifier scores 0 there whatever it accepts); on a journal that may be missing requests it scores 0 with the reason unless `gateway-env.toml` has `[verifiers.<name>] accepts_incomplete_journal = true`, which suits a check a missing request can only fail (`COUNT(*) > 0`), never `= 0`. A linked world's verifier gets no journal, so one that reads it scores 0. Columns, fail-closed rules, declarative and `verify.py` checks over the same requests: `worlds-getting-started` `references/journal-checks.md`.
|
|
365
364
|
- `--world <alias>=<verifier>` grades a linked world's state too (`schema link` first).
|
|
366
365
|
- Stored task specs (`instruction`, `seed`, `grader` of kind `assertions` | `python` | `rubric`):
|
|
367
366
|
`gateway worlds task create task.json --world <slug>`; `worlds-getting-started` §6.
|
|
@@ -496,5 +495,5 @@ Every run records the version it ran against; `results` shows the per-version tr
|
|
|
496
495
|
|
|
497
496
|
- `worlds-getting-started`: the world itself — contract, data, sessions, stored tasks, links.
|
|
498
497
|
- `world-data-ingestion`: rows in, calls out. `connector-schema-authoring`: contract, fidelity shapes.
|
|
499
|
-
- Consumer pages: `
|
|
498
|
+
- Consumer pages (docs.surfacearea.ai): `cli/benchmarks`, `worlds/benchmarks`, `evaluation/benchmarks-harbor`.
|
|
500
499
|
- `documentation/features/benchmark-containers.md`: the container substrate, kinds, the runner.
|
|
@@ -273,8 +273,12 @@ These source checks do not imply live-provider validation or executed fidelity s
|
|
|
273
273
|
|
|
274
274
|
A world can describe the vendor it mirrors in `connector.toml` at its root: base URL,
|
|
275
275
|
auth scheme (`header`, `bearer`, `basic`, `query`, `oauth2`, `none`) with the secret named by an
|
|
276
|
-
environment variable (
|
|
277
|
-
|
|
276
|
+
environment variable (a `bearer`'s `prefixes` lists the `Authorization` words the vendor takes,
|
|
277
|
+
`["Bearer", "token"]` for GitHub, the first carried; a `basic` key is the Basic user with an empty
|
|
278
|
+
password, or with `credential = "password"` the password under any user, as Atlassian and
|
|
279
|
+
ServiceNow take it), the vendor's own login route (`[auth.login]`; an oauth2 `token_url` path may
|
|
280
|
+
name a segment, `/ccx/oauth2/{tenant}/token`) and presigned download routes (`[auth.downloads]`),
|
|
281
|
+
capture pacing, a redaction policy (`hash` or `drop` fields at any
|
|
278
282
|
depth), and the operations the agent actually calls (method, path with `{args}`, query
|
|
279
283
|
params, and the entity plus results path their rows land in). The same file is meant to
|
|
280
284
|
drive the platform's custom source later, so describe once.
|
|
@@ -316,6 +320,10 @@ Three refinements the first three vendors forced:
|
|
|
316
320
|
as well as a path argument; a parameter the caller leaves out does not narrow, and one
|
|
317
321
|
sent more than once (`status[]=a&status[]=b`) matches rows carrying any of its values.
|
|
318
322
|
Handlers read every value from `ctx.query_all`; the `query` argument keeps the first.
|
|
323
|
+
- **Uploads.** A multipart form reaches a handler as its fields (a file part reads
|
|
324
|
+
`{"filename", "content_type", "size"}`) with the bytes in `ctx.files[name].content`; a raw
|
|
325
|
+
body (`application/octet-stream`, `text/csv`) reads `body` as `{}` with the bytes in
|
|
326
|
+
`ctx.raw`. A declared route refuses an upload as the vendor's 400.
|
|
319
327
|
|
|
320
328
|
Four more the Brex retrospective forced (each cost a day of one-field-per-run fixing):
|
|
321
329
|
|
|
@@ -399,23 +407,40 @@ them:
|
|
|
399
407
|
- `unauthorized` / `not_found` / `invalid` are JSON strings replayed on 401 / 404 / 400, so
|
|
400
408
|
the vendor's own error bodies come back; `""` (or `'null'`) means no body at all. A route
|
|
401
409
|
whose missing row the vendor answers its own way sets `not_found` and `not_found_status`
|
|
402
|
-
under `[operations.api]`, falling back to `[api]`.
|
|
410
|
+
under `[operations.api]`, falling back to `[api]`. A path the world does not serve answers
|
|
411
|
+
`not_found`; a path it serves, called with another method, answers `wrong_method` (405 with
|
|
412
|
+
`Allow`, no body unless declared; `wrong_method_status` for a vendor that says 404). A write
|
|
413
|
+
the contract refuses answers `invalid` exactly as declared; the refusal's rule goes to the
|
|
414
|
+
host's log, never the answer.
|
|
415
|
+
- `link_header = true` sends RFC 8288 `Link: <next>; rel="next"` (and `rel="prev"`) on list
|
|
416
|
+
answers, each URL the request's own (GitHub's paging); `cors = true` answers a browser's
|
|
417
|
+
preflight before any credential and names any origin (exposing `Link`) on every answer, for a vendor whose
|
|
418
|
+
API serves pages (`worlds serve` answers only this machine's pages). HEAD answers as GET
|
|
419
|
+
without a body on every world.
|
|
403
420
|
- `[operations.api] action` makes a route a `create`, `update`, `delete` or `handler`; handlers live in the world's `api/handlers.py` and carry computed results and workflow state (`ctx.ladder` on the clock). A world is read _and_ write: an agent that creates a search and gets 501 is not testing anything. Full reference: `documentation/features/connector-worlds.md`.
|
|
404
421
|
- `[operations.api] filter = { email = "attributes.email" }` maps a path argument to the
|
|
405
422
|
row field it selects on, matched case-insensitively at any depth. An operation without
|
|
406
423
|
an `entity` answers 501 `not_modelled` rather than inventing a response.
|
|
424
|
+
- `[auth.login]` mirrors a vendor that trades its key for a token (`path`, `credential = { json |
|
|
425
|
+
form | header = "<field>" }`, `required`, `reply = '{"data": {"token": "{token}"}}'`);
|
|
426
|
+
later calls carry the token the way `scheme` says, and only tokens minted for that session count.
|
|
427
|
+
`[auth.downloads] routes = ["download_x"]` lets those GET routes answer a link a handler signs with
|
|
428
|
+
`ctx.signed_url(path)` without the key. Tokens and links live as long as the session; set
|
|
429
|
+
`ttl_seconds` (login, downloads, oauth2) only when the eval needs expiry. `mode = "open"` under
|
|
430
|
+
`[auth]` opens sessions with no credential at all (a capability URL; `--auth` on the session's
|
|
431
|
+
open overrides; `scheme = "open"` is refused). Full grammar: connector-worlds.md, "The door".
|
|
407
432
|
- `[ui] static = "ui/static"` (+ `entry`, `api_path`, `generated`) is the page over the
|
|
408
433
|
routes: generated by `compile` from the contract unless `generated = false`, served by the
|
|
409
434
|
host on the session's ui origin (`?token=` once, then a cookie; `api_path` rewritten onto
|
|
410
435
|
the routes) and locally on its own port (and at `/__ui/`). A page holds no key; the host presents the world's
|
|
411
|
-
credential for it, the way it does for a
|
|
436
|
+
credential for it, the way it does for a caller holding the session token in `X-World-Session`.
|
|
412
437
|
|
|
413
438
|
`gateway worlds serve PATH --port 8080 [--api-key K]` runs it locally; the runner's admin
|
|
414
439
|
endpoints (`/__world/seed`, `/__world/reset`, `/__world/state`) sit beside the vendor routes
|
|
415
440
|
and answer only the run's `X-World-Admin-Token` (`gateway worlds test` hands a world's pytest
|
|
416
441
|
files theirs as `WORLD_ADMIN_TOKEN`; by hand, use `/__session/*`); `/healthz` answers everyone,
|
|
417
|
-
and a World Host session's doors answer none of the
|
|
442
|
+
and a World Host session's api and tool doors answer none of them (there `/healthz` is the world's own path; its ui origin answers `/healthz` under `api_path` to the twin's own page). `gateway bench push PATH` publishes the world: for a world carrying
|
|
418
443
|
`connector.toml` it packages the api-kind bundle the platform hosts on the way (root
|
|
419
444
|
legacy-shaped on purpose, since the api runner refuses a schema bundle there; `server/`
|
|
420
445
|
needs nothing installed). Open a session with `--surface api` and hand the client that
|
|
421
|
-
URL; the vendor's auth
|
|
446
|
+
URL; the vendor's auth applies as `[auth]` declares it, with the session's own api token as the credential.
|
|
@@ -86,10 +86,131 @@ when that runs out, prints the session with `ready: false` and exits 1 while it
|
|
|
86
86
|
on a cold version pass `--no-wait` and poll `gateway worlds session status <sessionId>` until
|
|
87
87
|
`ready`.
|
|
88
88
|
|
|
89
|
-
The world's API
|
|
90
|
-
|
|
91
|
-
and `WORLD_API_KEY` applies only to a legacy api bundle.
|
|
92
|
-
|
|
89
|
+
The world's API mirrors the vendor's auth as `[auth]` declares it (`door.py`, "The door" below). A
|
|
90
|
+
hosted session's credential is its own api token, `worlds serve`'s the value `--api-key` pins
|
|
91
|
+
(default `conform`), and `WORLD_API_KEY` applies only to a legacy api bundle. A hosted session takes
|
|
92
|
+
that token the way the vendor takes its key: after an `Authorization` word the vendor takes, in the
|
|
93
|
+
declared header, as the Basic user (or password), in the declared query parameter, a token the
|
|
94
|
+
declared login minted for this session, or a signed download link. Platform tooling sends it in the
|
|
95
|
+
platform's own `X-World-Session` header, which every world takes and no vendor's client sends
|
|
96
|
+
(both SDKs' `apiHeaders`); the platform's `Authorization: Bearer` of the session token reaches
|
|
97
|
+
every world for good (the host counts it as `requests.platform_bearer` where the vendor takes its
|
|
98
|
+
key another way). An **open** session asks for nothing: a world sets `mode = "open"` under `[auth]`
|
|
99
|
+
as its sessions' default, and a session's open (`auth: "open"`, `--auth open`) chooses either way.
|
|
100
|
+
Its api and ui answer without a credential on a capability hostname,
|
|
101
|
+
`o-<sid>-<label>[-api].<worlds domain>`, the label a 130-bit secret derived from the session token
|
|
102
|
+
and kept for the session's life (a reopen, a reset, a restart keep the link); the session's own
|
|
103
|
+
`w-<sid>` hostname keeps asking for the credential. On the capability hostname the host is the
|
|
104
|
+
proof, so `ttl_seconds` bounds nothing there; a link the world signs still names the `w-` host. Anything
|
|
105
|
+
else answers the vendor's refusal configured under `[api].unauthorized_status` and
|
|
106
|
+
`[api].unauthorized` (`""` or `'null'` for a bare status with no body, the way some vendors
|
|
107
|
+
answer: `unauthorized_status = 403` with `'{"message": "Invalid API key"}'`); a world that declares
|
|
108
|
+
neither gets the host's own `401 {"error": "invalid session token"}`.
|
|
109
|
+
|
|
110
|
+
### The door
|
|
111
|
+
|
|
112
|
+
`[auth]` says where the credential rides and, optionally, how the vendor trades it for a token and
|
|
113
|
+
which files it hands out as links (synthetic examples; every name is invented):
|
|
114
|
+
|
|
115
|
+
```toml
|
|
116
|
+
[auth] # a key in the URL (?key=...)
|
|
117
|
+
scheme = "query"
|
|
118
|
+
name = "key"
|
|
119
|
+
secret_env = "VENDOR_API_KEY"
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
```toml
|
|
123
|
+
[auth] # Authorization: Bearer <key> or token <key>, as GitHub takes it
|
|
124
|
+
scheme = "bearer"
|
|
125
|
+
prefixes = ["Bearer", "token"] # the words the vendor takes, any case; calls carry the first
|
|
126
|
+
secret_env = "VENDOR_TOKEN"
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
```toml
|
|
130
|
+
[auth] # Basic email:api_token, the key the password under any user
|
|
131
|
+
scheme = "basic"
|
|
132
|
+
credential = "password" # default "user": the key the user, the password empty
|
|
133
|
+
secret_env = "VENDOR_API_TOKEN"
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
```toml
|
|
137
|
+
[auth] # a JSON login that answers a token; calls carry it as a bearer
|
|
138
|
+
scheme = "bearer"
|
|
139
|
+
secret_env = "VENDOR_PASSWORD"
|
|
140
|
+
|
|
141
|
+
[auth.login]
|
|
142
|
+
method = "POST" # the default
|
|
143
|
+
path = "/token" # under base_url's path, as routes are; never /__…
|
|
144
|
+
credential = { json = "password" } # or { form = "secret" }, { header = "X-Api-Key" }; a dotted json path reaches inside
|
|
145
|
+
required = ["username", "realm"] # what else the vendor demands, read from the same place
|
|
146
|
+
reply = '{"access_token": "{token}", "expires_in": "{expires_in}", "token_type": "Bearer"}'
|
|
147
|
+
ttl_seconds = 86400 # opt in to expiry; absent, a token lives as long as the session
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
```toml
|
|
151
|
+
[auth.login] # an api-key login whose reply nests the token: {"data": {"token": …}}
|
|
152
|
+
path = "/login"
|
|
153
|
+
credential = { json = "api_key" }
|
|
154
|
+
reply = '{"data": {"token": "{token}"}}'
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
```toml
|
|
158
|
+
[auth.downloads] # presigned links: these GET routes answer a signed URL without the key
|
|
159
|
+
routes = ["download_results"]
|
|
160
|
+
ttl_seconds = 900 # opt in to expiry; absent, a link lives as long as the session
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
- **The session's credential is the key.** On the login route the session token (or the pin) goes
|
|
164
|
+
where `credential` says; a wrong or missing one is the vendor's 401 (`[api].unauthorized`), a
|
|
165
|
+
missing `required` field or an unreadable body its 400 (`[api].invalid`). The right one is
|
|
166
|
+
answered `reply` (default `{"access_token", "token_type", "expires_in"}`) with the token at
|
|
167
|
+
`"{token}"` and the lifetime at `"{expires_in}"`, a key whose value is `"{expires_in}"` left out
|
|
168
|
+
when the world declares no `ttl_seconds`; `status` (default 200) is the vendor's. On an open
|
|
169
|
+
session's capability hostname the login answers whatever credential the client sends, or none.
|
|
170
|
+
- **Later calls carry the token the way `scheme` says** (bearer, header, Basic, query parameter). A
|
|
171
|
+
login world takes only tokens it minted, never the raw key, as the vendor does; the platform's
|
|
172
|
+
`X-World-Session` header still reaches a hosted session (the SDKs send it), and the host then
|
|
173
|
+
presents the world a freshly minted token and drops the header. A refresh token an oauth2 world
|
|
174
|
+
minted answers every time it is presented, wherever the session runs, and lives as long as the
|
|
175
|
+
session even where `[auth]` declares `ttl_seconds`, which bounds the access token alone: the
|
|
176
|
+
short access token and long refresh real vendors hand out. An oauth2
|
|
177
|
+
`token_url` path may name a segment (`/ccx/oauth2/{tenant}/token`), which any value fills.
|
|
178
|
+
- **Tokens and links are bound to the session.** A token is an HS256 JWT, a link an HMAC-SHA256 over
|
|
179
|
+
its path and expiry, each under a key derived from the session's credential (no new secret): a
|
|
180
|
+
token or link from another session never verifies, a tampered one is refused, one past a declared
|
|
181
|
+
lifetime is refused, and a supervisor, its version worker and any host the session moves to check
|
|
182
|
+
them alike. Nothing expires unless the world declares `ttl_seconds`. Comparisons run in constant
|
|
183
|
+
time.
|
|
184
|
+
- **A download route is never open.** A handler hands out a link with
|
|
185
|
+
`ctx.signed_url("/exports/<id>/results.zip")` (the path as the client sends it under base_url's
|
|
186
|
+
path; an optional second argument gives it a lifetime in seconds); the route answers that link
|
|
187
|
+
without the key for the session's life, or until its declared lifetime ends, and the credential
|
|
188
|
+
as any other route does. The route names its object in the path and declares no `params`, and a
|
|
189
|
+
link carries `signature` (and `expires`, when it has a lifetime) alone, so it reaches the one
|
|
190
|
+
object it was signed for.
|
|
191
|
+
- **Nothing logs a key.** The call log and the request ledger drop a query key and a link's
|
|
192
|
+
`signature`; a token or a link's signature inside a longer string reads `[redacted]` in place;
|
|
193
|
+
the ledger keeps no login request body and scrubs every body it keeps (JSON or text, the answer
|
|
194
|
+
that hands out a link and a page link the world echoes included) of the session credential, a
|
|
195
|
+
carried parameter's value, a minted token and a signature; the host writes no request URL to its
|
|
196
|
+
log, and the chart's World Host Ingress keeps no nginx access log (an ALB access log you turn on,
|
|
197
|
+
or another controller's default access log, would record URLs).
|
|
198
|
+
- **A link reaches its own route.** A request a signed link admits is routed among the
|
|
199
|
+
`[auth.downloads]` routes alone, so a sibling route on the same path (one that takes `params`)
|
|
200
|
+
never answers a link.
|
|
201
|
+
- **Capture does not log in.** `connector capture` and the live hook lanes send `secret_env` in the
|
|
202
|
+
scheme's carrier as it is (oauth2 alone exchanges first): for a vendor that trades its key for a
|
|
203
|
+
token, set `secret_env` to a token its own login minted.
|
|
204
|
+
- `scheme = "oauth2"` is the RFC 6749 token route (below); it takes no `[auth.login]`, and neither
|
|
205
|
+
does `none`. A login path that is also a declared route of the same method is refused, as is a
|
|
206
|
+
download route that answers anything but GET or declares any `params`, and a setting of another
|
|
207
|
+
scheme (`prefixes` on `header`, `credential` on `bearer`).
|
|
208
|
+
- **Uploads reach handlers.** A write's body is read by its media type: JSON, a form, a multipart
|
|
209
|
+
form (each field's first part; a file part reads `{"filename", "content_type", "size"}` with its
|
|
210
|
+
bytes in `ctx.files[name].content`) or multipart/related (the first part as JSON, the rest in
|
|
211
|
+
`ctx.files["1"]`, …). Any other type (`application/octet-stream`, `text/csv`) is an upload: a
|
|
212
|
+
handler route reads `body` as `{}` and the bytes in `ctx.raw`; a declared route refuses it as the
|
|
213
|
+
vendor's 400, as it refuses JSON that does not parse.
|
|
93
214
|
|
|
94
215
|
The runner's own routes (`/__world/seed|reset|state|calls`) answer only `X-World-Admin-Token`. A session
|
|
95
216
|
pod and a Batch run mint that token themselves. A twin you host yourself (`hostedUrl`) gets it from
|
|
@@ -126,10 +247,10 @@ least one `[[operations]]` entry.
|
|
|
126
247
|
| Table | Keys | Notes |
|
|
127
248
|
| --------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
128
249
|
| `[connector]` | `slug`, `base_url` | `base_url` is an http(s) origin plus optional path, no query. Requests carrying that path prefix are accepted too. |
|
|
129
|
-
| `[auth]` | `scheme` (`header`, `bearer`, `basic`, `query`, `oauth2`, `none`), `name`, `secret_env`, `headers`, `token_url`, `client_id_env`, `scope`
|
|
250
|
+
| `[auth]` | `scheme` (`header`, `bearer`, `basic`, `query`, `oauth2`, `none`), `name`, `secret_env`, `headers`, `prefixes`, `credential`, `token_url`, `client_id_env`, `scope`, `[auth.login]`, `[auth.downloads]` | The secret is only ever named by an env var. `prefixes` (bearer, oauth2) lists the `Authorization` words the vendor takes; `credential = "password"` (basic) reads the key as the Basic password. A token path (`token_url`'s, `[api] token_path`) is a route path (each `{name}` once) that no declared POST operation answers too. `headers = { "X-App-Token" = "VENDOR_APP_TOKEN" }` adds more credential headers. `oauth2` is the client-credentials grant: capture exchanges `client_id_env` + `secret_env` at `token_url` (a path under `base_url` or an absolute URL, default `/oauth/token`) for the bearer it then sends, and the served world mints bearers on the same path. `[auth.login]` is the vendor's own login route and `[auth.downloads]` its presigned download routes ("The door" above). |
|
|
130
251
|
| `[capture]` | `requests_per_minute`, `timeout_seconds`, `max_pages`, `cursor_param`, `cursor_field`, `cursor_header` | Pacing between pages; providers ask us to be sparing. The next cursor is read from a body field or a response header (Torchlight sends `Cursor:`), never both, and is sent back as `cursor_param`; a cursor that is an absolute URL on the connection's host (DRF's `next`) is followed by its query. |
|
|
131
252
|
| `[redact]` | `hash`, `drop`, `round`, `preserve` | Applied before a body reaches disk. `preserve` hashes shape-preservingly (same length, digits stay digits, letters keep their case, separators stay; slug-salted, deterministic) so an account number still meets the field's format — `hash` made it a 71-char `sha256:` string the contract refused (Brex). `hash` writes `sha256:<hex>` salted with the slug so joins survive, element by element for arrays so the shape the contract expects survives too; `drop` removes; `round = { lat = 2, lon = 2 }` keeps numbers numeric but coarsens them. A bare name matches at any depth; a dotted path (`results.*.email`, `*` for list items) matches that place only, so a catalog's `assets` count map keeps its `email` key. Field names keep the vendor's casing. |
|
|
132
|
-
| `[api]` | `envelope`, `hits_key`, `cursor_key`, `page`, `limit_param`, `offset_param`, `cursor_param`, `page_size`, `previous_key`, `page_links`, `page_headers`, `cursor_absent_when_done`, `unauthorized`, `not_found`, `invalid`, `*_status`, `error_content_type`, `token_path`, `handlers`, `before` | `envelope = "results"` wraps rows as `{results, hits, cursor}` under the vendor's key names; empty means a bare array. `hits_key = ""` leaves the count out and `cursor_absent_when_done = true` leaves the cursor key out of the last page (instead of `""`), together the `{items, next_cursor?}` shape Brex-style vendors answer and clients test with `"next_cursor" in body`. `page` is `none`, `offset` or `cursor`, with the vendor's parameter names; `previous_key` adds the previous page and `page_links = true` renders next/previous as absolute URLs of the served host, the way Django REST Framework answers; `page_headers = { cursor = "Cursor", total = "Total-Count" }` sends the same values as response headers for vendors that page that way (Torchlight). Error bodies are JSON strings replayed verbatim with the vendor's statuses and content type (`application/problem+json` for RFC 7807 vendors); `""` (or `'null'`) means no body at all, for vendors that answer a bare 401. An oauth2 world mints bearers on `token_path` (default the spec's `token_url`) and honours only those. `handlers` names the world's Python module (default `api/handlers.py` when present); `before` names a gate in it that runs before every operation, declarative ones included, and returns `None` to let the request through or an answer (quota, allow-list, key state) to stop it. |
|
|
253
|
+
| `[api]` | `envelope`, `hits_key`, `cursor_key`, `page`, `limit_param`, `offset_param`, `cursor_param`, `page_size`, `previous_key`, `page_links`, `page_headers`, `link_header`, `cursor_absent_when_done`, `unauthorized`, `not_found`, `invalid`, `wrong_method`, `*_status`, `error_content_type`, `cors`, `token_path`, `handlers`, `before` | `envelope = "results"` wraps rows as `{results, hits, cursor}` under the vendor's key names; empty means a bare array. `hits_key = ""` leaves the count out and `cursor_absent_when_done = true` leaves the cursor key out of the last page (instead of `""`), together the `{items, next_cursor?}` shape Brex-style vendors answer and clients test with `"next_cursor" in body`. `page` is `none`, `offset` or `cursor`, with the vendor's parameter names; `previous_key` adds the previous page and `page_links = true` renders next/previous as absolute URLs of the served host, the way Django REST Framework answers; `page_headers = { cursor = "Cursor", total = "Total-Count" }` sends the same values as response headers for vendors that page that way (Torchlight). Error bodies are JSON strings replayed verbatim with the vendor's statuses and content type (`application/problem+json` for RFC 7807 vendors); `""` (or `'null'`) means no body at all, for vendors that answer a bare 401. A path the world does not serve answers `not_found`; a path it serves, called with a method none of its routes answer, answers `wrong_method` (405 by default with the `Allow` header RFC 9110 requires, no body unless declared; `wrong_method_status` for a vendor that answers another status, which carries no `Allow`). A write the contract refuses answers `invalid` exactly as declared, and the refusal's rule and schema path go to the host's log, never the answer (the request ledger keeps the request). `link_header = true` sends RFC 8288 `Link: <next>; rel="next"` (and `rel="prev"`) on list answers, each URL the request's own with the page's paging parameters, as GitHub pages. `cors = true` is a vendor whose API answers browsers: an `OPTIONS` preflight is answered 204 before any credential (`Access-Control-Allow-Origin: *`, the methods the world serves, the headers the page asked to send) and every answer names any origin and exposes its headers (`Link`); without it an `OPTIONS` is routed like any other method. `gateway worlds serve` (and `worlds test`, which opens it) answers only this machine's pages: a request from another site's `Origin` is refused 403, since its pin is public. A `HEAD` answers as `GET` without a body, and a `DELETE` that carries a body is read like a `POST`'s. An oauth2 world mints bearers on `token_path` (default the spec's `token_url`) and honours only those. `handlers` names the world's Python module (default `api/handlers.py` when present); `before` names a gate in it that runs before every operation, declarative ones included, and returns `None` to let the request through or an answer (quota, allow-list, key state) to stop it. |
|
|
133
254
|
| `[[operations]]` | `name`, `method`, `path`, `params`, `entity`, `results` | `path` may carry `{args}`; a `params` name keeps the vendor's spelling, dots included (`query.bibliographic`, `filter.overallStatus`), and may end in `[]` (`expand[]`, `status[]`) for a key the vendor repeats, which capture then sends repeated and the served route filters on any value of; `results` is the dotted path to the row array (empty when the body is the row). An operation without `entity` never seeds and answers 501 `not_modelled`. Routes are never tools: an agent calls them over HTTP (the api url and token `session open --surface api` prints), and a tool exists only where `x-gateway.tools` declares one with a handler. An `entity` (or an `explode` target) that `schema/world.json` does not declare is refused by `connector validate`, `schema compile`, `schema check` and the platform's describe (`gateway worlds describe`, local or platform) alike — `operation X lands on entity Y, which schema/world.json does not declare (declared: …)` at `/operations/<i>/entity` — and nothing is written; hand-edited TOML meets the same gate `route add` applies. |
|
|
134
255
|
| `[operations.ingest]` | `drop`, `explode`, `carry_args` | `drop` removes vendor fields the entity does not carry; each `[[operations.ingest.explode]]` (`field`, `entity`, `carry`, `take`) turns a nested array into child rows; `carry_args = { account_id = "id" }` stamps the capture's path argument or query parameter onto every row before projection (a statement only knows its account from the URL it was fetched under). `connector ingest --dry-run [--report FILE]` lists every violation of every capture and writes nothing. |
|
|
135
256
|
| `[operations.api]` | `filter`, `match`, `action`, `status`, `single`, `envelope`, `wrapper`, `page`, `body`, `defaults`, `generate`, `handler`, `sample`, `sample_headers`, `sample_body`, `not_found`, `not_found_status`, `cursor_absent_when_done` | `filter` maps a path argument or declared query parameter to the row field it selects on (case-insensitive, any depth; an absent parameter does not narrow, one sent more than once matches rows carrying any of its values); `match = { name = "contains" }` switches an argument from equality to substring or `prefix`, and a list field matches when any element does. `action` is `read` (default), `create`, `update`, `delete` or `handler`: writes persist through the contract, `body` maps row fields (dotted for nested) to request-body paths, `defaults` fixes constants, `generate` fills `uuid`, `int` or `now`; `update` and `delete` find their row through `filter` and answer the vendor's 404 otherwise; they, and a `single` read with a path argument, refuse (`finds its row by nothing`, 400) when neither `filter` nor a path argument naming a field selects a row, rather than act on some row; a singleton served under a scoped path answers through `action = "handler"`. `status` is the 2xx the vendor answers (201 for create, 204 for delete by default), `single` returns the row itself instead of an envelope, `envelope` and `page` override the world-wide setting per route. A dotted `envelope` nests the answer the way the vendor does (`message.items` answers `{message: {items, total-results, next-cursor}}`), and a single row is wrapped only by its own route's `envelope` (`message` answers `{message: row}`). `wrapper` is the rest of the vendor's envelope as a JSON object string (`'{"status": "ok", "message-type": "work-list"}'`), the answer laid onto it key by key; `schema init --from-openapi` writes it from the document. `handler` names a function `def name(ctx, args, query, body)` in the handlers module that returns a body, `(status, body)` or `(status, body, headers)`; `ctx` reads and writes the store (`rows`, `get`, `find`, `insert`, `update`, `delete`), keeps time (`now`, `elapsed`, `ladder` for status machines, frozen by `GATEWAY_WORLD_CLOCK`), mints ids (`uuid`) and shapes answers (`envelope`, `not_found`, `invalid`) and sees the request's `headers` and `query_all` (every value of every parameter, `{"expand[]": ["a", "b"]}`; the `query` argument keeps the first value only). `not_found` / `not_found_status` answer this route's missing row the vendor's own way (a 403 with its own message for one resource) and fall back to `[api]`; `cursor_absent_when_done` overrides `[api]` for this route. `sample = { ip = "203.0.113.5" }` gives the conformance run a value a validating route accepts when no seeded row can supply one; `sample_headers = { "Idempotency-Key" = "conform-1" }` and `sample_body = { reason = "conform" }` give it the headers and body a route demands (read by `connector conform` only; the served API never sends requests). |
|
|
@@ -192,7 +313,8 @@ does all of that from the same `connector.toml`:
|
|
|
192
313
|
handlers and the gate may answer with headers too (`SpyCloud-Error`, `Retry-After`).
|
|
193
314
|
- An oauth2 world answers the vendor's token route before any authorization, so a client that
|
|
194
315
|
performs the grant first works unchanged; a hosted session checks the client secret against its
|
|
195
|
-
own api token, `worlds serve` against the `--api-key` pin.
|
|
316
|
+
own api token, `worlds serve` against the `--api-key` pin. A declared `[auth.login]` route is
|
|
317
|
+
answered the same way, ahead of every handler.
|
|
196
318
|
|
|
197
319
|
The handlers ship inside the api bundle beside the world, so the hosted container runs them
|
|
198
320
|
with nothing installed. The abstraction is the platform's; the handlers are the customer's.
|
|
@@ -264,17 +264,18 @@ schema type, the same rule in the TypeScript CLI (`worldData/cast.ts`) and the P
|
|
|
264
264
|
|
|
265
265
|
| Field type | Cell | Value |
|
|
266
266
|
| --- | --- | --- |
|
|
267
|
-
| `integer` | `42`, `-7` | the number; `4.5` or `abc`
|
|
267
|
+
| `integer` | `42`, `-7` | the number; `4.5` or `abc` stays text, refused by the contract |
|
|
268
268
|
| `number` | `12.5`, `1e3`, `3` | the number (always a float) |
|
|
269
|
-
| `boolean` | `true`/`false`/`1`/`0`, any case | the boolean; `yes`
|
|
269
|
+
| `boolean` | `true`/`false`/`1`/`0`, any case | the boolean; `yes` stays text, refused by the contract |
|
|
270
270
|
| `array` / `object` | JSON text of that shape (quote it in the CSV: `"[""a"",""b""]"`) | the parsed value |
|
|
271
271
|
| `string` | anything | the text as given |
|
|
272
272
|
| `integer \| string` (union) | each member tried in order, `string` last | the first that fits |
|
|
273
|
-
| any, empty cell |
|
|
273
|
+
| any, empty cell | (nothing) | null when the type admits null; `""` for a required string; left out otherwise (a required one is then refused by the contract) |
|
|
274
274
|
|
|
275
|
-
A cell that fits none
|
|
276
|
-
|
|
277
|
-
|
|
275
|
+
A cell that fits none stays its text and the contract refuses the row as it refuses a JSON row
|
|
276
|
+
— `taxi_trip {"id":3}.fare: Expected number`, with the usual pointer — so a file reports every
|
|
277
|
+
bad row at once (`--report` writes them all) and nothing is written. A column the schema does
|
|
278
|
+
not name stays text and the gate refuses it as an unknown field. A CSV
|
|
278
279
|
on the drive (`drive:<path>`) is still refused: import it from the machine that holds it, or
|
|
279
280
|
through `worlds sandbox import` when it sits in a sandbox.
|
|
280
281
|
|
|
@@ -311,8 +312,11 @@ the package version keeps every snapshot (local, hosted, pinned), with no re-imp
|
|
|
311
312
|
that only adds to the domain (a new entity, a new field that is not required, another
|
|
312
313
|
`domain_version`) carries the rows: compile migrates the snapshots the tree holds and keeps the
|
|
313
314
|
pin with the stored domain's `lock` and `entities`, `sandbox check` says the commit migrates the
|
|
314
|
-
hosted snapshot, and `sandbox commit` migrates it
|
|
315
|
-
|
|
315
|
+
hosted snapshot, and `sandbox commit` migrates it as a job (a `migration` batch: the CLI follows it,
|
|
316
|
+
MCP answers `{batchId}` for `get_data_batch`) and re-pins the tree (a CLI push of such a change is
|
|
317
|
+
refused, naming the sandbox). Importing exactly the batch behind the served snapshot again (the
|
|
318
|
+
same chunks in the same order, with `redact` off) is a no-op: the batch lands on the same version
|
|
319
|
+
and names the one it repeats (`unchangedFrom`). Any other change to an entity, a field (its description
|
|
316
320
|
or title too), a key or a relationship refuses it, naming each change:
|
|
317
321
|
|
|
318
322
|
```text
|
|
@@ -53,10 +53,9 @@ A CSV/TSV is decided by its name and read as a stream (bounded memory at any siz
|
|
|
53
53
|
`--delimiter <char>` splits the cells (one character, `\t` for a tab; default `,`, tab for a
|
|
54
54
|
`.tsv`). The header row names the fields and **every cell is cast by the entity's schema
|
|
55
55
|
type**: `integer`/`number` parse as numbers, `boolean` takes true/false/1/0, `array`/`object`
|
|
56
|
-
parse as JSON, `string` stays text; an empty cell is null when the field admits null, left out
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
pinned by `tests/conformance/csv_cast.json`. Without `--entity` a CSV is refused.
|
|
56
|
+
parse as JSON, `string` stays text; an empty cell is null when the field admits null, `""` for a required string, left out otherwise. A cell that does not fit stays text and the contract refuses its row as it refuses a JSON row:
|
|
57
|
+
every bad row named by entity, key and field (`taxi_trip {"id":3}.fare: Expected number`), all counted, each in `--report`.
|
|
58
|
+
Same rule in both CLIs, pinned by `tests/conformance/csv_cast.json`. Without `--entity` a CSV is refused.
|
|
60
59
|
|
|
61
60
|
Rules:
|
|
62
61
|
|
|
@@ -156,10 +155,10 @@ JSONL chunks of `--chunk-bytes` (64 MiB default) or `--chunk-rows N`, uploads th
|
|
|
156
155
|
`--parallel 4` at a time straight to the bucket, and completes the batch; the platform
|
|
157
156
|
validates every row against the contract, applies the batch onto the world's data snapshot
|
|
158
157
|
and, with `--publish now` (default), cuts a data-only version (no image rebuild) on your open edit session
|
|
159
|
-
(`editSessionId`; only your credential reads it
|
|
158
|
+
(`editSessionId`; only your credential's unpinned reads see it, and `slug@main`, runs and tasks keep the sealed `main`, until `worlds edit close <slug>` or 60 idle minutes seal it). `--wait`
|
|
160
159
|
(default) polls to the end and prints the batch: `{batchId, state, rows, refusalCount,
|
|
161
160
|
refusals[:200], entityCounts, versionId?, batchDigest?}`. `--no-wait` prints `{batchId,
|
|
162
|
-
state}` and `gateway worlds data status <batchId
|
|
161
|
+
state}` and `gateway worlds data status <batchId> [--report <file>]` reads it later. Exit codes: 0 applied or
|
|
163
162
|
published, 2 refused (the report is written when `--report` is given: JSONL of refusal
|
|
164
163
|
records, at most 10,000 then `{"truncated": true, "total": N}`), 1 error.
|
|
165
164
|
|
|
@@ -285,7 +284,7 @@ what nests, what a write's `args` become, which call wins — in code you read t
|
|
|
285
284
|
write. The worked CRM example (nested contacts/deals exploded out of an account read, a
|
|
286
285
|
deal's nested account lifted, activity rows from `crm_log_activity` args keyed by the
|
|
287
286
|
observation id, a deletion turned into an event row, latest call wins) is on
|
|
288
|
-
|
|
287
|
+
https://docs.surfacearea.ai/worlds/data in both languages; the fixtures live in
|
|
289
288
|
`gatewaysdk/sdk/tests/environment/fixtures/crm/` (`records.jsonl`, `transform.py`,
|
|
290
289
|
`transform.ts`, `ingest.toml`, `rows.json`).
|
|
291
290
|
|
|
@@ -369,7 +368,7 @@ the console agent has `run_gateway_cli data ingest` on a sandboxed directory.
|
|
|
369
368
|
|
|
370
369
|
```bash
|
|
371
370
|
gateway worlds sandbox exec <id> 'curl -sSL https://example.test/trips.csv -o data/trips.csv' # or a script that writes rows
|
|
372
|
-
gateway worlds sandbox import <id> data/trips.csv --entity taxi_trip [--delimiter ...] [--mode append|replace] [--dry-run]
|
|
371
|
+
gateway worlds sandbox import <id> data/trips.csv --entity taxi_trip [--delimiter ...] [--mode append|replace] [--dry-run] [--report refusals.jsonl]
|
|
373
372
|
gateway worlds sandbox commit <id> -m "tools" # the SOURCE tree only; rows are not in a commit
|
|
374
373
|
```
|
|
375
374
|
|
|
@@ -378,8 +377,9 @@ sandbox (`POST /api/public/benchmark-containers/sandboxes/{id}/import`; the sand
|
|
|
378
377
|
chunks it to JSONL in a scratch dir the platform reads and removes) and streams it through the
|
|
379
378
|
same ledger as `data import <slug>`: contract, refusals with pointers, `--mode`, `--merge`,
|
|
380
379
|
`--redact`, `--per-chunk`, `-m`. Prints `{versionId, entityCounts, rowCount}`, or `{batchId,
|
|
381
|
-
state}` to poll with `data status`.
|
|
382
|
-
`
|
|
380
|
+
state}` to poll with `data status`. Refused rows exit 2: `{batchId, state: refused, refusalCount, refusals}` (all counted,
|
|
381
|
+
the first 200 inline); `--report` writes every refusal as JSONL, a `--per-chunk` import's and a polled batch's too. MCP:
|
|
382
|
+
`import_world_sandbox_data({ sandboxId, path, entity?, mode? })`. A new-world sandbox commits once first; `commit` never carries rows.
|
|
383
383
|
|
|
384
384
|
### 3. Verify on a live session
|
|
385
385
|
|
|
@@ -390,7 +390,7 @@ gateway worlds session open <slug> -t <scenario> --surface api # prints s
|
|
|
390
390
|
gateway worlds session seed <SESSION_ID> rows.json # kind:"seed" call, waits for the result
|
|
391
391
|
gateway worlds session seed <SESSION_ID> rows.json --redact refuse # same, refusing plaintext in a redacted field (apply hashes it first)
|
|
392
392
|
gateway worlds session seed <SESSION_ID> rows.json --mode replace # swap the named entities
|
|
393
|
-
curl -H "
|
|
393
|
+
curl -H "X-World-Session: $API_TOKEN" "$API_URL/<route>" # read it back (api.token from session open)
|
|
394
394
|
```
|
|
395
395
|
|
|
396
396
|
Prints `{sessionId, mode, result}`. MCP: `seed_world_session({ sessionId, rows, mode? })` waits
|
|
@@ -457,7 +457,7 @@ Sanitize before importing: hash or drop personal fields the world's `[redact]` p
|
|
|
457
457
|
| refused, kind `api` (or `harbor`), naming the file to edit | the world is a generated bundle or a task suite | edit the named source, `bench push` again |
|
|
458
458
|
| `version` missing from the result | a local directory was given | push with `gateway bench push <dir>` when ready |
|
|
459
459
|
| `... is over what a world's source tree carries as data (4 MiB / 100000 rows ...)`; `data/initial.json would be N MiB with these rows, over the 4 MiB publication reads a source file whole`; `exceed the inline cap ... without the bundled runtime` | a directory import above the tree cap; or no `python3`, so the rows travel whole (5,000 / 4 MiB) | `worlds sandbox import` from a sandbox, `data import <slug>` with keys (never keys into the sandbox); install python3 |
|
|
460
|
-
|
|
|
460
|
+
| `<entity> {"<key>":…}.<field>: Expected number` (integer, boolean, array, object) on a CSV row | a cell the field's schema type does not fit: it stays text and the contract refuses its row | fix the cell or the header; `--report` lists every row; nothing was written |
|
|
461
461
|
| exit 2, `state: refused` | the platform refused the batch; `--report` has every record | fix the rows named, `--resume` is not for this: run the import again |
|
|
462
462
|
| `nothing to resume` / `only an open or uploading batch resumes` | no incomplete manifest, or the batch already ran | run the import without `--resume` |
|
|
463
463
|
| `409 base_mismatch` from `/__world/seed` | the overlay named another snapshot digest than the pod is on | the runner posts the digest of the version's snapshot; a mismatch is a version mismatch, never reseed silently |
|
|
@@ -497,4 +497,4 @@ handler or route edit keeps the imported data while an entity, field or key edit
|
|
|
497
497
|
- `connector-schema-authoring`: the data model the rows are checked against.
|
|
498
498
|
- `documentation/features/world-data-ingestion.md`: what exists, the contract, code pointers.
|
|
499
499
|
- `documentation/features/connector-worlds.md`: captures, ingest, serving the vendor's API.
|
|
500
|
-
- Consumer page:
|
|
500
|
+
- Consumer page: https://docs.surfacearea.ai/worlds/data.
|