@withgateway/sdk 0.3.0 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/README.md +6 -1
  2. package/dist/cli.js +3971 -2338
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +3950 -2318
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +214 -41
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +206 -35
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/{index-DAb-FCMg.d.ts → index-DUHmS74p.d.ts} +3 -13
  13. package/dist/{index-D6oi1TWE.d.mts → index-mJhG3vZU.d.mts} +3 -13
  14. package/dist/index.d.mts +4 -4
  15. package/dist/index.d.ts +4 -4
  16. package/dist/index.js +1456 -564
  17. package/dist/index.js.map +1 -1
  18. package/dist/index.mjs +1433 -557
  19. package/dist/index.mjs.map +1 -1
  20. package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
  21. package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
  22. package/dist/traces/index.d.mts +1 -1
  23. package/dist/traces/index.d.ts +1 -1
  24. package/dist/traces/index.js +211 -38
  25. package/dist/traces/index.js.map +1 -1
  26. package/dist/traces/index.mjs +203 -32
  27. package/dist/traces/index.mjs.map +1 -1
  28. package/dist/tracing/index.d.mts +2 -2
  29. package/dist/tracing/index.d.ts +2 -2
  30. package/dist/tracing/index.js +96 -21
  31. package/dist/tracing/index.js.map +1 -1
  32. package/dist/tracing/index.mjs +96 -21
  33. package/dist/tracing/index.mjs.map +1 -1
  34. package/dist/worlds/index.d.mts +300 -123
  35. package/dist/worlds/index.d.ts +300 -123
  36. package/dist/worlds/index.js +1385 -508
  37. package/dist/worlds/index.js.map +1 -1
  38. package/dist/worlds/index.mjs +1377 -501
  39. package/dist/worlds/index.mjs.map +1 -1
  40. package/package.json +1 -1
  41. package/runtime/gatewaysdk/environment/_bundle.py +2 -27
  42. package/runtime/gatewaysdk/environment/_materialize.py +49 -69
  43. package/runtime/gatewaysdk/environment/_world.py +122 -197
  44. package/runtime/gatewaysdk/environment/backing.py +137 -0
  45. package/runtime/gatewaysdk/environment/core.py +1 -1
  46. package/runtime/gatewaysdk/environment/legacy_sqlite.py +301 -0
  47. package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
  48. package/runtime/gatewaysdk/environment/runtime.py +8 -13
  49. package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
  50. package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
  51. package/runtime/gatewaysdk/environment/schema/api.py +266 -515
  52. package/runtime/gatewaysdk/environment/schema/batch.py +42 -37
  53. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
  54. package/runtime/gatewaysdk/environment/schema/call_log.py +291 -0
  55. package/runtime/gatewaysdk/environment/schema/call_redaction.py +344 -0
  56. package/runtime/gatewaysdk/environment/schema/cast.py +31 -4
  57. package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
  58. package/runtime/gatewaysdk/environment/schema/conform.py +8 -12
  59. package/runtime/gatewaysdk/environment/schema/connector.py +27 -62
  60. package/runtime/gatewaysdk/environment/schema/data.py +50 -207
  61. package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
  62. package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
  63. package/runtime/gatewaysdk/environment/schema/database/base.py +995 -0
  64. package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
  65. package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
  66. package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
  67. package/runtime/gatewaysdk/environment/schema/database/guard.py +208 -0
  68. package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
  69. package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
  70. package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
  71. package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
  72. package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
  73. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +367 -0
  74. package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
  75. package/runtime/gatewaysdk/environment/schema/envelope.py +169 -0
  76. package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
  77. package/runtime/gatewaysdk/environment/schema/grading.py +45 -30
  78. package/runtime/gatewaysdk/environment/schema/handler_errors.py +142 -12
  79. package/runtime/gatewaysdk/environment/schema/host.py +158 -250
  80. package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
  81. package/runtime/gatewaysdk/environment/schema/local.py +44 -71
  82. package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
  83. package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
  84. package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
  85. package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
  86. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
  87. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
  88. package/runtime/gatewaysdk/environment/schema/platform.py +124 -178
  89. package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
  90. package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
  91. package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
  92. package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
  93. package/runtime/gatewaysdk/environment/schema/runtime.py +77 -160
  94. package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
  95. package/runtime/gatewaysdk/environment/schema/session_files.py +166 -0
  96. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +2 -2
  97. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +22 -14
  98. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +29 -3
  99. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +16 -16
  100. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +18 -18
  101. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
  102. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +15 -0
  103. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
  104. package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
  105. package/runtime/gatewaysdk/environment/schema/snapshot.py +385 -129
  106. package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
  107. package/runtime/gatewaysdk/environment/schema/tasks.py +140 -55
  108. package/runtime/gatewaysdk/environment/schema/tools_world.py +110 -87
  109. package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
  110. package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
  111. package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
  112. package/runtime/gatewaysdk/environment/schema/validation.py +166 -40
  113. package/runtime/gatewaysdk/environment/schema/workers.py +142 -57
  114. package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
  115. package/runtime/gatewaysdk/environment/schema/world_tests.py +71 -39
  116. package/runtime/gatewaysdk/environment/score.py +57 -0
  117. package/runtime/gatewaysdk/environment/secret_values.py +36 -0
  118. package/runtime/gatewaysdk/environment/world.py +42 -3
  119. package/runtime/gatewaysdk/environment/world_session_contract.py +32 -5
  120. package/skills/benchmarks-getting-started/SKILL.md +2 -2
  121. package/skills/connector-schema-authoring/SKILL.md +22 -14
  122. package/skills/connector-schema-authoring/connector-worlds.md +26 -10
  123. package/skills/connector-schema-authoring/world-schema.md +111 -51
  124. package/skills/world-data-ingestion/ROWS.md +29 -3
  125. package/skills/world-data-ingestion/SKILL.md +16 -16
  126. package/skills/worlds-getting-started/SKILL.md +18 -18
  127. package/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
  128. package/skills/worlds-getting-started/references/gotchas.md +15 -0
  129. package/skills/worlds-getting-started/references/openapi-import.md +50 -21
  130. package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
@@ -21,16 +21,15 @@ import shutil
21
21
  import sys
22
22
  import tarfile
23
23
  import tempfile
24
- from dataclasses import asdict
25
24
  from pathlib import Path, PurePosixPath
26
25
  from typing import TYPE_CHECKING, Any
27
26
 
28
27
  from .._materialize import materialize_template
29
28
  from ..world import WorldTree
30
- from . import ContractError, SQLiteStore, bundle_mode, check_bundle, compile_bundle, platform_import, platform_query
29
+ from . import ContractError, WorldDatabase, bundle_mode, check_bundle, compile_bundle, platform_import, platform_query
31
30
  from . import snapshot as snapshots
32
31
  from .cast import world_fields
33
- from .compiler import retarget_empty_journals
32
+ from .compiler import CompilePlan, retarget_journals
34
33
  from .handler_scan import HandlerScan
35
34
  from .redaction import check_world_redaction
36
35
  from .scaffold import request_files
@@ -74,6 +73,8 @@ COMMANDS = (
74
73
  "reconcile",
75
74
  "counts",
76
75
  "query",
76
+ "sql",
77
+ "verify-legacy",
77
78
  "api-bundle",
78
79
  "templates",
79
80
  "validate-rows",
@@ -91,20 +92,15 @@ def compile_world(root: Path) -> dict:
91
92
 
92
93
  The platform stores compiled trees, so a world whose authored inputs changed
93
94
  (``schema/world.json``, ``gateway-env.toml``, handlers) is recompiled here
94
- rather than rejected with ``lock_mismatch``. Only a journal that holds no
95
- state (``mutations == []``) follows the contract to its new identity, even
96
- across a changed ``$id`` (``retarget_empty_journals``); a journal with rows is
97
- refused, never migrated silently — state migration stays manual (re-import
98
- the rows against the new schema). The snapshots are then rebuilt loudly:
99
- data the contract refuses is a rejection, not a dropped snapshot.
95
+ rather than rejected with ``lock_mismatch``. A journal follows the contract when
96
+ its rows carry over (``retarget_journals``) and is refused otherwise, before
97
+ anything is written. The snapshots are then rebuilt loudly: data the contract
98
+ refuses is a rejection, not a dropped snapshot.
100
99
  """
101
100
  from .snapshot import build_all
102
101
 
103
102
  if bundle_mode(root) is None:
104
- fail(
105
- "mode",
106
- "Expected a schema-backed mock-tools world (gateway-env.toml with [schema] language)",
107
- )
103
+ fail("mode", "Expected a schema-backed mock-tools world (gateway-env.toml with [schema] language)")
108
104
  added = []
109
105
  base = root / "schema/base.json"
110
106
  if not base.exists():
@@ -113,7 +109,9 @@ def compile_world(root: Path) -> dict:
113
109
  base.parent.mkdir(parents=True, exist_ok=True)
114
110
  base.write_bytes((TEMPLATES / "base.json").read_bytes())
115
111
  added.append("schema/base.json")
116
- contract = compile_bundle(root, snapshots=False)
112
+ plan = CompilePlan.of(root)
113
+ retargeted = retarget_journals(root, plan.contract)
114
+ contract = plan.write()
117
115
  initial = root / "data/initial.json"
118
116
  if not initial.exists():
119
117
  initial.parent.mkdir(parents=True, exist_ok=True)
@@ -129,7 +127,6 @@ def compile_world(root: Path) -> dict:
129
127
  )
130
128
  )
131
129
  added.append("data/initial.json")
132
- retargeted = retarget_empty_journals(root, contract)
133
130
  build_all(root, contract)
134
131
  return {"added": added, "retargeted": retargeted}
135
132
 
@@ -144,6 +141,7 @@ def append_batch(root: Path, batch: dict, *, source: str | None = None) -> dict:
144
141
  Replaying every prior journal first makes a repeated batch id idempotent and
145
142
  a reused id with different content a conflict, exactly as in the store.
146
143
  """
144
+ snapshots.refuse_pinned_rows(root)
147
145
  contract = check_bundle(root)
148
146
  members(
149
147
  batch,
@@ -152,7 +150,7 @@ def append_batch(root: Path, batch: dict, *, source: str | None = None) -> dict:
152
150
  "/batch",
153
151
  )
154
152
  nonempty(batch["id"], "/batch/id")
155
- if "target" in batch and batch["target"] != contract.target:
153
+ if "target" in batch and not contract.targets(batch["target"]):
156
154
  fail(
157
155
  "batch_target",
158
156
  "Batch targets a different domain identity/version/hash",
@@ -183,7 +181,7 @@ def append_batch(root: Path, batch: dict, *, source: str | None = None) -> dict:
183
181
  journals = journal_paths(root)
184
182
  with (
185
183
  tempfile.TemporaryDirectory(prefix="gateway-apply-batch-") as temporary,
186
- SQLiteStore.create(Path(temporary) / "state.sqlite", contract) as store,
184
+ WorldDatabase.create(Path(temporary) / "world.db", contract) as store,
187
185
  ):
188
186
  for path in journals:
189
187
  store.apply_batch(read_json(path))
@@ -196,9 +194,8 @@ def append_batch(root: Path, batch: dict, *, source: str | None = None) -> dict:
196
194
  existing = len(journals) - 1
197
195
  target = batches / f"{existing + 1:06d}-{digest(batch['id'].encode())[:16]}.json"
198
196
  target.write_bytes(encoded(resolved))
199
- if snapshots.default_snapshot(root).is_file():
200
- # The journals grew; a snapshot built before this batch would be served
201
- # over them. Rebuild so the tree stays one thing (default + scenarios).
197
+ if snapshots.has_default_snapshot(root):
198
+ # A snapshot built before this batch would be served over the grown journals.
202
199
  from .compiler import build
203
200
  from .snapshot import build_all
204
201
 
@@ -206,38 +203,6 @@ def append_batch(root: Path, batch: dict, *, source: str | None = None) -> dict:
206
203
  return {**result, "journal": str(target.relative_to(root))}
207
204
 
208
205
 
209
- def _pinned_state(bundle: Path, contract) -> dict[str, int] | None:
210
- """The entity counts of a data-only version whose hosted snapshot is not in the
211
- tree: ``db/snapshot.json`` alone. The platform audited that snapshot through
212
- this contract when it built it, so the pin stands in for the state check —
213
- provided it names this contract; a snapshot built through another contract
214
- is refused, since the tree's data files cannot reproduce it either. A tree
215
- that carries its snapshot, or no pin at all, materializes as usual."""
216
- from .snapshot import contract_hash, current_default, pinned_default
217
-
218
- pin = pinned_default(bundle)
219
- if pin is None or current_default(bundle, contract) is not None:
220
- return None
221
- if pin.get("contractHash") != contract_hash(contract):
222
- fail(
223
- "snapshot_pinned",
224
- "db/snapshot.json pins a hosted snapshot built through a different contract; "
225
- "re-import its rows against the current schema (import_world_data, "
226
- "gateway worlds data import) or restore the schema it was built against",
227
- "/db/snapshot.json",
228
- )
229
- counts = pin.get("entityCounts")
230
- if not isinstance(counts, dict) or any(
231
- not isinstance(value, int) or isinstance(value, bool) for value in counts.values()
232
- ):
233
- fail(
234
- "snapshot_pin",
235
- "db/snapshot.json must carry the hosted snapshot's entityCounts",
236
- "/db/snapshot.json",
237
- )
238
- return {entity: int(counts.get(entity, 0)) for entity in contract.entities}
239
-
240
-
241
206
  def validate_bundle(bundle: str | Path, *, execute: bool = False) -> dict:
242
207
  """Check a schema world: the contract, the handlers by syntax and by name, the
243
208
  initial state, every verifier over it, the tasks and the links. ``execute``
@@ -256,14 +221,13 @@ def validate_bundle(bundle: str | Path, *, execute: bool = False) -> dict:
256
221
  load_api_settings(bundle, parse_spec((bundle / "connector.toml").read_text(encoding="utf-8")))
257
222
  _check_mappings(bundle, contract)
258
223
  check_world_redaction(bundle)
259
- pinned = _pinned_state(bundle, contract)
224
+ pinned = snapshots.pinned_counts(bundle, contract)
260
225
  verifiers: dict[str, Any] = {}
261
226
  if pinned is not None:
262
227
  counts, initial_state = pinned, "pinned"
263
- # The pinned snapshot is not in the tree, so the verifiers run over the
264
- # bare schema: an error still surfaces; a score means nothing here.
228
+ # The pinned snapshot is not in the tree: an error still surfaces, a score means nothing.
265
229
  with tempfile.TemporaryDirectory(prefix="gateway-state-check-") as staging:
266
- with SQLiteStore.create(Path(staging) / "world.db", contract) as store:
230
+ with WorldDatabase.create(Path(staging) / "world.db", contract) as store:
267
231
  unattached = _attach_dependencies(bundle, store, Path(staging))
268
232
  verifiers = _check_verifiers(bundle, store, unattached, scored=False)
269
233
  else:
@@ -271,7 +235,7 @@ def validate_bundle(bundle: str | Path, *, execute: bool = False) -> dict:
271
235
  with tempfile.TemporaryDirectory(prefix="gateway-state-check-") as staging:
272
236
  database = Path(staging) / "world.db"
273
237
  materialize_template(bundle, database)
274
- with SQLiteStore.open(database, contract) as store:
238
+ with WorldDatabase.open(database, contract) as store:
275
239
  store.audit()
276
240
  counts = {entity: len(store.list(entity)) for entity in contract.entities}
277
241
  unattached = _attach_dependencies(bundle, store, Path(staging))
@@ -281,8 +245,9 @@ def validate_bundle(bundle: str | Path, *, execute: bool = False) -> dict:
281
245
  "domain": contract.target,
282
246
  "package": contract.metadata["package"],
283
247
  "initial_state": initial_state,
284
- "handler_execution": _handler_execution(bundle, execute, declared),
248
+ "handler_execution": _handler_execution(bundle, execute, declared, hosted=pinned is not None),
285
249
  "entity_counts": counts,
250
+ "data_migration": snapshots.migration_view(bundle, contract),
286
251
  "tools": sorted(declared),
287
252
  "verifiers": verifiers,
288
253
  "tasks": _check_tasks(bundle, sorted(verifiers)),
@@ -295,7 +260,7 @@ def _check_mappings(bundle: Path, contract: Contract) -> None:
295
260
  """Each ``[dependencies.<alias>.entities]`` mapping against this contract
296
261
  and, for a dependency whose ``path`` is here, against that contract too."""
297
262
  from .. import _bundle
298
- from .dependencies import FieldMap, check_mappings, parse_dependencies
263
+ from .dependencies import check_mappings, parse_dependencies
299
264
 
300
265
  dependencies = parse_dependencies(_bundle.parse_manifest(bundle).raw)
301
266
  deps: dict[str, FieldMap] = {}
@@ -312,7 +277,7 @@ def _field_map(contract: Contract) -> FieldMap:
312
277
  return {name: set(entity["schema"].get("properties", {})) for name, entity in contract.entities.items()}
313
278
 
314
279
 
315
- def _attach_dependencies(bundle: Path, store: SQLiteStore, staging: Path) -> dict[str, str]:
280
+ def _attach_dependencies(bundle: Path, store: WorldDatabase, staging: Path) -> dict[str, str]:
316
281
  """Each ``[dependencies]`` entry whose local ``path`` is on this machine,
317
282
  materialized beside the root's store and attached as ``<alias>.<table>``,
318
283
  so a cross-world verifier runs here the way it runs in a session. Returns
@@ -344,7 +309,7 @@ def _attach_dependencies(bundle: Path, store: SQLiteStore, staging: Path) -> dic
344
309
  return unattached
345
310
 
346
311
 
347
- def _check_verifiers(bundle: Path, store: SQLiteStore, unattached: dict[str, str], *, scored: bool) -> dict[str, Any]:
312
+ def _check_verifiers(bundle: Path, store: WorldDatabase, unattached: dict[str, str], *, scored: bool) -> dict[str, Any]:
348
313
  """Every ``verifiers/*.sql`` run over the template through the store's
349
314
  read-only authorizer, the way a session grades. A verifier that names a
350
315
  table or column the contract lacks is refused here — at check — not at
@@ -363,11 +328,13 @@ def _check_verifiers(bundle: Path, store: SQLiteStore, unattached: dict[str, str
363
328
  return out
364
329
 
365
330
 
366
- def _check_verifier(path: Path, store: SQLiteStore, unattached: dict[str, str], *, scored: bool) -> dict[str, Any]:
331
+ def _check_verifier(path: Path, store: WorldDatabase, unattached: dict[str, str], *, scored: bool) -> dict[str, Any]:
367
332
  from .._world import SqlVerifier, VerifierError
368
333
 
369
334
  try:
370
- score, _sql = SqlVerifier.read(path).score(lambda statement: store.query(statement, private=True))
335
+ score, _sql = SqlVerifier.read(path, store.split_statements).score(
336
+ lambda statement: store.query(statement, private=True)
337
+ )
371
338
  except ContractError as error:
372
339
  alias = _refuse_verifier(path, store, unattached, error)
373
340
  return {
@@ -382,15 +349,16 @@ def _check_verifier(path: Path, store: SQLiteStore, unattached: dict[str, str],
382
349
  return {"status": "ok", "initial_score": score}
383
350
 
384
351
 
385
- def _refuse_verifier(path: Path, store: SQLiteStore, unattached: dict[str, str], error: ContractError) -> str:
352
+ def _refuse_verifier(path: Path, store: WorldDatabase, unattached: dict[str, str], error: ContractError) -> str:
386
353
  """The unattached dependency alias the failure names — the one refusal
387
354
  check reports rather than raises; every other failure is refused here."""
388
355
  reason = error.__cause__ if error.__cause__ is not None else error.detail["message"]
389
- alias = _missing_dependency_alias(str(reason), unattached)
356
+ missing = store.missing_table(reason)
357
+ alias = _missing_dependency_alias(missing, unattached)
390
358
  if alias is not None:
391
359
  return alias
392
360
  hint = ""
393
- if "no such table" in str(reason):
361
+ if missing is not None:
394
362
  tables = sorted(store.contract.entities)
395
363
  tables += [f"{a}.{t}" for a, ts in sorted(store.attached.items()) for t in sorted(ts)]
396
364
  hint = f"; the world's entities are {', '.join(tables) or '(none)'}"
@@ -401,14 +369,9 @@ def _refuse_verifier(path: Path, store: SQLiteStore, unattached: dict[str, str],
401
369
  )
402
370
 
403
371
 
404
- def _missing_dependency_alias(reason: str, unattached: dict[str, str]) -> str | None:
405
- """The alias of an unattached dependency a ``no such table: alias.table``
406
- failure names, else ``None``."""
407
- marker = "no such table: "
408
- if marker not in reason:
409
- return None
410
- table = reason.split(marker, 1)[1].strip()
411
- alias, dot, _ = table.partition(".")
372
+ def _missing_dependency_alias(table: str | None, unattached: dict[str, str]) -> str | None:
373
+ """The alias of an unattached dependency a missing ``alias.table`` names, else ``None``."""
374
+ alias, dot, _ = (table or "").partition(".")
412
375
  if dot and alias in unattached:
413
376
  return alias
414
377
  return None
@@ -421,7 +384,7 @@ def _check_tasks(bundle: Path, verifiers: list[str]) -> list[dict[str, Any]]:
421
384
  from .tools_world import load_tasks
422
385
 
423
386
  manifest = _bundle.parse_manifest(bundle)
424
- tasks = load_tasks(bundle, manifest, verifiers, required=False)
387
+ tasks = load_tasks(bundle, manifest, verifiers)
425
388
  out = []
426
389
  for task in tasks.values():
427
390
  unknown = sorted(set(task.selected(verifiers)) - set(verifiers))
@@ -476,9 +439,7 @@ def describe_dependencies(bundle: Path) -> list[dict[str, Any]]:
476
439
  return out
477
440
 
478
441
 
479
- #: Set by a runtime that may run the world's own Python — the CLI's bundled
480
- #: runtime on the author's machine. The platform's bridge never sets it, so a
481
- #: request that asks for execution there gets ``not_run`` and the reason.
442
+ # Only the CLI's bundled runtime sets this; the platform's bridge never runs the world's own Python.
482
443
  EXECUTE_ENV = "GATEWAY_RUNTIME_EXECUTE"
483
444
 
484
445
 
@@ -486,14 +447,15 @@ def may_execute() -> bool:
486
447
  return os.environ.get(EXECUTE_ENV) == "1"
487
448
 
488
449
 
489
- def _handler_execution(bundle: Path, execute: bool, declared: set[str]) -> dict[str, Any]:
450
+ def _handler_execution(bundle: Path, execute: bool, declared: set[str], *, hosted: bool) -> dict[str, Any]:
490
451
  """Whether the handlers were run: imported, every declared tool bound, the
491
452
  ``[dependencies]`` opened from their paths — one session of the world,
492
- opened and closed. Only where authored code may run. ``tools`` is the
493
- declared handlers the session bound (the report's top-level ``tools``);
494
- ``served`` every name ``worlds call`` answers there — handlers, the connector
495
- routes as operations, a dependency's ``alias.tool`` — so the two lists differ
496
- by design and say what each counts."""
453
+ opened and closed. Only where authored code may run, and only over rows this
454
+ tree holds (``hosted``: its data is a snapshot the platform keeps). ``tools`` is
455
+ the declared handlers the session bound (the report's top-level ``tools``);
456
+ ``served`` every name ``worlds call`` answers there — the handlers and a
457
+ dependency's ``alias.tool``; a connector route answers over HTTP, never by
458
+ name — so the two lists differ by design and say what each counts."""
497
459
  if not execute:
498
460
  return {
499
461
  "status": "not_run",
@@ -506,6 +468,12 @@ def _handler_execution(bundle: Path, execute: bool, declared: set[str]) -> dict[
506
468
  "reason": "this runtime does not run a world's Python (the platform's "
507
469
  "bridge); check with the CLI's bundled runtime on your machine",
508
470
  }
471
+ if hosted:
472
+ return {
473
+ "status": "not_run",
474
+ "reason": "this world's rows are the hosted snapshot db/snapshot.json pins, which "
475
+ "is not in this tree; `gateway worlds session open <slug>` runs the handlers over it",
476
+ }
509
477
  from .local import open_world
510
478
 
511
479
  with open_world(bundle, http=False) as session:
@@ -521,11 +489,10 @@ def _handler_execution(bundle: Path, execute: bool, declared: set[str]) -> dict[
521
489
 
522
490
 
523
491
  def world_root(root: Path) -> Path:
524
- """The world inside the tree the platform holds (``WorldTree.world``).
525
- ``bench push`` uploads a schema world as its api bundle, so a version's
526
- tree is ``gateway-api-world.toml`` + ``seed/`` + ``scenarios/`` +
527
- ``server/world/<the authoring world>``: that nested world is what data
528
- commands operate on. A plain tree is its own root."""
492
+ """The world inside the tree the platform holds (``WorldTree.world``). A
493
+ version stored as an api bundle is ``gateway-api-world.toml`` + ``seed/`` +
494
+ ``scenarios/`` + ``server/world/<the authoring world>``: that nested world
495
+ is what data commands operate on. A plain tree is its own root."""
529
496
  return WorldTree.from_dir(root).world
530
497
 
531
498
 
@@ -627,8 +594,7 @@ def _capture(request: dict) -> dict:
627
594
  fail("input", "Expected the connector.toml text and an operation name")
628
595
  if not isinstance(args, dict) or not all(isinstance(k, str) and isinstance(v, str) for k, v in args.items()):
629
596
  fail("input", "args map names to strings")
630
- # params arrive as [[name, value], ...] so a repeated key survives the wire; the
631
- # {name: value} form older callers send is accepted too.
597
+ # Pairs so a repeated key survives the wire; older callers still send a mapping.
632
598
  try:
633
599
  params = pairs_of(request.get("params"))
634
600
  except ContractError:
@@ -692,7 +658,6 @@ def _template_summary(files: dict[str, bytes]) -> dict:
692
658
  from .redaction import parse_env_policy
693
659
  from .scaffold import template_digest
694
660
 
695
- # A template keeps world.json at its root; an initialized world keeps it under schema/.
696
661
  world_path = next((path for path in ("world.json", "schema/world.json") if path in files), None)
697
662
  if world_path is None:
698
663
  fail(
@@ -719,7 +684,7 @@ def _template_summary(files: dict[str, bytes]) -> dict:
719
684
  "primaryKey": list(entity.get("primary_key") or []) if isinstance(entity, dict) else [],
720
685
  "unique": list(entity.get("unique") or []) if isinstance(entity, dict) else [],
721
686
  "relationships": list(entity.get("relationships") or []) if isinstance(entity, dict) else [],
722
- "fields": [asdict(field) for field in fields_by_entity.get(name, [])],
687
+ "fields": [field.describe() for field in fields_by_entity.get(name, [])],
723
688
  }
724
689
  )
725
690
  _annotate_redaction(entities, policy)
@@ -760,8 +725,6 @@ def _template_summary(files: dict[str, bytes]) -> dict:
760
725
 
761
726
 
762
727
  COMPILED_MARKERS = ("schema/world.json", "tool-schemas.json")
763
- # The compiled world's files a description is read from; the rest of a bundle
764
- # (seed data, verifiers, tool sources) says nothing about the contract.
765
728
  DESCRIBE_PATHS = (
766
729
  "schema/world.json",
767
730
  "tool-schemas.json",
@@ -782,13 +745,13 @@ def _describe(request: dict) -> dict:
782
745
  straight through. Never runs the connector's Python."""
783
746
  from .connector import load_spec, parse_spec
784
747
  from .redaction import env_policy
785
- from .ui import ui_kind
786
748
  from .scaffold import (
787
749
  TEMPLATES,
788
750
  init_bundle,
789
751
  init_bundle_from_files,
790
752
  template_digest,
791
753
  )
754
+ from .ui import ui_kind
792
755
 
793
756
  connector = request.get("connector")
794
757
  with tempfile.TemporaryDirectory(prefix="gateway-schema-") as directory:
@@ -842,8 +805,7 @@ def _describe(request: dict) -> dict:
842
805
  try:
843
806
  settings = load_api_settings(root, spec)
844
807
  except ContractError:
845
- # A template without its handlers still has a shape worth showing;
846
- # the routes render with the default action and status.
808
+ # A template without its handlers still has a shape worth showing.
847
809
  settings = None
848
810
  _annotate_redaction(summary["entities"], spec)
849
811
  summary["connection"] = {
@@ -915,7 +877,7 @@ def _ingest_calls(request: dict) -> dict:
915
877
  )
916
878
  answer: dict = {"status": "ok", **report}
917
879
  if report["changed"] and report.get("kind") == KIND_SCHEMA:
918
- if snapshots.default_snapshot(world).is_file():
880
+ if snapshots.has_default_snapshot(world):
919
881
  from .compiler import build
920
882
  from .snapshot import build_all
921
883
 
@@ -929,9 +891,6 @@ def _ingest_calls(request: dict) -> dict:
929
891
  return answer
930
892
 
931
893
 
932
- # ---------------------------------------------------------------- batches
933
-
934
-
935
894
  def _workdir(request: dict) -> Path:
936
895
  """The absolute directory the caller prepared; every path in the request is under it."""
937
896
  workdir = request.get("workdir")
@@ -956,7 +915,7 @@ def _chunk_specs(workdir: Path, request: dict) -> list:
956
915
  from .batch import ChunkSpec
957
916
 
958
917
  listed = request.get("chunks")
959
- if not isinstance(listed, list) or not listed:
918
+ if not isinstance(listed, list):
960
919
  fail("input", "Expected chunks as [{ordinal, path, entity?}]")
961
920
  specs = []
962
921
  seen = set()
@@ -1039,14 +998,16 @@ def _redactor(root: Path, request: dict):
1039
998
 
1040
999
 
1041
1000
  def _snapshot_build(request: dict) -> dict:
1042
- """``snapshot-build``: the parent snapshot (or the tree's default) with the chunks
1043
- applied in dependency order, one transaction (atomic) or one per chunk; the new
1044
- snapshot saved under workdir with its sidecar, or nothing on dryRun or an atomic
1045
- refusal."""
1001
+ """``snapshot-build``: the parent snapshot (or the tree's default), carried to the
1002
+ tree's contract when that only adds to the domain its rows were stored under, with
1003
+ the chunks applied in dependency order, one transaction (atomic) or one per chunk;
1004
+ the new snapshot saved under workdir with its sidecar (its absolute path answered),
1005
+ or nothing on dryRun or an atomic refusal."""
1046
1006
  from .batch import build_batch
1047
1007
  from .compiler import check_bundle
1048
1008
  from .data import MERGES, MODES
1049
- from .snapshot import EMPTY_DIGEST, build_default, default_snapshot
1009
+ from .domain_migration import DomainMigration
1010
+ from .snapshot import EMPTY_DIGEST, SIDECAR, build_default, default_snapshot, snapshot_name
1050
1011
 
1051
1012
  workdir = _workdir(request)
1052
1013
  mode = request.get("mode", "append")
@@ -1067,21 +1028,21 @@ def _snapshot_build(request: dict) -> dict:
1067
1028
  contract = check_bundle(root)
1068
1029
  redactor = _redactor(root, request)
1069
1030
  chunks = _chunk_specs(workdir, request)
1070
- # One directory per batch, so a parent built by an earlier call in the same
1071
- # workdir is never the file being written.
1031
+ # One directory per batch, so a parent built by an earlier call is never the file being written.
1072
1032
  out_dir = workdir / "snapshot" / batch_id
1073
1033
  out_dir.mkdir(parents=True, exist_ok=True)
1074
- out = out_dir / "state.sqlite"
1034
+ out = out_dir / snapshot_name(contract)
1075
1035
  out.unlink(missing_ok=True)
1076
- out.with_name("state.json").unlink(missing_ok=True)
1036
+ out.with_name(SIDECAR).unlink(missing_ok=True)
1077
1037
  if request.get("parentSnapshot") is not None:
1078
1038
  parent = _under(workdir, request["parentSnapshot"], "parentSnapshot")
1079
1039
  if not parent.is_file():
1080
1040
  fail("input", "parentSnapshot is missing under workdir")
1081
- shutil.copyfile(parent, out)
1041
+ WorldDatabase.clone_file(parent, out)
1042
+ DomainMigration.carry(out, contract, root)
1082
1043
  else:
1083
1044
  build_default(root, contract)
1084
- shutil.copyfile(default_snapshot(root), out)
1045
+ WorldDatabase.clone_file(default_snapshot(root, contract), out)
1085
1046
  result = build_batch(
1086
1047
  contract,
1087
1048
  out,
@@ -1095,7 +1056,6 @@ def _snapshot_build(request: dict) -> dict:
1095
1056
  redactor=redactor,
1096
1057
  )
1097
1058
  if "snapshot" in result:
1098
- # Absolute, under workdir: the caller opens it as it is.
1099
1059
  result["snapshot"] = str(result["snapshot"])
1100
1060
  return {"status": "ok", **result}
1101
1061
 
@@ -1104,20 +1064,20 @@ def _snapshot_default(request: dict) -> dict:
1104
1064
  """``snapshot-default``: the tree's default snapshot and every scenario snapshot,
1105
1065
  built under ``workdir/tree``; the sidecars come back with absolute paths."""
1106
1066
  from .compiler import check_bundle
1107
- from .snapshot import build_all, default_snapshot, scenario_snapshot
1067
+ from .snapshot import build_all, build_default, default_snapshot, scenario_snapshot
1108
1068
 
1109
1069
  workdir = _workdir(request)
1110
1070
  root = _write_tree(workdir, request)
1111
- built = build_all(root, check_bundle(root))
1112
- # Paths are absolute, under workdir; the sidecar sits both at the top level and
1113
- # keyed by its tree path, so a reader may take either.
1071
+ contract = check_bundle(root)
1072
+ build_default(root, contract)
1073
+ built = build_all(root, contract)
1114
1074
  return {
1115
1075
  "status": "ok",
1116
- "snapshot": str(default_snapshot(root)),
1076
+ "snapshot": str(default_snapshot(root, contract)),
1117
1077
  "sidecar": built["default"],
1118
1078
  "sidecars": {"db/state.json": built["default"]},
1119
1079
  "scenarios": {
1120
- name: {"snapshot": str(scenario_snapshot(root, name)), "sidecar": sidecar}
1080
+ name: {"snapshot": str(scenario_snapshot(root, name, contract)), "sidecar": sidecar}
1121
1081
  for name, sidecar in built["scenarios"].items()
1122
1082
  },
1123
1083
  }
@@ -1158,7 +1118,6 @@ def _ingest(request: dict) -> dict:
1158
1118
  root.mkdir()
1159
1119
  _write_files(root, request.get("files"))
1160
1120
  if dry_run:
1161
- # Every violation of every capture, grouped; no files come back since none changed.
1162
1121
  return {"status": "ok", "result": check(world_root(root))}
1163
1122
  result = ingest(world_root(root), batch_id=batch_id)
1164
1123
  return {"status": "ok", "result": result, "files": _tree(root)}
@@ -1327,7 +1286,7 @@ def _dry_run(request: dict) -> dict:
1327
1286
  task card (prompt, tools, verifiers), the state's rows per entity, and
1328
1287
  ``grade`` — or, when a verifier cannot run (bad SQL, a missing entity),
1329
1288
  ``grade: null`` and ``verifier_errors`` naming each verifier that failed
1330
- with SQLite's own words. That is the point of the dry run: a benchmark
1289
+ with the database engine's own words. That is the point of the dry run: a benchmark
1331
1290
  whose grader dies is found here, not on the runner. Only where this
1332
1291
  runtime may run a world's Python (:func:`may_execute`)."""
1333
1292
  from ..errors import VerifierError
@@ -1368,7 +1327,7 @@ def _dry_run(request: dict) -> dict:
1368
1327
  return answer
1369
1328
  except (ContractError, VerifierError):
1370
1329
  pass
1371
- # Name the verifier: the grade runs them together, so run each alone.
1330
+ # The grade runs the verifiers together, so run each alone to name the failing one.
1372
1331
  errors: dict[str, str] = {}
1373
1332
  for name in card["verifiers"]:
1374
1333
  try:
@@ -1413,66 +1372,54 @@ def _templates(request: dict) -> dict:
1413
1372
  return {"status": "ok", "templates": templates}
1414
1373
 
1415
1374
 
1375
+ def _answering_commands() -> dict:
1376
+ """The commands that answer on their own, without the shared write-validate-return tree below."""
1377
+ return {
1378
+ "validate-rows": _validate_rows,
1379
+ "snapshot-build": _snapshot_build,
1380
+ "snapshot-default": _snapshot_default,
1381
+ "capture": _capture,
1382
+ "import-data": platform_import.import_data,
1383
+ "import-files": lambda request: platform_import.import_files(request, _world_for),
1384
+ "chunk-files": lambda request: platform_import.chunk_files(request, _world_for),
1385
+ "ingest": _ingest,
1386
+ "ingest-calls": _ingest_calls,
1387
+ "conform": _conform,
1388
+ "reconcile": _reconcile,
1389
+ "run-tests": _run_tests,
1390
+ "call": _call,
1391
+ "dry-run": _dry_run,
1392
+ "counts": lambda request: platform_query.counts(request, _world_for),
1393
+ "query": lambda request: platform_query.query(request, _world_for),
1394
+ "sql": lambda request: platform_query.sql(request, _world_for),
1395
+ "verify-legacy": lambda request: platform_query.verify_legacy(request, _world_for),
1396
+ "templates": _templates,
1397
+ "api-bundle": _api_bundle,
1398
+ "template-summary": lambda request: _template_summary(request_files(request)),
1399
+ "describe": _describe,
1400
+ }
1401
+
1402
+
1416
1403
  def execute(request: dict) -> dict:
1417
1404
  if not isinstance(request, dict) or request.get("command") not in COMMANDS:
1418
1405
  fail(
1419
1406
  "command",
1420
1407
  "Expected one of " + ", ".join(COMMANDS[:-1]) + " or " + COMMANDS[-1],
1421
1408
  )
1422
- if request["command"] == "validate-rows":
1423
- return _validate_rows(request)
1424
- if request["command"] == "snapshot-build":
1425
- return _snapshot_build(request)
1426
- if request["command"] == "snapshot-default":
1427
- return _snapshot_default(request)
1428
- if request["command"] == "capture":
1429
- return _capture(request)
1430
- if request["command"] == "import-data":
1431
- return platform_import.import_data(request)
1432
- if request["command"] == "import-files":
1433
- return platform_import.import_files(request, _world_for)
1434
- if request["command"] == "chunk-files":
1435
- return platform_import.chunk_files(request, _world_for)
1436
- if request["command"] == "ingest":
1437
- return _ingest(request)
1438
- if request["command"] == "ingest-calls":
1439
- return _ingest_calls(request)
1440
- if request["command"] == "conform":
1441
- return _conform(request)
1442
- if request["command"] == "reconcile":
1443
- return _reconcile(request)
1444
- if request["command"] == "run-tests":
1445
- return _run_tests(request)
1446
- if request["command"] == "call":
1447
- return _call(request)
1448
- if request["command"] == "dry-run":
1449
- return _dry_run(request)
1450
- if request["command"] == "counts":
1451
- return platform_query.counts(request, _world_for)
1452
- if request["command"] == "query":
1453
- return platform_query.query(request, _world_for)
1454
- if request["command"] == "templates":
1455
- return _templates(request)
1456
- if request["command"] == "api-bundle":
1457
- return _api_bundle(request)
1458
- if request["command"] == "template-summary":
1459
- return _template_summary(request_files(request))
1460
- if request["command"] == "describe":
1461
- return _describe(request)
1409
+ handler = _answering_commands().get(request["command"])
1410
+ if handler is not None:
1411
+ return handler(request)
1462
1412
  extra: dict = {}
1463
1413
  with tempfile.TemporaryDirectory(prefix="gateway-schema-") as directory:
1464
1414
  root = Path(directory) / "bundle"
1465
1415
  root.mkdir()
1466
1416
  if request["command"] == "validate-archive":
1467
1417
  _write_files(root, _archive_files(request.get("archive")))
1468
- # A stored version may be the api bundle of a schema world: the
1469
- # world lives under server/world, and that is what gets validated.
1470
1418
  root = world_root(root)
1471
1419
  if bundle_mode(root) is None:
1472
1420
  return {"status": "ok", "mode": "legacy"}
1473
1421
  if request["command"] == "validate" and request.get("world") is not None:
1474
- # The CLI's bundled runtime checks a directory in place, so the
1475
- # ``[dependencies]`` paths beside it resolve and its handlers may run.
1422
+ # Checked in place, so the ``[dependencies]`` paths beside it resolve.
1476
1423
  root = _world_for(request, Path(directory))
1477
1424
  elif request["command"] in (
1478
1425
  "validate",
@@ -1481,13 +1428,10 @@ def execute(request: dict) -> dict:
1481
1428
  "refresh-slack",
1482
1429
  ):
1483
1430
  _write_files(root, request.get("files"))
1484
- # The tree the platform sent may be a schema world's api bundle; the
1485
- # world itself sits under server/world and is what compiles, takes a
1486
- # batch and validates. The whole tree still travels back.
1431
+ # A schema world's api bundle keeps the world under server/world; the whole tree still travels back.
1487
1432
  world = world_root(root)
1488
1433
  if request["command"] == "compile":
1489
- # Arbitrary custom worlds: the agent authors schema/world.json (+ manifest,
1490
- # handlers, journals); the trusted compiler owns every generated artifact.
1434
+ # The agent authors schema/world.json; the trusted compiler owns every generated artifact.
1491
1435
  extra["compile"] = compile_world(world)
1492
1436
  if world != root and (world / "data/initial.json").is_file():
1493
1437
  from .api import write_base_seed
@@ -1502,8 +1446,6 @@ def execute(request: dict) -> dict:
1502
1446
  )
1503
1447
  extra["batch"] = append_batch(world, batch, source=source)
1504
1448
  if request["command"] == "init-template":
1505
- # The platform's "create world" for every connector template: the same
1506
- # init + compile the CLI runs, on a fresh empty tree, with nothing seeded.
1507
1449
  from .scaffold import init_bundle
1508
1450
 
1509
1451
  connector = request.get("connector")
@@ -1517,8 +1459,7 @@ def execute(request: dict) -> dict:
1517
1459
  with_example_tool=request.get("with_example_tool") is True,
1518
1460
  )
1519
1461
  if request["command"] == "init-template-files":
1520
- # A workspace connector: the template arrives as files, never as a path
1521
- # the request chose, and the world is built exactly as from an installed one.
1462
+ # The template arrives as files, never as a path the request chose.
1522
1463
  from .scaffold import init_bundle_from_files
1523
1464
 
1524
1465
  connector = request.get("connector")
@@ -1564,11 +1505,16 @@ def main():
1564
1505
  raw = sys.stdin.buffer.read(2 * MAX_BYTES + 1)
1565
1506
  if len(raw) > 2 * MAX_BYTES:
1566
1507
  fail("size", "Bridge request is too large")
1567
- result = execute(json.loads(raw))
1508
+ request = json.loads(raw)
1509
+ platform_query.cap_memory_for(request)
1510
+ result = execute(request)
1568
1511
  print(json.dumps(result, sort_keys=True))
1569
1512
  except ContractError as error:
1570
1513
  print(json.dumps({"status": "rejected", "error": error.detail}, sort_keys=True))
1571
1514
  return 1
1515
+ except MemoryError:
1516
+ print(json.dumps({"status": "rejected", "error": {"code": "size", "message": "Out of memory"}}))
1517
+ return 1
1572
1518
  except (OSError, ValueError, TypeError):
1573
1519
  # Never dump request content or provider records in process diagnostics.
1574
1520
  print(