@withgateway/sdk 0.3.1 → 0.3.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (129) hide show
  1. package/README.md +4 -3
  2. package/dist/cli.js +2880 -2032
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +2982 -2135
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +180 -41
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +172 -35
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
  13. package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
  14. package/dist/index.d.mts +4 -4
  15. package/dist/index.d.ts +4 -4
  16. package/dist/index.js +1309 -540
  17. package/dist/index.js.map +1 -1
  18. package/dist/index.mjs +1287 -532
  19. package/dist/index.mjs.map +1 -1
  20. package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
  21. package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
  22. package/dist/traces/index.d.mts +1 -1
  23. package/dist/traces/index.d.ts +1 -1
  24. package/dist/traces/index.js +177 -38
  25. package/dist/traces/index.js.map +1 -1
  26. package/dist/traces/index.mjs +169 -32
  27. package/dist/traces/index.mjs.map +1 -1
  28. package/dist/tracing/index.d.mts +2 -2
  29. package/dist/tracing/index.d.ts +2 -2
  30. package/dist/tracing/index.js +69 -13
  31. package/dist/tracing/index.js.map +1 -1
  32. package/dist/tracing/index.mjs +69 -13
  33. package/dist/tracing/index.mjs.map +1 -1
  34. package/dist/worlds/index.d.mts +249 -120
  35. package/dist/worlds/index.d.ts +249 -120
  36. package/dist/worlds/index.js +1240 -484
  37. package/dist/worlds/index.js.map +1 -1
  38. package/dist/worlds/index.mjs +1231 -476
  39. package/dist/worlds/index.mjs.map +1 -1
  40. package/package.json +1 -1
  41. package/runtime/gatewaysdk/environment/_bundle.py +2 -27
  42. package/runtime/gatewaysdk/environment/_materialize.py +49 -69
  43. package/runtime/gatewaysdk/environment/_world.py +122 -197
  44. package/runtime/gatewaysdk/environment/backing.py +137 -0
  45. package/runtime/gatewaysdk/environment/core.py +1 -1
  46. package/runtime/gatewaysdk/environment/legacy_sqlite.py +301 -0
  47. package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
  48. package/runtime/gatewaysdk/environment/runtime.py +8 -13
  49. package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
  50. package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
  51. package/runtime/gatewaysdk/environment/schema/api.py +266 -514
  52. package/runtime/gatewaysdk/environment/schema/batch.py +42 -37
  53. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
  54. package/runtime/gatewaysdk/environment/schema/call_log.py +291 -0
  55. package/runtime/gatewaysdk/environment/schema/call_redaction.py +344 -0
  56. package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
  57. package/runtime/gatewaysdk/environment/schema/conform.py +8 -12
  58. package/runtime/gatewaysdk/environment/schema/connector.py +27 -62
  59. package/runtime/gatewaysdk/environment/schema/data.py +50 -207
  60. package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
  61. package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
  62. package/runtime/gatewaysdk/environment/schema/database/base.py +995 -0
  63. package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
  64. package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
  65. package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
  66. package/runtime/gatewaysdk/environment/schema/database/guard.py +208 -0
  67. package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
  68. package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
  69. package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
  70. package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
  71. package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
  72. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +367 -0
  73. package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
  74. package/runtime/gatewaysdk/environment/schema/envelope.py +169 -0
  75. package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
  76. package/runtime/gatewaysdk/environment/schema/grading.py +45 -30
  77. package/runtime/gatewaysdk/environment/schema/handler_errors.py +142 -12
  78. package/runtime/gatewaysdk/environment/schema/host.py +158 -250
  79. package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
  80. package/runtime/gatewaysdk/environment/schema/local.py +44 -71
  81. package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
  82. package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
  83. package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
  84. package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
  85. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
  86. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
  87. package/runtime/gatewaysdk/environment/schema/platform.py +123 -171
  88. package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
  89. package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
  90. package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
  91. package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
  92. package/runtime/gatewaysdk/environment/schema/runtime.py +77 -160
  93. package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
  94. package/runtime/gatewaysdk/environment/schema/session_files.py +166 -0
  95. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +2 -2
  96. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +19 -11
  97. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +29 -3
  98. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +14 -14
  99. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +12 -12
  100. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
  101. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +15 -0
  102. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
  103. package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
  104. package/runtime/gatewaysdk/environment/schema/snapshot.py +385 -129
  105. package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
  106. package/runtime/gatewaysdk/environment/schema/tasks.py +140 -55
  107. package/runtime/gatewaysdk/environment/schema/tools_world.py +110 -87
  108. package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
  109. package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
  110. package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
  111. package/runtime/gatewaysdk/environment/schema/validation.py +166 -40
  112. package/runtime/gatewaysdk/environment/schema/workers.py +142 -57
  113. package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
  114. package/runtime/gatewaysdk/environment/schema/world_tests.py +71 -39
  115. package/runtime/gatewaysdk/environment/score.py +57 -0
  116. package/runtime/gatewaysdk/environment/secret_values.py +36 -0
  117. package/runtime/gatewaysdk/environment/world.py +42 -3
  118. package/runtime/gatewaysdk/environment/world_session_contract.py +32 -5
  119. package/skills/benchmarks-getting-started/SKILL.md +2 -2
  120. package/skills/connector-schema-authoring/SKILL.md +19 -11
  121. package/skills/connector-schema-authoring/connector-worlds.md +13 -4
  122. package/skills/connector-schema-authoring/world-schema.md +103 -51
  123. package/skills/world-data-ingestion/ROWS.md +29 -3
  124. package/skills/world-data-ingestion/SKILL.md +14 -14
  125. package/skills/worlds-getting-started/SKILL.md +12 -12
  126. package/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
  127. package/skills/worlds-getting-started/references/gotchas.md +15 -0
  128. package/skills/worlds-getting-started/references/openapi-import.md +50 -21
  129. package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
@@ -1,22 +1,31 @@
1
- """World data snapshots: the validated SQLite, built once and opened at start.
1
+ """World data snapshots: the validated world database file, built once and opened at start.
2
2
 
3
- A snapshot is a ``state.sqlite`` the store wrote (contract lock and batch ledger
4
- inside it, as always) plus a ``state.json`` sidecar naming what it holds::
3
+ A snapshot is a file the store wrote, named by its engine (``state.sqlite`` for
4
+ SQLite; contract lock and batch ledger inside it, as always) plus a ``state.json``
5
+ sidecar naming what it holds::
5
6
 
6
7
  {"contractHash": ..., "batchDigest": ..., "entityCounts": {...},
7
8
  "builtAt": ..., "runtime": "<gatewaysdk version>", "fileSha256": ...}
8
9
 
9
- Keyed by ``(contractHash, batchDigest)``: the same key is the same file, never
10
- rebuilt. ``db/state.sqlite`` is the default snapshot (``data/initial.json``
11
- applied, plus any ``data/batches/*.json``); ``scenarios/<name>/state.sqlite`` is
12
- the default with that scenario's ``seed/*.json`` overlays replacing whole
13
- entities. ``compile`` builds them, ``check`` verifies them, the served world
14
- copies one to its temp dir and opens it instead of replaying the JSON.
10
+ ``batchDigest`` says which data it holds. A snapshot is valid for every contract with
11
+ the :class:`~.compiler.StorageContract` its rows were stored under, which the file
12
+ records and the store checks on every open: a tool, handler, route, test, package
13
+ version or documentation-file edit keeps it. A domain edit that only adds to the domain
14
+ (:class:`~.domain_migration.DomainMigration`: a new entity, a new field no row has to
15
+ carry, another ``domain_version``) carries it: compile migrates the snapshots the tree
16
+ holds, and keeps a pin's proof of the stored domain so the platform migrates the hosted
17
+ rows when it commits the change. Any other domain edit refuses it, naming the change.
18
+ ``contractHash`` is the lock it was last built or verified under, a record rather than
19
+ the rule. ``db/<snapshot name>`` is the default snapshot (``data/initial.json`` applied,
20
+ plus any ``data/batches/*.json``); ``scenarios/<name>/<snapshot name>`` is the default
21
+ with that scenario's ``seed/*.json`` overlays replacing whole entities. ``compile``
22
+ builds them (one that already holds the same data under the same storage contract is
23
+ kept), ``check`` verifies them, the served world copies one to its temp dir and opens it
24
+ instead of replaying the JSON.
15
25
 
16
26
  Digests (the contract, section 0):
17
27
 
18
- * contract hash: sha256 of the canonical contract lock, the bytes the store's
19
- ``_check_contract`` compares;
28
+ * contract hash: sha256 of the canonical contract lock;
20
29
  * batch digest: ``sha256(parent + "\\n" + batchId + "\\n" + "\\n".join(sorted(chunk digests)))``,
21
30
  the empty world's parent being sixty-four zeros;
22
31
  * the default batch's chunks are the tree's data files (``data/initial.json``
@@ -24,11 +33,13 @@ Digests (the contract, section 0):
24
33
  is ``scenario:<name>`` over the default with its seed files as chunks.
25
34
 
26
35
  A tree the platform published as a data-only version carries ``db/snapshot.json``,
27
- the **pin**: the hosted snapshot's ``batchDigest`` (and sidecar fields), which
28
- replaces the data files as the default batch. The tree's ``data/*.json`` then no
29
- longer describes the world's data — only that snapshot does — so a pinned tree is
30
- never rebuilt from its data files; the platform hands the file down
31
- (``load_world_to_sandbox``, the pod's ``WORLD_SNAPSHOT_*``).
36
+ the **pin** (:class:`SnapshotPin`): the hosted snapshot's ``batchDigest`` (and sidecar
37
+ fields), which replaces the data files as the default batch. The tree's ``data/*.json``
38
+ then no longer describes the world's data — only that snapshot does — so a pinned tree
39
+ is never rebuilt from, nor served from, its data files; the platform hands the file
40
+ down (``load_world_to_sandbox``, the pod's ``WORLD_SNAPSHOT_*``). Where the file is not
41
+ handed down (a sandbox), the pin itself proves which domain the hosted rows were stored
42
+ under, so the contract still compiles and checks.
32
43
 
33
44
  Everything here is standard library, so the api bundle's server can load it.
34
45
  """
@@ -40,16 +51,17 @@ import hashlib
40
51
  import json
41
52
  import shutil
42
53
  import urllib.request
54
+ from dataclasses import dataclass
43
55
  from datetime import datetime, timezone
44
56
  from pathlib import Path
45
57
  from typing import Any, Iterable
46
58
 
47
59
  from .._toml import tomllib
48
- from .compiler import Contract
49
- from .store import SQLiteStore
60
+ from .compiler import LOCK, Contract, StorageContract, check_bundle
61
+ from .database import WorldDatabase
62
+ from .domain_migration import DomainMigration
50
63
  from .validation import ContractError, canonical, fail, read_json
51
64
 
52
- SNAPSHOT = "state.sqlite"
53
65
  SIDECAR = "state.json"
54
66
  PIN = "snapshot.json"
55
67
  DEFAULT_DIR = "db"
@@ -60,14 +72,19 @@ SIDECAR_KEYS = ("contractHash", "batchDigest", "entityCounts", "builtAt", "runti
60
72
  ENV_URL = "WORLD_SNAPSHOT_URL"
61
73
  ENV_SHA256 = "WORLD_SNAPSHOT_SHA256"
62
74
  ENV_DIGEST = "WORLD_SNAPSHOT_DIGEST"
75
+ PIN_POINTER = f"/{DEFAULT_DIR}/{PIN}"
76
+ #: The codes a pin is refused with: never a data refusal, so never discarded as one.
77
+ PIN_REFUSALS = ("snapshot_pin", "snapshot_pinned")
78
+ REIMPORT = f"remove {DEFAULT_DIR}/{PIN} and import every row again (`gateway worlds data import <slug> <rows>`)"
63
79
  _BLOCK = 1024 * 1024
64
80
 
65
81
 
66
- # ---------------------------------------------------------------- digests
82
+ def lock_hash(lock: Any) -> str:
83
+ return hashlib.sha256(canonical(lock).encode("utf-8")).hexdigest()
67
84
 
68
85
 
69
86
  def contract_hash(contract: Contract) -> str:
70
- return hashlib.sha256(canonical(contract.lock).encode("utf-8")).hexdigest()
87
+ return lock_hash(contract.lock)
71
88
 
72
89
 
73
90
  def batch_digest(parent: str, batch_id: str, chunk_digests: Iterable[str]) -> str:
@@ -113,25 +130,159 @@ def pin_path(world: str | Path) -> Path:
113
130
  return Path(world) / DEFAULT_DIR / PIN
114
131
 
115
132
 
116
- def pinned_default(world: str | Path) -> dict | None:
117
- """The platform's pin when the tree is a data-only version: ``db/snapshot.json``
118
- naming the hosted snapshot (``batchDigest`` at least); None for an authored tree."""
119
- path = pin_path(world)
120
- if not path.is_file():
121
- return None
122
- pin = read_json(path)
123
- if not isinstance(pin, dict) or not isinstance(pin.get("batchDigest"), str) or len(pin["batchDigest"]) != 64:
133
+ def refuse_unheld_pin(world: str | Path, contract: Contract) -> None:
134
+ """A pinned tree that does not hold its hosted snapshot has no rows here: nothing
135
+ is built, copied or served from its data files in their place."""
136
+ pin = SnapshotPin.of(world)
137
+ if pin is not None:
138
+ fail(
139
+ "snapshot_pinned",
140
+ f"{DEFAULT_DIR}/{PIN} pins the hosted snapshot {pin.batch_digest}, which this tree does not hold: "
141
+ f"run the world on the platform (`gateway worlds session open <slug>`, load_world_to_sandbox), "
142
+ f"which serves that snapshot, or remove {DEFAULT_DIR}/{PIN} to rebuild from the data files",
143
+ f"/{DEFAULT_DIR}/{snapshot_name(contract)}",
144
+ )
145
+
146
+
147
+ def refuse_pinned_rows(world: str | Path) -> None:
148
+ """Rows written into a pinned tree's data files would never be served: the hosted
149
+ snapshot, not the data files, is the world's data."""
150
+ if SnapshotPin.of(world) is not None:
124
151
  fail(
125
- "snapshot_pin", f"{DEFAULT_DIR}/{PIN} must name the hosted snapshot's batchDigest", f"/{DEFAULT_DIR}/{PIN}"
152
+ "snapshot_pinned",
153
+ "This world's data is a hosted snapshot (db/snapshot.json), which its data files "
154
+ "cannot reproduce: import rows through the platform (`gateway worlds data import "
155
+ "<slug>` or import_world_data) instead of into the checkout",
156
+ PIN_POINTER,
126
157
  )
127
- return pin
158
+
159
+
160
+ @dataclass(frozen=True)
161
+ class SnapshotPin:
162
+ """``db/snapshot.json``: the hosted snapshot a data-only version opens in place of its
163
+ data files, and the lock that snapshot was built under — its ``contractHash``, plus the
164
+ ``lock`` itself once a compile has moved ``schema/lock.json`` off it, so the pin keeps
165
+ proving which domain the hosted rows were stored under, and that domain's compiled
166
+ ``entities`` once a change adds to it, so the platform can migrate the rows. The
167
+ platform binds ``contractHash`` to the snapshot it recorded; nothing here is trusted
168
+ beyond what hashes to it."""
169
+
170
+ world: Path
171
+ fields: dict
172
+
173
+ @classmethod
174
+ def of(cls, world: str | Path) -> "SnapshotPin | None":
175
+ path = pin_path(world)
176
+ if not path.is_file():
177
+ return None
178
+ fields = read_json(path)
179
+ digest = fields.get("batchDigest") if isinstance(fields, dict) else None
180
+ if not isinstance(digest, str) or len(digest) != 64:
181
+ fail("snapshot_pin", f"{DEFAULT_DIR}/{PIN} must name the hosted snapshot's batchDigest", PIN_POINTER)
182
+ return cls(Path(world), fields)
183
+
184
+ @property
185
+ def batch_digest(self) -> str:
186
+ return self.fields["batchDigest"]
187
+
188
+ def counts(self, contract: Contract) -> dict[str, int]:
189
+ """The hosted snapshot's entity counts as the platform recorded them, per entity."""
190
+ counts = self.fields.get("entityCounts")
191
+ if not isinstance(counts, dict) or any(
192
+ not isinstance(value, int) or isinstance(value, bool) for value in counts.values()
193
+ ):
194
+ fail("snapshot_pin", f"{DEFAULT_DIR}/{PIN} must carry the hosted snapshot's entityCounts", PIN_POINTER)
195
+ return {entity: int(counts.get(entity, 0)) for entity in contract.entities}
196
+
197
+ def view(self, contract: Contract) -> dict:
198
+ """What compile reports for a pinned tree that does not hold its hosted snapshot."""
199
+ migration = self.migration(contract)
200
+ shown = {
201
+ "pinned": True,
202
+ "batchDigest": self.batch_digest,
203
+ "contractHash": self.fields.get("contractHash"),
204
+ "entityCounts": self.counts(contract),
205
+ }
206
+ return shown if migration is None else {**shown, "migrates": migration.view()}
207
+
208
+ def require_storage(self, contract: Contract) -> None:
209
+ """Refuse ``contract`` unless the hosted rows carry over to it."""
210
+ self.migration(contract)
211
+
212
+ def migration(self, contract: Contract) -> DomainMigration | None:
213
+ """How the hosted rows move to ``contract``: None when they open under it as they are,
214
+ refused when they cannot carry over or this tree cannot prove the domain they were
215
+ stored under."""
216
+ if self.fields.get("contractHash") == contract_hash(contract):
217
+ return None
218
+ migration = DomainMigration.of_lock(self._built_under(), contract, DomainMigration.recorded(self.world))
219
+ migration.require("snapshot_pinned", PIN_POINTER, REIMPORT)
220
+ return migration if migration.needed else None
221
+
222
+ def carried(self, contract: Contract) -> "SnapshotPin | None":
223
+ """The pin as compile writes it for ``contract``: the lock the hosted rows were stored
224
+ under while the tree's own lock is another, and that domain's entities while the rows
225
+ await a migration. ``None`` when the pin stays as it is."""
226
+ migration = self.migration(contract)
227
+ kept = {key: value for key, value in self.fields.items() if key not in ("lock", "entities")}
228
+ if self.fields.get("contractHash") != contract_hash(contract):
229
+ self.counts(contract)
230
+ kept["lock"] = self._built_under()
231
+ if migration is not None:
232
+ kept["entities"] = migration.entities
233
+ return None if kept == self.fields else SnapshotPin(self.world, kept)
234
+
235
+ def write(self) -> None:
236
+ text = json.dumps(self.fields, indent=2, ensure_ascii=False) + "\n"
237
+ pin_path(self.world).write_text(text, encoding="utf-8")
238
+
239
+ def _built_under(self) -> dict:
240
+ lock = self._proven_lock()
241
+ if lock is not None:
242
+ return lock
243
+ named = str(self.fields.get("contractHash"))[:12]
244
+ fail(
245
+ "snapshot_pinned",
246
+ f"{DEFAULT_DIR}/{PIN} names a snapshot built under a different contract ({named}) and "
247
+ f"{LOCK} is no longer that contract, so this tree cannot prove the domain is unchanged. "
248
+ f"Restore {LOCK} as the version you started from has it (a fresh `gateway worlds sandbox "
249
+ f"open <slug>`, or `gateway bench pull <slug>`) and compile again, or {REIMPORT}",
250
+ PIN_POINTER,
251
+ )
252
+
253
+ def _proven_lock(self) -> dict | None:
254
+ """The lock the hosted snapshot was built under, when this tree proves it: the pin's
255
+ own ``lock`` (refused unless it hashes to ``contractHash``), else ``schema/lock.json``
256
+ while it still is that lock."""
257
+ if "lock" in self.fields:
258
+ return self._authentic(self.fields["lock"])
259
+ on_disk = self._tree_lock()
260
+ if on_disk is None or lock_hash(on_disk) != self.fields.get("contractHash"):
261
+ return None
262
+ return on_disk
263
+
264
+ def _authentic(self, lock: Any) -> dict:
265
+ if not isinstance(lock, dict) or lock_hash(lock) != self.fields.get("contractHash"):
266
+ fail(
267
+ "snapshot_pin",
268
+ f"The lock in {DEFAULT_DIR}/{PIN} is not the contract its contractHash names: "
269
+ "restore the pin the version came with",
270
+ PIN_POINTER,
271
+ )
272
+ return lock
273
+
274
+ def _tree_lock(self) -> Any:
275
+ try:
276
+ return read_json(self.world / LOCK)
277
+ except ContractError:
278
+ return None
128
279
 
129
280
 
130
281
  def expected_default_digest(world: str | Path) -> str:
131
282
  """The batch digest the default snapshot must carry: the pin's for a data-only
132
283
  version, the data files' for an authored tree."""
133
- pin = pinned_default(world)
134
- return pin["batchDigest"] if pin is not None else default_batch_digest(world)
284
+ pin = SnapshotPin.of(world)
285
+ return pin.batch_digest if pin is not None else default_batch_digest(world)
135
286
 
136
287
 
137
288
  def scenario_dirs(world: str | Path) -> dict[str, Path]:
@@ -179,13 +330,16 @@ def scenario_batch_digest(scenario: Path, default_digest: str) -> str:
179
330
  )
180
331
 
181
332
 
182
- # ---------------------------------------------------------------- sidecars
183
-
184
-
185
333
  def sidecar_path(snapshot: Path) -> Path:
186
334
  return snapshot.with_name(SIDECAR)
187
335
 
188
336
 
337
+ def _shown(path: Path) -> str:
338
+ """A snapshot file as a refusal names it: inside the world, never where the runtime
339
+ keeps it (a server's temporary directory)."""
340
+ return f"{path.parent.name}/{path.name}"
341
+
342
+
189
343
  def read_sidecar(snapshot: Path) -> dict | None:
190
344
  path = sidecar_path(snapshot)
191
345
  if not path.is_file():
@@ -193,9 +347,10 @@ def read_sidecar(snapshot: Path) -> dict | None:
193
347
  try:
194
348
  sidecar = json.loads(path.read_text(encoding="utf-8"))
195
349
  except (OSError, ValueError) as error:
196
- raise ContractError("snapshot", f"Cannot read the snapshot sidecar {path.name}", str(path)) from error
350
+ shown = _shown(path)
351
+ raise ContractError("snapshot", f"Cannot read the snapshot sidecar {shown}", schema_path=shown) from error
197
352
  if not isinstance(sidecar, dict) or any(key not in sidecar for key in SIDECAR_KEYS):
198
- fail("snapshot", f"The snapshot sidecar {path.name} lacks {', '.join(SIDECAR_KEYS)}", str(path))
353
+ fail("snapshot", f"The snapshot sidecar {_shown(path)} lacks {', '.join(SIDECAR_KEYS)}", _shown(path))
199
354
  return sidecar
200
355
 
201
356
 
@@ -212,39 +367,93 @@ def write_sidecar(snapshot: Path, contract_digest: str, digest: str, counts: dic
212
367
  return sidecar
213
368
 
214
369
 
215
- def current(snapshot: Path, contract_digest: str, digest: str) -> dict | None:
216
- """The sidecar when this snapshot already is ``(contract_digest, digest)``, else None."""
370
+ def _verified_for(snapshot: Path, sidecar: dict, contract: Contract) -> dict:
371
+ """The sidecar, naming ``contract`` as the lock this snapshot was last verified under."""
372
+ if sidecar["contractHash"] == contract_hash(contract):
373
+ return sidecar
374
+ verified = {**sidecar, "contractHash": contract_hash(contract)}
375
+ sidecar_path(snapshot).write_text(json.dumps(verified, indent=1) + "\n", encoding="utf-8")
376
+ return verified
377
+
378
+
379
+ def _holding(snapshot: Path, digest: str) -> dict | None:
217
380
  if not snapshot.is_file():
218
381
  return None
219
382
  sidecar = read_sidecar(snapshot)
220
- if sidecar and sidecar["contractHash"] == contract_digest and sidecar["batchDigest"] == digest:
221
- return sidecar
222
- return None
383
+ return sidecar if sidecar is not None and sidecar["batchDigest"] == digest else None
384
+
385
+
386
+ def holds(snapshot: Path, contract: Contract, digest: str) -> dict | None:
387
+ """The sidecar when this snapshot holds batch ``digest``, its bytes are the ones the
388
+ sidecar names, and ``contract`` may open its rows (stored under its storage contract),
389
+ whatever lock the sidecar names: the one validity rule, which :func:`verify` explains.
390
+ The bytes are checked before the file is opened."""
391
+ sidecar = _holding(snapshot, digest)
392
+ if sidecar is None or sidecar["fileSha256"] != file_sha256(snapshot):
393
+ return None
394
+ return sidecar if storage_refusal(snapshot, contract) is None else None
395
+
396
+
397
+ def storage_refusal(snapshot: Path, contract: Contract) -> ContractError | None:
398
+ """Why ``contract`` may not open the rows in ``snapshot`` (the store's own check,
399
+ without the row audit), or None when it may."""
400
+ engine = WorldDatabase.for_contract(contract)
401
+ try:
402
+ with engine.open(snapshot, contract, audit=False):
403
+ return None
404
+ except engine.native_error as error:
405
+ return ContractError("state_contract", f"{snapshot.name} is not a world store ({error})")
406
+ except ContractError as error:
407
+ if error.detail["code"] != "state_contract":
408
+ raise
409
+ return error
223
410
 
224
411
 
225
412
  def verify(snapshot: Path, contract: Contract) -> dict:
226
- """The sidecar of a snapshot whose bytes and contract it names: anything else is refused."""
413
+ """The sidecar of a snapshot whose bytes it names and whose rows ``contract`` may
414
+ open: anything else is refused."""
415
+ shown = _shown(snapshot)
227
416
  sidecar = read_sidecar(snapshot)
228
417
  if sidecar is None:
229
- fail("snapshot", f"{snapshot} has no {SIDECAR} sidecar; recompile", str(snapshot))
230
- if sidecar["contractHash"] != contract_hash(contract):
231
- fail("snapshot", f"{snapshot} was built for a different contract; recompile", str(snapshot))
418
+ fail("snapshot", f"{shown} has no {SIDECAR} sidecar; recompile", shown)
232
419
  if not snapshot.is_file():
233
- fail("snapshot", f"{snapshot} is missing; recompile", str(snapshot))
420
+ fail("snapshot", f"{shown} is missing; recompile", shown)
234
421
  if sidecar["fileSha256"] != file_sha256(snapshot):
235
- fail("snapshot", f"{snapshot} differs from its sidecar's fileSha256; recompile", str(snapshot))
422
+ fail("snapshot", f"{shown} differs from its sidecar's fileSha256; recompile", shown)
423
+ refusal = storage_refusal(snapshot, contract)
424
+ if refusal is not None:
425
+ fail(
426
+ "snapshot",
427
+ f"{shown} holds rows stored under a different contract ({refusal.detail['message']}); "
428
+ "recompile to rebuild it from the data files, or restore the domain it was built under",
429
+ shown,
430
+ changed=refusal.detail.get("changed", []),
431
+ )
236
432
  return sidecar
237
433
 
238
434
 
239
- # ---------------------------------------------------------------- build
435
+ def snapshot_name(contract: Contract) -> str:
436
+ """The file name the contract's engine writes a snapshot to."""
437
+ return WorldDatabase.for_contract(contract).snapshot_name
438
+
439
+
440
+ def default_snapshot(world: str | Path, contract: Contract) -> Path:
441
+ return Path(world) / DEFAULT_DIR / snapshot_name(contract)
442
+
240
443
 
444
+ def scenario_snapshot(world: str | Path, name: str, contract: Contract) -> Path:
445
+ return Path(world) / SCENARIOS / name / snapshot_name(contract)
241
446
 
242
- def default_snapshot(world: str | Path) -> Path:
243
- return Path(world) / DEFAULT_DIR / SNAPSHOT
244
447
 
448
+ def has_default_snapshot(world: str | Path) -> bool:
449
+ """Whether any engine's default snapshot is in the tree."""
450
+ return any((Path(world) / DEFAULT_DIR / engine.snapshot_name).is_file() for engine in WorldDatabase.engines())
245
451
 
246
- def scenario_snapshot(world: str | Path, name: str) -> Path:
247
- return Path(world) / SCENARIOS / name / SNAPSHOT
452
+
453
+ def build_names() -> tuple[str, ...]:
454
+ """Every engine's snapshot and sidecar names and their in-progress files: never authored, never copied."""
455
+ names = [engine.snapshot_name for engine in WorldDatabase.engines()]
456
+ return (*names, SIDECAR, *(f".{name}.*" for name in names))
248
457
 
249
458
 
250
459
  def _replace(snapshot: Path, fresh: Path) -> None:
@@ -255,32 +464,23 @@ def _replace(snapshot: Path, fresh: Path) -> None:
255
464
 
256
465
 
257
466
  def build_default(world: str | Path, contract: Contract, *, force: bool = False) -> dict:
258
- """``db/state.sqlite`` for the tree's data files; a snapshot already at this key is
259
- kept. A pinned tree's default is the hosted snapshot, which the data files cannot
260
- reproduce: it is kept when present and refused when absent."""
467
+ """``db/state.sqlite`` for the tree's data files; a snapshot already holding them in
468
+ rows this contract may open is kept. A pinned tree's default is the hosted snapshot,
469
+ which the data files cannot reproduce: it is kept when held and refused when not."""
261
470
  world = Path(world)
262
- snapshot = default_snapshot(world)
263
- contract_digest = contract_hash(contract)
264
- digest = expected_default_digest(world)
265
- kept = current(snapshot, contract_digest, digest)
266
- if pin_path(world).is_file():
267
- if kept is None:
268
- fail(
269
- "snapshot_pinned",
270
- f"{DEFAULT_DIR}/{PIN} pins the hosted snapshot {digest}, which is not in the tree: "
271
- f"run the world on the platform (`gateway worlds session open <slug>`, "
272
- f"load_world_to_sandbox), which serves that snapshot, or remove {DEFAULT_DIR}/{PIN} "
273
- f"to rebuild from the data files",
274
- f"/{DEFAULT_DIR}/{SNAPSHOT}",
275
- )
276
- return kept
277
- if not force and kept is not None:
278
- return kept
279
- fresh = snapshot.with_name(f".{SNAPSHOT}.building")
471
+ snapshot = default_snapshot(world, contract)
472
+ pin = SnapshotPin.of(world)
473
+ digest = pin.batch_digest if pin is not None else default_batch_digest(world)
474
+ kept = holds(snapshot, contract, digest)
475
+ if kept is None:
476
+ refuse_unheld_pin(world, contract)
477
+ if pin is not None or (kept is not None and not force):
478
+ return _verified_for(snapshot, kept, contract)
479
+ fresh = snapshot.with_name(f".{snapshot.name}.building")
280
480
  fresh.parent.mkdir(parents=True, exist_ok=True)
281
481
  fresh.unlink(missing_ok=True)
282
482
  try:
283
- with SQLiteStore.create(fresh, contract) as store:
483
+ with WorldDatabase.create(fresh, contract) as store:
284
484
  for path in _data_files(world):
285
485
  store.apply_batch(read_json(path))
286
486
  counts = store.counts()
@@ -288,7 +488,7 @@ def build_default(world: str | Path, contract: Contract, *, force: bool = False)
288
488
  fresh.unlink(missing_ok=True)
289
489
  raise
290
490
  _replace(snapshot, fresh)
291
- return write_sidecar(snapshot, contract_digest, digest, counts)
491
+ return write_sidecar(snapshot, contract_hash(contract), digest, counts)
292
492
 
293
493
 
294
494
  def build_scenario(world: str | Path, name: str, contract: Contract, default: dict, *, force: bool = False) -> dict:
@@ -297,20 +497,18 @@ def build_scenario(world: str | Path, name: str, contract: Contract, default: di
297
497
  scenario = scenario_dirs(world).get(name)
298
498
  if scenario is None:
299
499
  fail("scenario", f"No scenario {name}", f"/{SCENARIOS}/{name}")
300
- snapshot = scenario_snapshot(world, name)
301
- contract_digest = contract_hash(contract)
500
+ snapshot = scenario_snapshot(world, name, contract)
302
501
  digest = scenario_batch_digest(scenario, default["batchDigest"])
303
- if not force:
304
- kept = current(snapshot, contract_digest, digest)
305
- if kept is not None:
306
- return kept
502
+ kept = None if force else holds(snapshot, contract, digest)
503
+ if kept is not None:
504
+ return _verified_for(snapshot, kept, contract)
307
505
  from .data import seed_batch
308
506
 
309
- fresh = snapshot.with_name(f".{SNAPSHOT}.building")
507
+ fresh = snapshot.with_name(f".{snapshot.name}.building")
310
508
  fresh.unlink(missing_ok=True)
311
- shutil.copyfile(default_snapshot(world), fresh)
509
+ WorldDatabase.clone_file(default_snapshot(world, contract), fresh)
312
510
  try:
313
- with SQLiteStore.open(fresh, contract, audit=False) as store:
511
+ with WorldDatabase.open(fresh, contract, audit=False) as store:
314
512
  overlay = scenario_overlay(scenario)
315
513
  if overlay:
316
514
  store.apply_batch(seed_batch(store, contract, overlay, "replace", batch_id=f"scenario:{name}"))
@@ -319,25 +517,56 @@ def build_scenario(world: str | Path, name: str, contract: Contract, default: di
319
517
  fresh.unlink(missing_ok=True)
320
518
  raise
321
519
  _replace(snapshot, fresh)
322
- return write_sidecar(snapshot, contract_digest, digest, counts)
520
+ return write_sidecar(snapshot, contract_hash(contract), digest, counts)
323
521
 
324
522
 
325
523
  def build_all(world: str | Path, contract: Contract, *, force: bool = False) -> dict:
326
- """The default snapshot and one per scenario; ``{"default": sidecar, "scenarios": {name: sidecar}}``."""
524
+ """The default snapshot and one per scenario; ``{"default": sidecar, "scenarios": {name: sidecar}}``.
525
+ A pinned tree that does not hold its hosted snapshot builds none (its data lives on the
526
+ platform): the default is the pin's view, once the pin proves the rows' domain."""
327
527
  world = Path(world)
528
+ pin = SnapshotPin.of(world)
529
+ if pin is not None:
530
+ carry_held(world, contract)
531
+ if pin is not None and holds(default_snapshot(world, contract), contract, pin.batch_digest) is None:
532
+ return {"default": pin.view(contract), "scenarios": {}}
328
533
  default = build_default(world, contract, force=force)
329
534
  scenarios = {name: build_scenario(world, name, contract, default, force=force) for name in scenario_dirs(world)}
330
535
  return {"default": default, "scenarios": scenarios}
331
536
 
332
537
 
333
- def discard_all(world: str | Path) -> None:
334
- """Remove every snapshot in the tree: nothing may claim data that no longer applies."""
335
- world = Path(world)
336
- targets = [default_snapshot(world)]
538
+ def _snapshot_paths(world: Path) -> list[Path]:
337
539
  scenarios = world / SCENARIOS
540
+ directories = [world / DEFAULT_DIR]
338
541
  if scenarios.is_dir():
339
- targets.extend(path / SNAPSHOT for path in scenarios.iterdir() if path.is_dir())
340
- for snapshot in targets:
542
+ directories.extend(path for path in scenarios.iterdir() if path.is_dir())
543
+ return [directory / engine.snapshot_name for directory in directories for engine in WorldDatabase.engines()]
544
+
545
+
546
+ def carry_held(world: str | Path, contract: Contract) -> None:
547
+ """Every snapshot a pinned tree holds, carried to ``contract`` when its rows were stored
548
+ under a domain the contract only adds to, so ``check`` opens it as compile left it. A
549
+ snapshot the contract cannot take is left for ``check`` to refuse, naming why."""
550
+ world = Path(world)
551
+ engine = WorldDatabase.for_contract(contract)
552
+ for snapshot in _snapshot_paths(world):
553
+ if snapshot.name != engine.snapshot_name:
554
+ continue
555
+ sidecar = read_sidecar(snapshot) if snapshot.is_file() else None
556
+ if sidecar is None or sidecar["fileSha256"] != file_sha256(snapshot):
557
+ continue
558
+ if storage_refusal(snapshot, contract) is None:
559
+ continue
560
+ migration = DomainMigration.of_lock(engine.stored_lock(snapshot), contract, DomainMigration.recorded(world))
561
+ if not (migration.needed and migration.carries):
562
+ continue
563
+ counts = DomainMigration.carry(snapshot, contract, world)
564
+ write_sidecar(snapshot, contract_hash(contract), sidecar["batchDigest"], counts)
565
+
566
+
567
+ def discard_all(world: str | Path) -> None:
568
+ """Remove every snapshot in the tree: nothing may claim data that no longer applies."""
569
+ for snapshot in _snapshot_paths(Path(world)):
341
570
  snapshot.unlink(missing_ok=True)
342
571
  sidecar_path(snapshot).unlink(missing_ok=True)
343
572
 
@@ -346,67 +575,84 @@ def check_all(world: str | Path, contract: Contract) -> dict:
346
575
  """Every snapshot present in the tree, verified against its sidecar; absent ones are not required."""
347
576
  world = Path(world)
348
577
  found: dict[str, Any] = {"default": None, "scenarios": {}}
349
- snapshot = default_snapshot(world)
578
+ snapshot = default_snapshot(world, contract)
350
579
  if snapshot.is_file() or sidecar_path(snapshot).is_file():
351
580
  found["default"] = verify(snapshot, contract)
352
581
  # A snapshot built from earlier data than the tree holds is reported, not
353
582
  # refused: authoring edits journals and opens the world, and a served world
354
583
  # replays the journals past a stale snapshot (WorldState opens only a
355
584
  # current one). `stale` is in every check report so it never passes unseen.
356
- found["default"]["stale"] = pinned_default(world) is None and found["default"][
585
+ found["default"]["stale"] = SnapshotPin.of(world) is None and found["default"][
357
586
  "batchDigest"
358
587
  ] != default_batch_digest(world)
359
588
  for name in scenario_dirs(world):
360
- snapshot = scenario_snapshot(world, name)
589
+ snapshot = scenario_snapshot(world, name, contract)
361
590
  if snapshot.is_file() or sidecar_path(snapshot).is_file():
362
591
  found["scenarios"][name] = verify(snapshot, contract)
363
592
  return found
364
593
 
365
594
 
366
- # ---------------------------------------------------------------- open
367
-
368
-
369
- def usable(snapshot: Path, contract: Contract) -> dict | None:
370
- """The sidecar when the snapshot exists and was built for this contract: what a
371
- served world opens. A snapshot for another contract is ignored, not an error;
372
- the legacy path (replay the JSON) still serves the world."""
373
- if not snapshot.is_file():
595
+ def pinned_counts(world: str | Path, contract: Contract) -> dict[str, int] | None:
596
+ """For validation: None for a tree whose own data stands (no pin, or the hosted snapshot
597
+ held beside it), else the hosted snapshot's entity counts. Either way a pin whose rows
598
+ were stored under another domain, or whose domain this tree cannot prove, is refused."""
599
+ pin = SnapshotPin.of(world)
600
+ if pin is None:
374
601
  return None
375
- sidecar = read_sidecar(snapshot)
376
- if sidecar is None or sidecar["contractHash"] != contract_hash(contract):
602
+ pin.require_storage(contract)
603
+ if current_default(world, contract) is not None:
377
604
  return None
378
- return sidecar
605
+ return pin.counts(contract)
606
+
607
+
608
+ def migration_view(world: str | Path, contract: Contract) -> dict | None:
609
+ """What the platform must migrate before a version of this tree can open its hosted
610
+ rows: the lock they were stored under and what the change adds; None when nothing."""
611
+ pin = SnapshotPin.of(world)
612
+ migration = None if pin is None else pin.migration(contract)
613
+ return None if migration is None else migration.view()
379
614
 
380
615
 
381
616
  def current_default(world: str | Path, contract: Contract) -> dict | None:
382
- """The default snapshot's sidecar when it is exactly the tree's data through this
383
- contract (what ``compile`` would skip); None when absent or stale."""
617
+ """The default snapshot's sidecar when it holds exactly the tree's data (or the pin's)
618
+ in rows this contract may open; None when absent or stale."""
384
619
  world = Path(world)
385
- snapshot = default_snapshot(world)
620
+ snapshot = default_snapshot(world, contract)
386
621
  if not snapshot.is_file() or not (world / "data/initial.json").is_file():
387
622
  return None
388
- return current(snapshot, contract_hash(contract), expected_default_digest(world))
623
+ return holds(snapshot, contract, expected_default_digest(world))
389
624
 
390
625
 
391
- def open_copy(snapshot: Path, contract: Contract, destination: Path) -> SQLiteStore:
392
- """A session's own copy of the snapshot, opened without the row audit: the file
393
- was audited when it was written and verified by digest since."""
394
- destination.parent.mkdir(parents=True, exist_ok=True)
395
- shutil.copyfile(snapshot, destination)
396
- return SQLiteStore.open(destination, contract, audit=False)
626
+ @dataclass(frozen=True)
627
+ class ServedSnapshot:
628
+ """What a served world opens: its contract, the tree checked against it, and the
629
+ sidecar of the default snapshot when that holds the tree's data (or the pin's) in
630
+ rows the contract may open, None on the legacy path. A pinned tree that holds no
631
+ snapshot is refused. Checking hashes the snapshot, so a version cache checks its
632
+ immutable tree once and hands the answer to every session of the version."""
397
633
 
634
+ contract: Contract
635
+ sidecar: dict | None
398
636
 
399
- # ---------------------------------------------------------------- hosted pod
637
+ @classmethod
638
+ def of(cls, world: str | Path) -> "ServedSnapshot":
639
+ contract = check_bundle(world)
640
+ sidecar = current_default(world, contract)
641
+ if sidecar is None:
642
+ refuse_unheld_pin(world, contract)
643
+ return cls(contract, sidecar)
400
644
 
401
645
 
402
646
  def fetch(world: str | Path, contract: Contract, url: str, sha256: str, digest: str) -> dict:
403
- """Download the pod's snapshot to ``db/state.sqlite``: gunzip when the URL ends in
404
- ``.gz`` or the response says so, verify the sha256 of the file bytes, write the
405
- sidecar. A mismatch is a refusal naming the digest; nothing usable is left behind."""
647
+ """Download the pod's snapshot to the default snapshot: gunzip when the URL ends in
648
+ ``.gz`` or the response says so, verify the sha256 of the file bytes, open it under
649
+ the version's contract (refused when its rows were stored under another domain),
650
+ write the sidecar. A mismatch is a refusal naming the digest; nothing usable is left
651
+ behind."""
406
652
  world = Path(world)
407
- snapshot = default_snapshot(world)
653
+ snapshot = default_snapshot(world, contract)
408
654
  snapshot.parent.mkdir(parents=True, exist_ok=True)
409
- partial = snapshot.with_name(f".{SNAPSHOT}.download")
655
+ partial = snapshot.with_name(f".{snapshot.name}.download")
410
656
  hasher = hashlib.sha256()
411
657
  try:
412
658
  with urllib.request.urlopen(urllib.request.Request(url), timeout=300) as response:
@@ -428,7 +674,7 @@ def fetch(world: str | Path, contract: Contract, url: str, sha256: str, digest:
428
674
  f"Snapshot {digest} downloaded with sha256 {hasher.hexdigest()}, expected {sha256}",
429
675
  )
430
676
  try:
431
- with SQLiteStore.open(partial, contract, audit=False) as store:
677
+ with WorldDatabase.open(partial, contract, audit=False) as store:
432
678
  counts = store.counts()
433
679
  except ContractError:
434
680
  partial.unlink(missing_ok=True)
@@ -446,15 +692,18 @@ def fetch(world: str | Path, contract: Contract, url: str, sha256: str, digest:
446
692
 
447
693
  __all__ = [
448
694
  "EMPTY_DIGEST",
695
+ "PIN_REFUSALS",
449
696
  "ENV_DIGEST",
450
697
  "ENV_SHA256",
451
698
  "ENV_URL",
452
699
  "SIDECAR",
453
- "SNAPSHOT",
700
+ "ServedSnapshot",
701
+ "SnapshotPin",
454
702
  "batch_digest",
455
703
  "build_all",
456
704
  "build_default",
457
705
  "build_scenario",
706
+ "carry_held",
458
707
  "check_all",
459
708
  "contract_hash",
460
709
  "current_default",
@@ -464,13 +713,20 @@ __all__ = [
464
713
  "expected_default_digest",
465
714
  "fetch",
466
715
  "file_sha256",
467
- "open_copy",
468
- "pinned_default",
716
+ "holds",
717
+ "lock_hash",
718
+ "migration_view",
719
+ "has_default_snapshot",
720
+ "build_names",
721
+ "snapshot_name",
722
+ "pinned_counts",
469
723
  "read_sidecar",
724
+ "refuse_pinned_rows",
725
+ "refuse_unheld_pin",
470
726
  "scenario_dirs",
471
727
  "scenario_overlay",
472
728
  "scenario_snapshot",
473
- "usable",
729
+ "storage_refusal",
474
730
  "verify",
475
731
  "write_sidecar",
476
732
  ]