@withgateway/sdk 0.3.1 → 0.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +2880 -2032
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2982 -2135
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +180 -41
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +172 -35
- package/dist/files/index.mjs.map +1 -1
- package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
- package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1309 -540
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1287 -532
- package/dist/index.mjs.map +1 -1
- package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
- package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
- package/dist/traces/index.d.mts +1 -1
- package/dist/traces/index.d.ts +1 -1
- package/dist/traces/index.js +177 -38
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +169 -32
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +2 -2
- package/dist/tracing/index.d.ts +2 -2
- package/dist/tracing/index.js +69 -13
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +69 -13
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +249 -120
- package/dist/worlds/index.d.ts +249 -120
- package/dist/worlds/index.js +1240 -484
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1231 -476
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +2 -27
- package/runtime/gatewaysdk/environment/_materialize.py +49 -69
- package/runtime/gatewaysdk/environment/_world.py +122 -197
- package/runtime/gatewaysdk/environment/backing.py +137 -0
- package/runtime/gatewaysdk/environment/core.py +1 -1
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +301 -0
- package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
- package/runtime/gatewaysdk/environment/runtime.py +8 -13
- package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
- package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
- package/runtime/gatewaysdk/environment/schema/api.py +266 -514
- package/runtime/gatewaysdk/environment/schema/batch.py +42 -37
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
- package/runtime/gatewaysdk/environment/schema/call_log.py +291 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +344 -0
- package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
- package/runtime/gatewaysdk/environment/schema/conform.py +8 -12
- package/runtime/gatewaysdk/environment/schema/connector.py +27 -62
- package/runtime/gatewaysdk/environment/schema/data.py +50 -207
- package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
- package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +995 -0
- package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
- package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
- package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
- package/runtime/gatewaysdk/environment/schema/database/guard.py +208 -0
- package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
- package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
- package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
- package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
- package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +367 -0
- package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +169 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
- package/runtime/gatewaysdk/environment/schema/grading.py +45 -30
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +142 -12
- package/runtime/gatewaysdk/environment/schema/host.py +158 -250
- package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
- package/runtime/gatewaysdk/environment/schema/local.py +44 -71
- package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
- package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
- package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
- package/runtime/gatewaysdk/environment/schema/platform.py +123 -171
- package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
- package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
- package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
- package/runtime/gatewaysdk/environment/schema/runtime.py +77 -160
- package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
- package/runtime/gatewaysdk/environment/schema/session_files.py +166 -0
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +2 -2
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +19 -11
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +29 -3
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +14 -14
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +12 -12
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +15 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
- package/runtime/gatewaysdk/environment/schema/snapshot.py +385 -129
- package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
- package/runtime/gatewaysdk/environment/schema/tasks.py +140 -55
- package/runtime/gatewaysdk/environment/schema/tools_world.py +110 -87
- package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
- package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
- package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
- package/runtime/gatewaysdk/environment/schema/validation.py +166 -40
- package/runtime/gatewaysdk/environment/schema/workers.py +142 -57
- package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +71 -39
- package/runtime/gatewaysdk/environment/score.py +57 -0
- package/runtime/gatewaysdk/environment/secret_values.py +36 -0
- package/runtime/gatewaysdk/environment/world.py +42 -3
- package/runtime/gatewaysdk/environment/world_session_contract.py +32 -5
- package/skills/benchmarks-getting-started/SKILL.md +2 -2
- package/skills/connector-schema-authoring/SKILL.md +19 -11
- package/skills/connector-schema-authoring/connector-worlds.md +13 -4
- package/skills/connector-schema-authoring/world-schema.md +103 -51
- package/skills/world-data-ingestion/ROWS.md +29 -3
- package/skills/world-data-ingestion/SKILL.md +14 -14
- package/skills/worlds-getting-started/SKILL.md +12 -12
- package/skills/worlds-getting-started/references/cli-mcp-map.md +3 -2
- package/skills/worlds-getting-started/references/gotchas.md +15 -0
- package/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
|
@@ -1,22 +1,31 @@
|
|
|
1
|
-
"""World data snapshots: the validated
|
|
1
|
+
"""World data snapshots: the validated world database file, built once and opened at start.
|
|
2
2
|
|
|
3
|
-
A snapshot is a
|
|
4
|
-
inside it, as always) plus a ``state.json``
|
|
3
|
+
A snapshot is a file the store wrote, named by its engine (``state.sqlite`` for
|
|
4
|
+
SQLite; contract lock and batch ledger inside it, as always) plus a ``state.json``
|
|
5
|
+
sidecar naming what it holds::
|
|
5
6
|
|
|
6
7
|
{"contractHash": ..., "batchDigest": ..., "entityCounts": {...},
|
|
7
8
|
"builtAt": ..., "runtime": "<gatewaysdk version>", "fileSha256": ...}
|
|
8
9
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
10
|
+
``batchDigest`` says which data it holds. A snapshot is valid for every contract with
|
|
11
|
+
the :class:`~.compiler.StorageContract` its rows were stored under, which the file
|
|
12
|
+
records and the store checks on every open: a tool, handler, route, test, package
|
|
13
|
+
version or documentation-file edit keeps it. A domain edit that only adds to the domain
|
|
14
|
+
(:class:`~.domain_migration.DomainMigration`: a new entity, a new field no row has to
|
|
15
|
+
carry, another ``domain_version``) carries it: compile migrates the snapshots the tree
|
|
16
|
+
holds, and keeps a pin's proof of the stored domain so the platform migrates the hosted
|
|
17
|
+
rows when it commits the change. Any other domain edit refuses it, naming the change.
|
|
18
|
+
``contractHash`` is the lock it was last built or verified under, a record rather than
|
|
19
|
+
the rule. ``db/<snapshot name>`` is the default snapshot (``data/initial.json`` applied,
|
|
20
|
+
plus any ``data/batches/*.json``); ``scenarios/<name>/<snapshot name>`` is the default
|
|
21
|
+
with that scenario's ``seed/*.json`` overlays replacing whole entities. ``compile``
|
|
22
|
+
builds them (one that already holds the same data under the same storage contract is
|
|
23
|
+
kept), ``check`` verifies them, the served world copies one to its temp dir and opens it
|
|
24
|
+
instead of replaying the JSON.
|
|
15
25
|
|
|
16
26
|
Digests (the contract, section 0):
|
|
17
27
|
|
|
18
|
-
* contract hash: sha256 of the canonical contract lock
|
|
19
|
-
``_check_contract`` compares;
|
|
28
|
+
* contract hash: sha256 of the canonical contract lock;
|
|
20
29
|
* batch digest: ``sha256(parent + "\\n" + batchId + "\\n" + "\\n".join(sorted(chunk digests)))``,
|
|
21
30
|
the empty world's parent being sixty-four zeros;
|
|
22
31
|
* the default batch's chunks are the tree's data files (``data/initial.json``
|
|
@@ -24,11 +33,13 @@ Digests (the contract, section 0):
|
|
|
24
33
|
is ``scenario:<name>`` over the default with its seed files as chunks.
|
|
25
34
|
|
|
26
35
|
A tree the platform published as a data-only version carries ``db/snapshot.json``,
|
|
27
|
-
the **pin
|
|
28
|
-
replaces the data files as the default batch. The tree's ``data/*.json``
|
|
29
|
-
longer describes the world's data — only that snapshot does — so a pinned tree
|
|
30
|
-
never rebuilt from its data files; the platform hands the file
|
|
31
|
-
(``load_world_to_sandbox``, the pod's ``WORLD_SNAPSHOT_*``).
|
|
36
|
+
the **pin** (:class:`SnapshotPin`): the hosted snapshot's ``batchDigest`` (and sidecar
|
|
37
|
+
fields), which replaces the data files as the default batch. The tree's ``data/*.json``
|
|
38
|
+
then no longer describes the world's data — only that snapshot does — so a pinned tree
|
|
39
|
+
is never rebuilt from, nor served from, its data files; the platform hands the file
|
|
40
|
+
down (``load_world_to_sandbox``, the pod's ``WORLD_SNAPSHOT_*``). Where the file is not
|
|
41
|
+
handed down (a sandbox), the pin itself proves which domain the hosted rows were stored
|
|
42
|
+
under, so the contract still compiles and checks.
|
|
32
43
|
|
|
33
44
|
Everything here is standard library, so the api bundle's server can load it.
|
|
34
45
|
"""
|
|
@@ -40,16 +51,17 @@ import hashlib
|
|
|
40
51
|
import json
|
|
41
52
|
import shutil
|
|
42
53
|
import urllib.request
|
|
54
|
+
from dataclasses import dataclass
|
|
43
55
|
from datetime import datetime, timezone
|
|
44
56
|
from pathlib import Path
|
|
45
57
|
from typing import Any, Iterable
|
|
46
58
|
|
|
47
59
|
from .._toml import tomllib
|
|
48
|
-
from .compiler import Contract
|
|
49
|
-
from .
|
|
60
|
+
from .compiler import LOCK, Contract, StorageContract, check_bundle
|
|
61
|
+
from .database import WorldDatabase
|
|
62
|
+
from .domain_migration import DomainMigration
|
|
50
63
|
from .validation import ContractError, canonical, fail, read_json
|
|
51
64
|
|
|
52
|
-
SNAPSHOT = "state.sqlite"
|
|
53
65
|
SIDECAR = "state.json"
|
|
54
66
|
PIN = "snapshot.json"
|
|
55
67
|
DEFAULT_DIR = "db"
|
|
@@ -60,14 +72,19 @@ SIDECAR_KEYS = ("contractHash", "batchDigest", "entityCounts", "builtAt", "runti
|
|
|
60
72
|
ENV_URL = "WORLD_SNAPSHOT_URL"
|
|
61
73
|
ENV_SHA256 = "WORLD_SNAPSHOT_SHA256"
|
|
62
74
|
ENV_DIGEST = "WORLD_SNAPSHOT_DIGEST"
|
|
75
|
+
PIN_POINTER = f"/{DEFAULT_DIR}/{PIN}"
|
|
76
|
+
#: The codes a pin is refused with: never a data refusal, so never discarded as one.
|
|
77
|
+
PIN_REFUSALS = ("snapshot_pin", "snapshot_pinned")
|
|
78
|
+
REIMPORT = f"remove {DEFAULT_DIR}/{PIN} and import every row again (`gateway worlds data import <slug> <rows>`)"
|
|
63
79
|
_BLOCK = 1024 * 1024
|
|
64
80
|
|
|
65
81
|
|
|
66
|
-
|
|
82
|
+
def lock_hash(lock: Any) -> str:
|
|
83
|
+
return hashlib.sha256(canonical(lock).encode("utf-8")).hexdigest()
|
|
67
84
|
|
|
68
85
|
|
|
69
86
|
def contract_hash(contract: Contract) -> str:
|
|
70
|
-
return
|
|
87
|
+
return lock_hash(contract.lock)
|
|
71
88
|
|
|
72
89
|
|
|
73
90
|
def batch_digest(parent: str, batch_id: str, chunk_digests: Iterable[str]) -> str:
|
|
@@ -113,25 +130,159 @@ def pin_path(world: str | Path) -> Path:
|
|
|
113
130
|
return Path(world) / DEFAULT_DIR / PIN
|
|
114
131
|
|
|
115
132
|
|
|
116
|
-
def
|
|
117
|
-
"""
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
if not
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
133
|
+
def refuse_unheld_pin(world: str | Path, contract: Contract) -> None:
|
|
134
|
+
"""A pinned tree that does not hold its hosted snapshot has no rows here: nothing
|
|
135
|
+
is built, copied or served from its data files in their place."""
|
|
136
|
+
pin = SnapshotPin.of(world)
|
|
137
|
+
if pin is not None:
|
|
138
|
+
fail(
|
|
139
|
+
"snapshot_pinned",
|
|
140
|
+
f"{DEFAULT_DIR}/{PIN} pins the hosted snapshot {pin.batch_digest}, which this tree does not hold: "
|
|
141
|
+
f"run the world on the platform (`gateway worlds session open <slug>`, load_world_to_sandbox), "
|
|
142
|
+
f"which serves that snapshot, or remove {DEFAULT_DIR}/{PIN} to rebuild from the data files",
|
|
143
|
+
f"/{DEFAULT_DIR}/{snapshot_name(contract)}",
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def refuse_pinned_rows(world: str | Path) -> None:
|
|
148
|
+
"""Rows written into a pinned tree's data files would never be served: the hosted
|
|
149
|
+
snapshot, not the data files, is the world's data."""
|
|
150
|
+
if SnapshotPin.of(world) is not None:
|
|
124
151
|
fail(
|
|
125
|
-
"
|
|
152
|
+
"snapshot_pinned",
|
|
153
|
+
"This world's data is a hosted snapshot (db/snapshot.json), which its data files "
|
|
154
|
+
"cannot reproduce: import rows through the platform (`gateway worlds data import "
|
|
155
|
+
"<slug>` or import_world_data) instead of into the checkout",
|
|
156
|
+
PIN_POINTER,
|
|
126
157
|
)
|
|
127
|
-
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
@dataclass(frozen=True)
|
|
161
|
+
class SnapshotPin:
|
|
162
|
+
"""``db/snapshot.json``: the hosted snapshot a data-only version opens in place of its
|
|
163
|
+
data files, and the lock that snapshot was built under — its ``contractHash``, plus the
|
|
164
|
+
``lock`` itself once a compile has moved ``schema/lock.json`` off it, so the pin keeps
|
|
165
|
+
proving which domain the hosted rows were stored under, and that domain's compiled
|
|
166
|
+
``entities`` once a change adds to it, so the platform can migrate the rows. The
|
|
167
|
+
platform binds ``contractHash`` to the snapshot it recorded; nothing here is trusted
|
|
168
|
+
beyond what hashes to it."""
|
|
169
|
+
|
|
170
|
+
world: Path
|
|
171
|
+
fields: dict
|
|
172
|
+
|
|
173
|
+
@classmethod
|
|
174
|
+
def of(cls, world: str | Path) -> "SnapshotPin | None":
|
|
175
|
+
path = pin_path(world)
|
|
176
|
+
if not path.is_file():
|
|
177
|
+
return None
|
|
178
|
+
fields = read_json(path)
|
|
179
|
+
digest = fields.get("batchDigest") if isinstance(fields, dict) else None
|
|
180
|
+
if not isinstance(digest, str) or len(digest) != 64:
|
|
181
|
+
fail("snapshot_pin", f"{DEFAULT_DIR}/{PIN} must name the hosted snapshot's batchDigest", PIN_POINTER)
|
|
182
|
+
return cls(Path(world), fields)
|
|
183
|
+
|
|
184
|
+
@property
|
|
185
|
+
def batch_digest(self) -> str:
|
|
186
|
+
return self.fields["batchDigest"]
|
|
187
|
+
|
|
188
|
+
def counts(self, contract: Contract) -> dict[str, int]:
|
|
189
|
+
"""The hosted snapshot's entity counts as the platform recorded them, per entity."""
|
|
190
|
+
counts = self.fields.get("entityCounts")
|
|
191
|
+
if not isinstance(counts, dict) or any(
|
|
192
|
+
not isinstance(value, int) or isinstance(value, bool) for value in counts.values()
|
|
193
|
+
):
|
|
194
|
+
fail("snapshot_pin", f"{DEFAULT_DIR}/{PIN} must carry the hosted snapshot's entityCounts", PIN_POINTER)
|
|
195
|
+
return {entity: int(counts.get(entity, 0)) for entity in contract.entities}
|
|
196
|
+
|
|
197
|
+
def view(self, contract: Contract) -> dict:
|
|
198
|
+
"""What compile reports for a pinned tree that does not hold its hosted snapshot."""
|
|
199
|
+
migration = self.migration(contract)
|
|
200
|
+
shown = {
|
|
201
|
+
"pinned": True,
|
|
202
|
+
"batchDigest": self.batch_digest,
|
|
203
|
+
"contractHash": self.fields.get("contractHash"),
|
|
204
|
+
"entityCounts": self.counts(contract),
|
|
205
|
+
}
|
|
206
|
+
return shown if migration is None else {**shown, "migrates": migration.view()}
|
|
207
|
+
|
|
208
|
+
def require_storage(self, contract: Contract) -> None:
|
|
209
|
+
"""Refuse ``contract`` unless the hosted rows carry over to it."""
|
|
210
|
+
self.migration(contract)
|
|
211
|
+
|
|
212
|
+
def migration(self, contract: Contract) -> DomainMigration | None:
|
|
213
|
+
"""How the hosted rows move to ``contract``: None when they open under it as they are,
|
|
214
|
+
refused when they cannot carry over or this tree cannot prove the domain they were
|
|
215
|
+
stored under."""
|
|
216
|
+
if self.fields.get("contractHash") == contract_hash(contract):
|
|
217
|
+
return None
|
|
218
|
+
migration = DomainMigration.of_lock(self._built_under(), contract, DomainMigration.recorded(self.world))
|
|
219
|
+
migration.require("snapshot_pinned", PIN_POINTER, REIMPORT)
|
|
220
|
+
return migration if migration.needed else None
|
|
221
|
+
|
|
222
|
+
def carried(self, contract: Contract) -> "SnapshotPin | None":
|
|
223
|
+
"""The pin as compile writes it for ``contract``: the lock the hosted rows were stored
|
|
224
|
+
under while the tree's own lock is another, and that domain's entities while the rows
|
|
225
|
+
await a migration. ``None`` when the pin stays as it is."""
|
|
226
|
+
migration = self.migration(contract)
|
|
227
|
+
kept = {key: value for key, value in self.fields.items() if key not in ("lock", "entities")}
|
|
228
|
+
if self.fields.get("contractHash") != contract_hash(contract):
|
|
229
|
+
self.counts(contract)
|
|
230
|
+
kept["lock"] = self._built_under()
|
|
231
|
+
if migration is not None:
|
|
232
|
+
kept["entities"] = migration.entities
|
|
233
|
+
return None if kept == self.fields else SnapshotPin(self.world, kept)
|
|
234
|
+
|
|
235
|
+
def write(self) -> None:
|
|
236
|
+
text = json.dumps(self.fields, indent=2, ensure_ascii=False) + "\n"
|
|
237
|
+
pin_path(self.world).write_text(text, encoding="utf-8")
|
|
238
|
+
|
|
239
|
+
def _built_under(self) -> dict:
|
|
240
|
+
lock = self._proven_lock()
|
|
241
|
+
if lock is not None:
|
|
242
|
+
return lock
|
|
243
|
+
named = str(self.fields.get("contractHash"))[:12]
|
|
244
|
+
fail(
|
|
245
|
+
"snapshot_pinned",
|
|
246
|
+
f"{DEFAULT_DIR}/{PIN} names a snapshot built under a different contract ({named}) and "
|
|
247
|
+
f"{LOCK} is no longer that contract, so this tree cannot prove the domain is unchanged. "
|
|
248
|
+
f"Restore {LOCK} as the version you started from has it (a fresh `gateway worlds sandbox "
|
|
249
|
+
f"open <slug>`, or `gateway bench pull <slug>`) and compile again, or {REIMPORT}",
|
|
250
|
+
PIN_POINTER,
|
|
251
|
+
)
|
|
252
|
+
|
|
253
|
+
def _proven_lock(self) -> dict | None:
|
|
254
|
+
"""The lock the hosted snapshot was built under, when this tree proves it: the pin's
|
|
255
|
+
own ``lock`` (refused unless it hashes to ``contractHash``), else ``schema/lock.json``
|
|
256
|
+
while it still is that lock."""
|
|
257
|
+
if "lock" in self.fields:
|
|
258
|
+
return self._authentic(self.fields["lock"])
|
|
259
|
+
on_disk = self._tree_lock()
|
|
260
|
+
if on_disk is None or lock_hash(on_disk) != self.fields.get("contractHash"):
|
|
261
|
+
return None
|
|
262
|
+
return on_disk
|
|
263
|
+
|
|
264
|
+
def _authentic(self, lock: Any) -> dict:
|
|
265
|
+
if not isinstance(lock, dict) or lock_hash(lock) != self.fields.get("contractHash"):
|
|
266
|
+
fail(
|
|
267
|
+
"snapshot_pin",
|
|
268
|
+
f"The lock in {DEFAULT_DIR}/{PIN} is not the contract its contractHash names: "
|
|
269
|
+
"restore the pin the version came with",
|
|
270
|
+
PIN_POINTER,
|
|
271
|
+
)
|
|
272
|
+
return lock
|
|
273
|
+
|
|
274
|
+
def _tree_lock(self) -> Any:
|
|
275
|
+
try:
|
|
276
|
+
return read_json(self.world / LOCK)
|
|
277
|
+
except ContractError:
|
|
278
|
+
return None
|
|
128
279
|
|
|
129
280
|
|
|
130
281
|
def expected_default_digest(world: str | Path) -> str:
|
|
131
282
|
"""The batch digest the default snapshot must carry: the pin's for a data-only
|
|
132
283
|
version, the data files' for an authored tree."""
|
|
133
|
-
pin =
|
|
134
|
-
return pin
|
|
284
|
+
pin = SnapshotPin.of(world)
|
|
285
|
+
return pin.batch_digest if pin is not None else default_batch_digest(world)
|
|
135
286
|
|
|
136
287
|
|
|
137
288
|
def scenario_dirs(world: str | Path) -> dict[str, Path]:
|
|
@@ -179,13 +330,16 @@ def scenario_batch_digest(scenario: Path, default_digest: str) -> str:
|
|
|
179
330
|
)
|
|
180
331
|
|
|
181
332
|
|
|
182
|
-
# ---------------------------------------------------------------- sidecars
|
|
183
|
-
|
|
184
|
-
|
|
185
333
|
def sidecar_path(snapshot: Path) -> Path:
|
|
186
334
|
return snapshot.with_name(SIDECAR)
|
|
187
335
|
|
|
188
336
|
|
|
337
|
+
def _shown(path: Path) -> str:
|
|
338
|
+
"""A snapshot file as a refusal names it: inside the world, never where the runtime
|
|
339
|
+
keeps it (a server's temporary directory)."""
|
|
340
|
+
return f"{path.parent.name}/{path.name}"
|
|
341
|
+
|
|
342
|
+
|
|
189
343
|
def read_sidecar(snapshot: Path) -> dict | None:
|
|
190
344
|
path = sidecar_path(snapshot)
|
|
191
345
|
if not path.is_file():
|
|
@@ -193,9 +347,10 @@ def read_sidecar(snapshot: Path) -> dict | None:
|
|
|
193
347
|
try:
|
|
194
348
|
sidecar = json.loads(path.read_text(encoding="utf-8"))
|
|
195
349
|
except (OSError, ValueError) as error:
|
|
196
|
-
|
|
350
|
+
shown = _shown(path)
|
|
351
|
+
raise ContractError("snapshot", f"Cannot read the snapshot sidecar {shown}", schema_path=shown) from error
|
|
197
352
|
if not isinstance(sidecar, dict) or any(key not in sidecar for key in SIDECAR_KEYS):
|
|
198
|
-
fail("snapshot", f"The snapshot sidecar {path
|
|
353
|
+
fail("snapshot", f"The snapshot sidecar {_shown(path)} lacks {', '.join(SIDECAR_KEYS)}", _shown(path))
|
|
199
354
|
return sidecar
|
|
200
355
|
|
|
201
356
|
|
|
@@ -212,39 +367,93 @@ def write_sidecar(snapshot: Path, contract_digest: str, digest: str, counts: dic
|
|
|
212
367
|
return sidecar
|
|
213
368
|
|
|
214
369
|
|
|
215
|
-
def
|
|
216
|
-
"""The sidecar
|
|
370
|
+
def _verified_for(snapshot: Path, sidecar: dict, contract: Contract) -> dict:
|
|
371
|
+
"""The sidecar, naming ``contract`` as the lock this snapshot was last verified under."""
|
|
372
|
+
if sidecar["contractHash"] == contract_hash(contract):
|
|
373
|
+
return sidecar
|
|
374
|
+
verified = {**sidecar, "contractHash": contract_hash(contract)}
|
|
375
|
+
sidecar_path(snapshot).write_text(json.dumps(verified, indent=1) + "\n", encoding="utf-8")
|
|
376
|
+
return verified
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def _holding(snapshot: Path, digest: str) -> dict | None:
|
|
217
380
|
if not snapshot.is_file():
|
|
218
381
|
return None
|
|
219
382
|
sidecar = read_sidecar(snapshot)
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
383
|
+
return sidecar if sidecar is not None and sidecar["batchDigest"] == digest else None
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def holds(snapshot: Path, contract: Contract, digest: str) -> dict | None:
|
|
387
|
+
"""The sidecar when this snapshot holds batch ``digest``, its bytes are the ones the
|
|
388
|
+
sidecar names, and ``contract`` may open its rows (stored under its storage contract),
|
|
389
|
+
whatever lock the sidecar names: the one validity rule, which :func:`verify` explains.
|
|
390
|
+
The bytes are checked before the file is opened."""
|
|
391
|
+
sidecar = _holding(snapshot, digest)
|
|
392
|
+
if sidecar is None or sidecar["fileSha256"] != file_sha256(snapshot):
|
|
393
|
+
return None
|
|
394
|
+
return sidecar if storage_refusal(snapshot, contract) is None else None
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def storage_refusal(snapshot: Path, contract: Contract) -> ContractError | None:
|
|
398
|
+
"""Why ``contract`` may not open the rows in ``snapshot`` (the store's own check,
|
|
399
|
+
without the row audit), or None when it may."""
|
|
400
|
+
engine = WorldDatabase.for_contract(contract)
|
|
401
|
+
try:
|
|
402
|
+
with engine.open(snapshot, contract, audit=False):
|
|
403
|
+
return None
|
|
404
|
+
except engine.native_error as error:
|
|
405
|
+
return ContractError("state_contract", f"{snapshot.name} is not a world store ({error})")
|
|
406
|
+
except ContractError as error:
|
|
407
|
+
if error.detail["code"] != "state_contract":
|
|
408
|
+
raise
|
|
409
|
+
return error
|
|
223
410
|
|
|
224
411
|
|
|
225
412
|
def verify(snapshot: Path, contract: Contract) -> dict:
|
|
226
|
-
"""The sidecar of a snapshot whose bytes
|
|
413
|
+
"""The sidecar of a snapshot whose bytes it names and whose rows ``contract`` may
|
|
414
|
+
open: anything else is refused."""
|
|
415
|
+
shown = _shown(snapshot)
|
|
227
416
|
sidecar = read_sidecar(snapshot)
|
|
228
417
|
if sidecar is None:
|
|
229
|
-
fail("snapshot", f"{
|
|
230
|
-
if sidecar["contractHash"] != contract_hash(contract):
|
|
231
|
-
fail("snapshot", f"{snapshot} was built for a different contract; recompile", str(snapshot))
|
|
418
|
+
fail("snapshot", f"{shown} has no {SIDECAR} sidecar; recompile", shown)
|
|
232
419
|
if not snapshot.is_file():
|
|
233
|
-
fail("snapshot", f"{
|
|
420
|
+
fail("snapshot", f"{shown} is missing; recompile", shown)
|
|
234
421
|
if sidecar["fileSha256"] != file_sha256(snapshot):
|
|
235
|
-
fail("snapshot", f"{
|
|
422
|
+
fail("snapshot", f"{shown} differs from its sidecar's fileSha256; recompile", shown)
|
|
423
|
+
refusal = storage_refusal(snapshot, contract)
|
|
424
|
+
if refusal is not None:
|
|
425
|
+
fail(
|
|
426
|
+
"snapshot",
|
|
427
|
+
f"{shown} holds rows stored under a different contract ({refusal.detail['message']}); "
|
|
428
|
+
"recompile to rebuild it from the data files, or restore the domain it was built under",
|
|
429
|
+
shown,
|
|
430
|
+
changed=refusal.detail.get("changed", []),
|
|
431
|
+
)
|
|
236
432
|
return sidecar
|
|
237
433
|
|
|
238
434
|
|
|
239
|
-
|
|
435
|
+
def snapshot_name(contract: Contract) -> str:
|
|
436
|
+
"""The file name the contract's engine writes a snapshot to."""
|
|
437
|
+
return WorldDatabase.for_contract(contract).snapshot_name
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def default_snapshot(world: str | Path, contract: Contract) -> Path:
|
|
441
|
+
return Path(world) / DEFAULT_DIR / snapshot_name(contract)
|
|
442
|
+
|
|
240
443
|
|
|
444
|
+
def scenario_snapshot(world: str | Path, name: str, contract: Contract) -> Path:
|
|
445
|
+
return Path(world) / SCENARIOS / name / snapshot_name(contract)
|
|
241
446
|
|
|
242
|
-
def default_snapshot(world: str | Path) -> Path:
|
|
243
|
-
return Path(world) / DEFAULT_DIR / SNAPSHOT
|
|
244
447
|
|
|
448
|
+
def has_default_snapshot(world: str | Path) -> bool:
|
|
449
|
+
"""Whether any engine's default snapshot is in the tree."""
|
|
450
|
+
return any((Path(world) / DEFAULT_DIR / engine.snapshot_name).is_file() for engine in WorldDatabase.engines())
|
|
245
451
|
|
|
246
|
-
|
|
247
|
-
|
|
452
|
+
|
|
453
|
+
def build_names() -> tuple[str, ...]:
|
|
454
|
+
"""Every engine's snapshot and sidecar names and their in-progress files: never authored, never copied."""
|
|
455
|
+
names = [engine.snapshot_name for engine in WorldDatabase.engines()]
|
|
456
|
+
return (*names, SIDECAR, *(f".{name}.*" for name in names))
|
|
248
457
|
|
|
249
458
|
|
|
250
459
|
def _replace(snapshot: Path, fresh: Path) -> None:
|
|
@@ -255,32 +464,23 @@ def _replace(snapshot: Path, fresh: Path) -> None:
|
|
|
255
464
|
|
|
256
465
|
|
|
257
466
|
def build_default(world: str | Path, contract: Contract, *, force: bool = False) -> dict:
|
|
258
|
-
"""``db/state.sqlite`` for the tree's data files; a snapshot already
|
|
259
|
-
kept. A pinned tree's default is the hosted snapshot,
|
|
260
|
-
reproduce: it is kept when
|
|
467
|
+
"""``db/state.sqlite`` for the tree's data files; a snapshot already holding them in
|
|
468
|
+
rows this contract may open is kept. A pinned tree's default is the hosted snapshot,
|
|
469
|
+
which the data files cannot reproduce: it is kept when held and refused when not."""
|
|
261
470
|
world = Path(world)
|
|
262
|
-
snapshot = default_snapshot(world)
|
|
263
|
-
|
|
264
|
-
digest =
|
|
265
|
-
kept =
|
|
266
|
-
if
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
f"run the world on the platform (`gateway worlds session open <slug>`, "
|
|
272
|
-
f"load_world_to_sandbox), which serves that snapshot, or remove {DEFAULT_DIR}/{PIN} "
|
|
273
|
-
f"to rebuild from the data files",
|
|
274
|
-
f"/{DEFAULT_DIR}/{SNAPSHOT}",
|
|
275
|
-
)
|
|
276
|
-
return kept
|
|
277
|
-
if not force and kept is not None:
|
|
278
|
-
return kept
|
|
279
|
-
fresh = snapshot.with_name(f".{SNAPSHOT}.building")
|
|
471
|
+
snapshot = default_snapshot(world, contract)
|
|
472
|
+
pin = SnapshotPin.of(world)
|
|
473
|
+
digest = pin.batch_digest if pin is not None else default_batch_digest(world)
|
|
474
|
+
kept = holds(snapshot, contract, digest)
|
|
475
|
+
if kept is None:
|
|
476
|
+
refuse_unheld_pin(world, contract)
|
|
477
|
+
if pin is not None or (kept is not None and not force):
|
|
478
|
+
return _verified_for(snapshot, kept, contract)
|
|
479
|
+
fresh = snapshot.with_name(f".{snapshot.name}.building")
|
|
280
480
|
fresh.parent.mkdir(parents=True, exist_ok=True)
|
|
281
481
|
fresh.unlink(missing_ok=True)
|
|
282
482
|
try:
|
|
283
|
-
with
|
|
483
|
+
with WorldDatabase.create(fresh, contract) as store:
|
|
284
484
|
for path in _data_files(world):
|
|
285
485
|
store.apply_batch(read_json(path))
|
|
286
486
|
counts = store.counts()
|
|
@@ -288,7 +488,7 @@ def build_default(world: str | Path, contract: Contract, *, force: bool = False)
|
|
|
288
488
|
fresh.unlink(missing_ok=True)
|
|
289
489
|
raise
|
|
290
490
|
_replace(snapshot, fresh)
|
|
291
|
-
return write_sidecar(snapshot,
|
|
491
|
+
return write_sidecar(snapshot, contract_hash(contract), digest, counts)
|
|
292
492
|
|
|
293
493
|
|
|
294
494
|
def build_scenario(world: str | Path, name: str, contract: Contract, default: dict, *, force: bool = False) -> dict:
|
|
@@ -297,20 +497,18 @@ def build_scenario(world: str | Path, name: str, contract: Contract, default: di
|
|
|
297
497
|
scenario = scenario_dirs(world).get(name)
|
|
298
498
|
if scenario is None:
|
|
299
499
|
fail("scenario", f"No scenario {name}", f"/{SCENARIOS}/{name}")
|
|
300
|
-
snapshot = scenario_snapshot(world, name)
|
|
301
|
-
contract_digest = contract_hash(contract)
|
|
500
|
+
snapshot = scenario_snapshot(world, name, contract)
|
|
302
501
|
digest = scenario_batch_digest(scenario, default["batchDigest"])
|
|
303
|
-
if
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
return kept
|
|
502
|
+
kept = None if force else holds(snapshot, contract, digest)
|
|
503
|
+
if kept is not None:
|
|
504
|
+
return _verified_for(snapshot, kept, contract)
|
|
307
505
|
from .data import seed_batch
|
|
308
506
|
|
|
309
|
-
fresh = snapshot.with_name(f".{
|
|
507
|
+
fresh = snapshot.with_name(f".{snapshot.name}.building")
|
|
310
508
|
fresh.unlink(missing_ok=True)
|
|
311
|
-
|
|
509
|
+
WorldDatabase.clone_file(default_snapshot(world, contract), fresh)
|
|
312
510
|
try:
|
|
313
|
-
with
|
|
511
|
+
with WorldDatabase.open(fresh, contract, audit=False) as store:
|
|
314
512
|
overlay = scenario_overlay(scenario)
|
|
315
513
|
if overlay:
|
|
316
514
|
store.apply_batch(seed_batch(store, contract, overlay, "replace", batch_id=f"scenario:{name}"))
|
|
@@ -319,25 +517,56 @@ def build_scenario(world: str | Path, name: str, contract: Contract, default: di
|
|
|
319
517
|
fresh.unlink(missing_ok=True)
|
|
320
518
|
raise
|
|
321
519
|
_replace(snapshot, fresh)
|
|
322
|
-
return write_sidecar(snapshot,
|
|
520
|
+
return write_sidecar(snapshot, contract_hash(contract), digest, counts)
|
|
323
521
|
|
|
324
522
|
|
|
325
523
|
def build_all(world: str | Path, contract: Contract, *, force: bool = False) -> dict:
|
|
326
|
-
"""The default snapshot and one per scenario; ``{"default": sidecar, "scenarios": {name: sidecar}}``.
|
|
524
|
+
"""The default snapshot and one per scenario; ``{"default": sidecar, "scenarios": {name: sidecar}}``.
|
|
525
|
+
A pinned tree that does not hold its hosted snapshot builds none (its data lives on the
|
|
526
|
+
platform): the default is the pin's view, once the pin proves the rows' domain."""
|
|
327
527
|
world = Path(world)
|
|
528
|
+
pin = SnapshotPin.of(world)
|
|
529
|
+
if pin is not None:
|
|
530
|
+
carry_held(world, contract)
|
|
531
|
+
if pin is not None and holds(default_snapshot(world, contract), contract, pin.batch_digest) is None:
|
|
532
|
+
return {"default": pin.view(contract), "scenarios": {}}
|
|
328
533
|
default = build_default(world, contract, force=force)
|
|
329
534
|
scenarios = {name: build_scenario(world, name, contract, default, force=force) for name in scenario_dirs(world)}
|
|
330
535
|
return {"default": default, "scenarios": scenarios}
|
|
331
536
|
|
|
332
537
|
|
|
333
|
-
def
|
|
334
|
-
"""Remove every snapshot in the tree: nothing may claim data that no longer applies."""
|
|
335
|
-
world = Path(world)
|
|
336
|
-
targets = [default_snapshot(world)]
|
|
538
|
+
def _snapshot_paths(world: Path) -> list[Path]:
|
|
337
539
|
scenarios = world / SCENARIOS
|
|
540
|
+
directories = [world / DEFAULT_DIR]
|
|
338
541
|
if scenarios.is_dir():
|
|
339
|
-
|
|
340
|
-
for
|
|
542
|
+
directories.extend(path for path in scenarios.iterdir() if path.is_dir())
|
|
543
|
+
return [directory / engine.snapshot_name for directory in directories for engine in WorldDatabase.engines()]
|
|
544
|
+
|
|
545
|
+
|
|
546
|
+
def carry_held(world: str | Path, contract: Contract) -> None:
|
|
547
|
+
"""Every snapshot a pinned tree holds, carried to ``contract`` when its rows were stored
|
|
548
|
+
under a domain the contract only adds to, so ``check`` opens it as compile left it. A
|
|
549
|
+
snapshot the contract cannot take is left for ``check`` to refuse, naming why."""
|
|
550
|
+
world = Path(world)
|
|
551
|
+
engine = WorldDatabase.for_contract(contract)
|
|
552
|
+
for snapshot in _snapshot_paths(world):
|
|
553
|
+
if snapshot.name != engine.snapshot_name:
|
|
554
|
+
continue
|
|
555
|
+
sidecar = read_sidecar(snapshot) if snapshot.is_file() else None
|
|
556
|
+
if sidecar is None or sidecar["fileSha256"] != file_sha256(snapshot):
|
|
557
|
+
continue
|
|
558
|
+
if storage_refusal(snapshot, contract) is None:
|
|
559
|
+
continue
|
|
560
|
+
migration = DomainMigration.of_lock(engine.stored_lock(snapshot), contract, DomainMigration.recorded(world))
|
|
561
|
+
if not (migration.needed and migration.carries):
|
|
562
|
+
continue
|
|
563
|
+
counts = DomainMigration.carry(snapshot, contract, world)
|
|
564
|
+
write_sidecar(snapshot, contract_hash(contract), sidecar["batchDigest"], counts)
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
def discard_all(world: str | Path) -> None:
|
|
568
|
+
"""Remove every snapshot in the tree: nothing may claim data that no longer applies."""
|
|
569
|
+
for snapshot in _snapshot_paths(Path(world)):
|
|
341
570
|
snapshot.unlink(missing_ok=True)
|
|
342
571
|
sidecar_path(snapshot).unlink(missing_ok=True)
|
|
343
572
|
|
|
@@ -346,67 +575,84 @@ def check_all(world: str | Path, contract: Contract) -> dict:
|
|
|
346
575
|
"""Every snapshot present in the tree, verified against its sidecar; absent ones are not required."""
|
|
347
576
|
world = Path(world)
|
|
348
577
|
found: dict[str, Any] = {"default": None, "scenarios": {}}
|
|
349
|
-
snapshot = default_snapshot(world)
|
|
578
|
+
snapshot = default_snapshot(world, contract)
|
|
350
579
|
if snapshot.is_file() or sidecar_path(snapshot).is_file():
|
|
351
580
|
found["default"] = verify(snapshot, contract)
|
|
352
581
|
# A snapshot built from earlier data than the tree holds is reported, not
|
|
353
582
|
# refused: authoring edits journals and opens the world, and a served world
|
|
354
583
|
# replays the journals past a stale snapshot (WorldState opens only a
|
|
355
584
|
# current one). `stale` is in every check report so it never passes unseen.
|
|
356
|
-
found["default"]["stale"] =
|
|
585
|
+
found["default"]["stale"] = SnapshotPin.of(world) is None and found["default"][
|
|
357
586
|
"batchDigest"
|
|
358
587
|
] != default_batch_digest(world)
|
|
359
588
|
for name in scenario_dirs(world):
|
|
360
|
-
snapshot = scenario_snapshot(world, name)
|
|
589
|
+
snapshot = scenario_snapshot(world, name, contract)
|
|
361
590
|
if snapshot.is_file() or sidecar_path(snapshot).is_file():
|
|
362
591
|
found["scenarios"][name] = verify(snapshot, contract)
|
|
363
592
|
return found
|
|
364
593
|
|
|
365
594
|
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
the legacy path (replay the JSON) still serves the world."""
|
|
373
|
-
if not snapshot.is_file():
|
|
595
|
+
def pinned_counts(world: str | Path, contract: Contract) -> dict[str, int] | None:
|
|
596
|
+
"""For validation: None for a tree whose own data stands (no pin, or the hosted snapshot
|
|
597
|
+
held beside it), else the hosted snapshot's entity counts. Either way a pin whose rows
|
|
598
|
+
were stored under another domain, or whose domain this tree cannot prove, is refused."""
|
|
599
|
+
pin = SnapshotPin.of(world)
|
|
600
|
+
if pin is None:
|
|
374
601
|
return None
|
|
375
|
-
|
|
376
|
-
if
|
|
602
|
+
pin.require_storage(contract)
|
|
603
|
+
if current_default(world, contract) is not None:
|
|
377
604
|
return None
|
|
378
|
-
return
|
|
605
|
+
return pin.counts(contract)
|
|
606
|
+
|
|
607
|
+
|
|
608
|
+
def migration_view(world: str | Path, contract: Contract) -> dict | None:
|
|
609
|
+
"""What the platform must migrate before a version of this tree can open its hosted
|
|
610
|
+
rows: the lock they were stored under and what the change adds; None when nothing."""
|
|
611
|
+
pin = SnapshotPin.of(world)
|
|
612
|
+
migration = None if pin is None else pin.migration(contract)
|
|
613
|
+
return None if migration is None else migration.view()
|
|
379
614
|
|
|
380
615
|
|
|
381
616
|
def current_default(world: str | Path, contract: Contract) -> dict | None:
|
|
382
|
-
"""The default snapshot's sidecar when it
|
|
383
|
-
|
|
617
|
+
"""The default snapshot's sidecar when it holds exactly the tree's data (or the pin's)
|
|
618
|
+
in rows this contract may open; None when absent or stale."""
|
|
384
619
|
world = Path(world)
|
|
385
|
-
snapshot = default_snapshot(world)
|
|
620
|
+
snapshot = default_snapshot(world, contract)
|
|
386
621
|
if not snapshot.is_file() or not (world / "data/initial.json").is_file():
|
|
387
622
|
return None
|
|
388
|
-
return
|
|
623
|
+
return holds(snapshot, contract, expected_default_digest(world))
|
|
389
624
|
|
|
390
625
|
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
626
|
+
@dataclass(frozen=True)
|
|
627
|
+
class ServedSnapshot:
|
|
628
|
+
"""What a served world opens: its contract, the tree checked against it, and the
|
|
629
|
+
sidecar of the default snapshot when that holds the tree's data (or the pin's) in
|
|
630
|
+
rows the contract may open, None on the legacy path. A pinned tree that holds no
|
|
631
|
+
snapshot is refused. Checking hashes the snapshot, so a version cache checks its
|
|
632
|
+
immutable tree once and hands the answer to every session of the version."""
|
|
397
633
|
|
|
634
|
+
contract: Contract
|
|
635
|
+
sidecar: dict | None
|
|
398
636
|
|
|
399
|
-
|
|
637
|
+
@classmethod
|
|
638
|
+
def of(cls, world: str | Path) -> "ServedSnapshot":
|
|
639
|
+
contract = check_bundle(world)
|
|
640
|
+
sidecar = current_default(world, contract)
|
|
641
|
+
if sidecar is None:
|
|
642
|
+
refuse_unheld_pin(world, contract)
|
|
643
|
+
return cls(contract, sidecar)
|
|
400
644
|
|
|
401
645
|
|
|
402
646
|
def fetch(world: str | Path, contract: Contract, url: str, sha256: str, digest: str) -> dict:
|
|
403
|
-
"""Download the pod's snapshot to
|
|
404
|
-
``.gz`` or the response says so, verify the sha256 of the file bytes,
|
|
405
|
-
|
|
647
|
+
"""Download the pod's snapshot to the default snapshot: gunzip when the URL ends in
|
|
648
|
+
``.gz`` or the response says so, verify the sha256 of the file bytes, open it under
|
|
649
|
+
the version's contract (refused when its rows were stored under another domain),
|
|
650
|
+
write the sidecar. A mismatch is a refusal naming the digest; nothing usable is left
|
|
651
|
+
behind."""
|
|
406
652
|
world = Path(world)
|
|
407
|
-
snapshot = default_snapshot(world)
|
|
653
|
+
snapshot = default_snapshot(world, contract)
|
|
408
654
|
snapshot.parent.mkdir(parents=True, exist_ok=True)
|
|
409
|
-
partial = snapshot.with_name(f".{
|
|
655
|
+
partial = snapshot.with_name(f".{snapshot.name}.download")
|
|
410
656
|
hasher = hashlib.sha256()
|
|
411
657
|
try:
|
|
412
658
|
with urllib.request.urlopen(urllib.request.Request(url), timeout=300) as response:
|
|
@@ -428,7 +674,7 @@ def fetch(world: str | Path, contract: Contract, url: str, sha256: str, digest:
|
|
|
428
674
|
f"Snapshot {digest} downloaded with sha256 {hasher.hexdigest()}, expected {sha256}",
|
|
429
675
|
)
|
|
430
676
|
try:
|
|
431
|
-
with
|
|
677
|
+
with WorldDatabase.open(partial, contract, audit=False) as store:
|
|
432
678
|
counts = store.counts()
|
|
433
679
|
except ContractError:
|
|
434
680
|
partial.unlink(missing_ok=True)
|
|
@@ -446,15 +692,18 @@ def fetch(world: str | Path, contract: Contract, url: str, sha256: str, digest:
|
|
|
446
692
|
|
|
447
693
|
__all__ = [
|
|
448
694
|
"EMPTY_DIGEST",
|
|
695
|
+
"PIN_REFUSALS",
|
|
449
696
|
"ENV_DIGEST",
|
|
450
697
|
"ENV_SHA256",
|
|
451
698
|
"ENV_URL",
|
|
452
699
|
"SIDECAR",
|
|
453
|
-
"
|
|
700
|
+
"ServedSnapshot",
|
|
701
|
+
"SnapshotPin",
|
|
454
702
|
"batch_digest",
|
|
455
703
|
"build_all",
|
|
456
704
|
"build_default",
|
|
457
705
|
"build_scenario",
|
|
706
|
+
"carry_held",
|
|
458
707
|
"check_all",
|
|
459
708
|
"contract_hash",
|
|
460
709
|
"current_default",
|
|
@@ -464,13 +713,20 @@ __all__ = [
|
|
|
464
713
|
"expected_default_digest",
|
|
465
714
|
"fetch",
|
|
466
715
|
"file_sha256",
|
|
467
|
-
"
|
|
468
|
-
"
|
|
716
|
+
"holds",
|
|
717
|
+
"lock_hash",
|
|
718
|
+
"migration_view",
|
|
719
|
+
"has_default_snapshot",
|
|
720
|
+
"build_names",
|
|
721
|
+
"snapshot_name",
|
|
722
|
+
"pinned_counts",
|
|
469
723
|
"read_sidecar",
|
|
724
|
+
"refuse_pinned_rows",
|
|
725
|
+
"refuse_unheld_pin",
|
|
470
726
|
"scenario_dirs",
|
|
471
727
|
"scenario_overlay",
|
|
472
728
|
"scenario_snapshot",
|
|
473
|
-
"
|
|
729
|
+
"storage_refusal",
|
|
474
730
|
"verify",
|
|
475
731
|
"write_sidecar",
|
|
476
732
|
]
|