@withgateway/sdk 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -6
- package/dist/cli.js +4372 -2265
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +4362 -2256
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +259 -58
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +249 -50
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
- package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
- package/dist/index.d.mts +5 -5
- package/dist/index.d.ts +5 -5
- package/dist/index.js +2240 -789
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2150 -739
- package/dist/index.mjs.map +1 -1
- package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
- package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
- package/dist/traces/index.d.mts +1 -1
- package/dist/traces/index.d.ts +1 -1
- package/dist/traces/index.js +253 -52
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +243 -44
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +2 -2
- package/dist/tracing/index.d.ts +2 -2
- package/dist/tracing/index.js +75 -13
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +75 -13
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +893 -482
- package/dist/worlds/index.d.ts +893 -482
- package/dist/worlds/index.js +1968 -669
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1960 -662
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +2 -27
- package/runtime/gatewaysdk/environment/_materialize.py +48 -88
- package/runtime/gatewaysdk/environment/_world.py +155 -237
- package/runtime/gatewaysdk/environment/backing.py +133 -0
- package/runtime/gatewaysdk/environment/core.py +2 -17
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
- package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
- package/runtime/gatewaysdk/environment/runtime.py +8 -13
- package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
- package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +347 -892
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
- package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
- package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +60 -216
- package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
- package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
- package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
- package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
- package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
- package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
- package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
- package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
- package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
- package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
- package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
- package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
- package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +58 -170
- package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
- package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
- package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
- package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
- package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
- package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
- package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
- package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
- package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
- package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
- package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
- package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
- package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
- package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
- package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
- package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
- package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
- package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
- package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
- package/runtime/gatewaysdk/environment/score.py +57 -0
- package/runtime/gatewaysdk/environment/secret_values.py +36 -0
- package/runtime/gatewaysdk/environment/world.py +43 -4
- package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/skills/connector-schema-authoring/SKILL.md +49 -16
- package/skills/connector-schema-authoring/connector-worlds.md +142 -11
- package/skills/connector-schema-authoring/world-schema.md +103 -51
- package/skills/world-data-ingestion/ROWS.md +39 -9
- package/skills/world-data-ingestion/SKILL.md +27 -27
- package/skills/worlds-getting-started/SKILL.md +33 -33
- package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
|
@@ -1,22 +1,31 @@
|
|
|
1
|
-
"""World data snapshots: the validated
|
|
1
|
+
"""World data snapshots: the validated world database file, built once and opened at start.
|
|
2
2
|
|
|
3
|
-
A snapshot is a
|
|
4
|
-
inside it, as always) plus a ``state.json``
|
|
3
|
+
A snapshot is a file the store wrote, named by its engine (``state.sqlite`` for
|
|
4
|
+
SQLite; contract lock and batch ledger inside it, as always) plus a ``state.json``
|
|
5
|
+
sidecar naming what it holds::
|
|
5
6
|
|
|
6
7
|
{"contractHash": ..., "batchDigest": ..., "entityCounts": {...},
|
|
7
8
|
"builtAt": ..., "runtime": "<gatewaysdk version>", "fileSha256": ...}
|
|
8
9
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
10
|
+
``batchDigest`` says which data it holds. A snapshot is valid for every contract with
|
|
11
|
+
the :class:`~.compiler.StorageContract` its rows were stored under, which the file
|
|
12
|
+
records and the store checks on every open: a tool, handler, route, test, package
|
|
13
|
+
version or documentation-file edit keeps it. A domain edit that only adds to the domain
|
|
14
|
+
(:class:`~.domain_migration.DomainMigration`: a new entity, a new field no row has to
|
|
15
|
+
carry, another ``domain_version``) carries it: compile migrates the snapshots the tree
|
|
16
|
+
holds, and keeps a pin's proof of the stored domain so the platform migrates the hosted
|
|
17
|
+
rows when it commits the change. Any other domain edit refuses it, naming the change.
|
|
18
|
+
``contractHash`` is the lock it was last built or verified under, a record rather than
|
|
19
|
+
the rule. ``db/<snapshot name>`` is the default snapshot (``data/initial.json`` applied,
|
|
20
|
+
plus any ``data/batches/*.json``); ``scenarios/<name>/<snapshot name>`` is the default
|
|
21
|
+
with that scenario's ``seed/*.json`` overlays replacing whole entities. ``compile``
|
|
22
|
+
builds them (one that already holds the same data under the same storage contract is
|
|
23
|
+
kept), ``check`` verifies them, the served world copies one to its temp dir and opens it
|
|
24
|
+
instead of replaying the JSON.
|
|
15
25
|
|
|
16
26
|
Digests (the contract, section 0):
|
|
17
27
|
|
|
18
|
-
* contract hash: sha256 of the canonical contract lock
|
|
19
|
-
``_check_contract`` compares;
|
|
28
|
+
* contract hash: sha256 of the canonical contract lock;
|
|
20
29
|
* batch digest: ``sha256(parent + "\\n" + batchId + "\\n" + "\\n".join(sorted(chunk digests)))``,
|
|
21
30
|
the empty world's parent being sixty-four zeros;
|
|
22
31
|
* the default batch's chunks are the tree's data files (``data/initial.json``
|
|
@@ -24,11 +33,13 @@ Digests (the contract, section 0):
|
|
|
24
33
|
is ``scenario:<name>`` over the default with its seed files as chunks.
|
|
25
34
|
|
|
26
35
|
A tree the platform published as a data-only version carries ``db/snapshot.json``,
|
|
27
|
-
the **pin
|
|
28
|
-
replaces the data files as the default batch. The tree's ``data/*.json``
|
|
29
|
-
longer describes the world's data — only that snapshot does — so a pinned tree
|
|
30
|
-
never rebuilt from its data files; the platform hands the file
|
|
31
|
-
(``load_world_to_sandbox``, the pod's ``WORLD_SNAPSHOT_*``).
|
|
36
|
+
the **pin** (:class:`SnapshotPin`): the hosted snapshot's ``batchDigest`` (and sidecar
|
|
37
|
+
fields), which replaces the data files as the default batch. The tree's ``data/*.json``
|
|
38
|
+
then no longer describes the world's data — only that snapshot does — so a pinned tree
|
|
39
|
+
is never rebuilt from, nor served from, its data files; the platform hands the file
|
|
40
|
+
down (``load_world_to_sandbox``, the pod's ``WORLD_SNAPSHOT_*``). Where the file is not
|
|
41
|
+
handed down (a sandbox), the pin itself proves which domain the hosted rows were stored
|
|
42
|
+
under, so the contract still compiles and checks.
|
|
32
43
|
|
|
33
44
|
Everything here is standard library, so the api bundle's server can load it.
|
|
34
45
|
"""
|
|
@@ -38,18 +49,22 @@ from __future__ import annotations
|
|
|
38
49
|
import gzip
|
|
39
50
|
import hashlib
|
|
40
51
|
import json
|
|
52
|
+
import os
|
|
53
|
+
import re
|
|
41
54
|
import shutil
|
|
42
55
|
import urllib.request
|
|
56
|
+
import uuid
|
|
57
|
+
from dataclasses import dataclass
|
|
43
58
|
from datetime import datetime, timezone
|
|
44
59
|
from pathlib import Path
|
|
45
|
-
from typing import Any, Iterable
|
|
60
|
+
from typing import Any, Callable, Iterable
|
|
46
61
|
|
|
47
62
|
from .._toml import tomllib
|
|
48
|
-
from .compiler import Contract
|
|
49
|
-
from .
|
|
63
|
+
from .compiler import LOCK, Contract, check_bundle
|
|
64
|
+
from .database import WorldDatabase
|
|
65
|
+
from .domain_migration import DomainMigration
|
|
50
66
|
from .validation import ContractError, canonical, fail, read_json
|
|
51
67
|
|
|
52
|
-
SNAPSHOT = "state.sqlite"
|
|
53
68
|
SIDECAR = "state.json"
|
|
54
69
|
PIN = "snapshot.json"
|
|
55
70
|
DEFAULT_DIR = "db"
|
|
@@ -60,14 +75,24 @@ SIDECAR_KEYS = ("contractHash", "batchDigest", "entityCounts", "builtAt", "runti
|
|
|
60
75
|
ENV_URL = "WORLD_SNAPSHOT_URL"
|
|
61
76
|
ENV_SHA256 = "WORLD_SNAPSHOT_SHA256"
|
|
62
77
|
ENV_DIGEST = "WORLD_SNAPSHOT_DIGEST"
|
|
78
|
+
#: Where a sandbox keeps the hosted snapshots its trees pin, outside every tree (``<root>/<batchDigest>/``).
|
|
79
|
+
STORE_ENV = "GATEWAY_SNAPSHOT_STORE"
|
|
80
|
+
_BATCH_DIGEST = re.compile(r"[0-9a-f]{64}")
|
|
81
|
+
PIN_POINTER = f"/{DEFAULT_DIR}/{PIN}"
|
|
82
|
+
#: The codes a pin is refused with: never a data refusal, so never discarded as one.
|
|
83
|
+
PIN_REFUSALS = ("snapshot_pin", "snapshot_pinned")
|
|
84
|
+
REIMPORT = f"remove {DEFAULT_DIR}/{PIN} and import every row again (`gateway worlds data import <slug> <rows>`)"
|
|
85
|
+
REBUILD_REMEDY = "recompile to rebuild it from the data files, or restore the domain it was built under"
|
|
86
|
+
PINNED_REMEDY = "recompile to carry it to this contract, or restore the domain it was built under"
|
|
63
87
|
_BLOCK = 1024 * 1024
|
|
64
88
|
|
|
65
89
|
|
|
66
|
-
|
|
90
|
+
def lock_hash(lock: Any) -> str:
|
|
91
|
+
return hashlib.sha256(canonical(lock).encode("utf-8")).hexdigest()
|
|
67
92
|
|
|
68
93
|
|
|
69
94
|
def contract_hash(contract: Contract) -> str:
|
|
70
|
-
return
|
|
95
|
+
return lock_hash(contract.lock)
|
|
71
96
|
|
|
72
97
|
|
|
73
98
|
def batch_digest(parent: str, batch_id: str, chunk_digests: Iterable[str]) -> str:
|
|
@@ -113,25 +138,247 @@ def pin_path(world: str | Path) -> Path:
|
|
|
113
138
|
return Path(world) / DEFAULT_DIR / PIN
|
|
114
139
|
|
|
115
140
|
|
|
116
|
-
def
|
|
117
|
-
"""
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
pin = read_json(path)
|
|
123
|
-
if not isinstance(pin, dict) or not isinstance(pin.get("batchDigest"), str) or len(pin["batchDigest"]) != 64:
|
|
141
|
+
def refuse_unheld_pin(world: str | Path, contract: Contract) -> None:
|
|
142
|
+
"""A pinned tree that does not hold its hosted snapshot, and finds no copy of it in the
|
|
143
|
+
snapshot store, has no rows here: nothing is built, copied or served from its data files
|
|
144
|
+
in their place."""
|
|
145
|
+
pin = SnapshotPin.of(world)
|
|
146
|
+
if pin is not None:
|
|
124
147
|
fail(
|
|
125
|
-
"
|
|
148
|
+
"snapshot_pinned",
|
|
149
|
+
f"{DEFAULT_DIR}/{PIN} pins the hosted snapshot {pin.batch_digest}, which this tree does not hold"
|
|
150
|
+
+ SnapshotStore.configured().remedy(pin),
|
|
151
|
+
f"/{DEFAULT_DIR}/{snapshot_name(contract)}",
|
|
126
152
|
)
|
|
127
|
-
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def refuse_pinned_rows(world: str | Path) -> None:
|
|
156
|
+
"""Rows written into a pinned tree's data files would never be served: the hosted
|
|
157
|
+
snapshot, not the data files, is the world's data."""
|
|
158
|
+
if SnapshotPin.of(world) is not None:
|
|
159
|
+
fail(
|
|
160
|
+
"snapshot_pinned",
|
|
161
|
+
"This world's data is a hosted snapshot (db/snapshot.json), which its data files "
|
|
162
|
+
"cannot reproduce: import rows through the platform (`gateway worlds data import "
|
|
163
|
+
"<slug>` or import_world_data) instead of into the checkout",
|
|
164
|
+
PIN_POINTER,
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
@dataclass(frozen=True)
|
|
169
|
+
class SnapshotPin:
|
|
170
|
+
"""``db/snapshot.json``: the hosted snapshot a data-only version opens in place of its
|
|
171
|
+
data files, and the lock that snapshot was built under — its ``contractHash``, plus the
|
|
172
|
+
``lock`` itself once a compile has moved ``schema/lock.json`` off it, so the pin keeps
|
|
173
|
+
proving which domain the hosted rows were stored under, and that domain's compiled
|
|
174
|
+
``entities`` once a change adds to it, so the platform can migrate the rows. The
|
|
175
|
+
platform binds ``contractHash`` to the snapshot it recorded; nothing here is trusted
|
|
176
|
+
beyond what hashes to it."""
|
|
177
|
+
|
|
178
|
+
world: Path
|
|
179
|
+
fields: dict
|
|
180
|
+
|
|
181
|
+
@classmethod
|
|
182
|
+
def of(cls, world: str | Path) -> "SnapshotPin | None":
|
|
183
|
+
path = pin_path(world)
|
|
184
|
+
if not path.is_file():
|
|
185
|
+
return None
|
|
186
|
+
fields = read_json(path)
|
|
187
|
+
digest = fields.get("batchDigest") if isinstance(fields, dict) else None
|
|
188
|
+
if not isinstance(digest, str) or len(digest) != 64:
|
|
189
|
+
fail("snapshot_pin", f"{DEFAULT_DIR}/{PIN} must name the hosted snapshot's batchDigest", PIN_POINTER)
|
|
190
|
+
return cls(Path(world), fields)
|
|
191
|
+
|
|
192
|
+
@property
|
|
193
|
+
def batch_digest(self) -> str:
|
|
194
|
+
return self.fields["batchDigest"]
|
|
195
|
+
|
|
196
|
+
def held(self, contract: Contract) -> bool:
|
|
197
|
+
"""Whether the tree holds the pinned snapshot (the platform laid it beside the tree): its default
|
|
198
|
+
snapshot names this batch in the bytes its sidecar names. A pinned tree that does not hold it has
|
|
199
|
+
no rows of its own, so any snapshot in it was built from the data files the pin replaced."""
|
|
200
|
+
snapshot = default_snapshot(self.world, contract)
|
|
201
|
+
sidecar = _holding(snapshot, self.batch_digest)
|
|
202
|
+
return sidecar is not None and sidecar["fileSha256"] == file_sha256(snapshot)
|
|
203
|
+
|
|
204
|
+
def counts(self, contract: Contract) -> dict[str, int]:
|
|
205
|
+
"""The hosted snapshot's entity counts as the platform recorded them, per entity."""
|
|
206
|
+
counts = self.fields.get("entityCounts")
|
|
207
|
+
if not isinstance(counts, dict) or any(
|
|
208
|
+
not isinstance(value, int) or isinstance(value, bool) for value in counts.values()
|
|
209
|
+
):
|
|
210
|
+
fail("snapshot_pin", f"{DEFAULT_DIR}/{PIN} must carry the hosted snapshot's entityCounts", PIN_POINTER)
|
|
211
|
+
return {entity: int(counts.get(entity, 0)) for entity in contract.entities}
|
|
212
|
+
|
|
213
|
+
def view(self, contract: Contract) -> dict:
|
|
214
|
+
"""What compile reports for a pinned tree that does not hold its hosted snapshot."""
|
|
215
|
+
migration = self.migration(contract)
|
|
216
|
+
shown = {
|
|
217
|
+
"pinned": True,
|
|
218
|
+
"batchDigest": self.batch_digest,
|
|
219
|
+
"contractHash": self.fields.get("contractHash"),
|
|
220
|
+
"entityCounts": self.counts(contract),
|
|
221
|
+
}
|
|
222
|
+
return shown if migration is None else {**shown, "migrates": migration.view()}
|
|
223
|
+
|
|
224
|
+
def require_storage(self, contract: Contract) -> None:
|
|
225
|
+
"""Refuse ``contract`` unless the hosted rows carry over to it."""
|
|
226
|
+
self.migration(contract)
|
|
227
|
+
|
|
228
|
+
def migration(self, contract: Contract) -> DomainMigration | None:
|
|
229
|
+
"""How the hosted rows move to ``contract``: None when they open under it as they are,
|
|
230
|
+
refused when they cannot carry over or this tree cannot prove the domain they were
|
|
231
|
+
stored under."""
|
|
232
|
+
if self.fields.get("contractHash") == contract_hash(contract):
|
|
233
|
+
return None
|
|
234
|
+
migration = DomainMigration.of_lock(self._built_under(), contract, DomainMigration.recorded(self.world))
|
|
235
|
+
migration.require("snapshot_pinned", PIN_POINTER, REIMPORT)
|
|
236
|
+
return migration if migration.needed else None
|
|
237
|
+
|
|
238
|
+
def carried(self, contract: Contract) -> "SnapshotPin | None":
|
|
239
|
+
"""The pin as compile writes it for ``contract``: the lock the hosted rows were stored
|
|
240
|
+
under while the tree's own lock is another, and that domain's entities while the rows
|
|
241
|
+
await a migration. ``None`` when the pin stays as it is."""
|
|
242
|
+
migration = self.migration(contract)
|
|
243
|
+
kept = {key: value for key, value in self.fields.items() if key not in ("lock", "entities")}
|
|
244
|
+
if self.fields.get("contractHash") != contract_hash(contract):
|
|
245
|
+
self.counts(contract)
|
|
246
|
+
kept["lock"] = self._built_under()
|
|
247
|
+
if migration is not None:
|
|
248
|
+
kept["entities"] = migration.entities
|
|
249
|
+
return None if kept == self.fields else SnapshotPin(self.world, kept)
|
|
250
|
+
|
|
251
|
+
def write(self) -> None:
|
|
252
|
+
text = json.dumps(self.fields, indent=2, ensure_ascii=False) + "\n"
|
|
253
|
+
pin_path(self.world).write_text(text, encoding="utf-8")
|
|
254
|
+
|
|
255
|
+
def _built_under(self) -> dict:
|
|
256
|
+
lock = self._proven_lock()
|
|
257
|
+
if lock is not None:
|
|
258
|
+
return lock
|
|
259
|
+
named = str(self.fields.get("contractHash"))[:12]
|
|
260
|
+
fail(
|
|
261
|
+
"snapshot_pinned",
|
|
262
|
+
f"{DEFAULT_DIR}/{PIN} names a snapshot built under a different contract ({named}) and "
|
|
263
|
+
f"{LOCK} is no longer that contract, so this tree cannot prove the domain is unchanged. "
|
|
264
|
+
f"Restore {LOCK} as the version you started from has it (a fresh `gateway worlds sandbox "
|
|
265
|
+
f"open <slug>`, or `gateway bench pull <slug>`) and compile again, or {REIMPORT}",
|
|
266
|
+
PIN_POINTER,
|
|
267
|
+
)
|
|
268
|
+
|
|
269
|
+
def _proven_lock(self) -> dict | None:
|
|
270
|
+
"""The lock the hosted snapshot was built under, when this tree proves it: the pin's
|
|
271
|
+
own ``lock`` (refused unless it hashes to ``contractHash``), else ``schema/lock.json``
|
|
272
|
+
while it still is that lock."""
|
|
273
|
+
if "lock" in self.fields:
|
|
274
|
+
return self._authentic(self.fields["lock"])
|
|
275
|
+
on_disk = self._tree_lock()
|
|
276
|
+
if on_disk is None or lock_hash(on_disk) != self.fields.get("contractHash"):
|
|
277
|
+
return None
|
|
278
|
+
return on_disk
|
|
279
|
+
|
|
280
|
+
def _authentic(self, lock: Any) -> dict:
|
|
281
|
+
if not isinstance(lock, dict) or lock_hash(lock) != self.fields.get("contractHash"):
|
|
282
|
+
fail(
|
|
283
|
+
"snapshot_pin",
|
|
284
|
+
f"The lock in {DEFAULT_DIR}/{PIN} is not the contract its contractHash names: "
|
|
285
|
+
"restore the pin the version came with",
|
|
286
|
+
PIN_POINTER,
|
|
287
|
+
)
|
|
288
|
+
return lock
|
|
289
|
+
|
|
290
|
+
def _tree_lock(self) -> Any:
|
|
291
|
+
try:
|
|
292
|
+
return read_json(self.world / LOCK)
|
|
293
|
+
except ContractError:
|
|
294
|
+
return None
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
@dataclass(frozen=True)
|
|
298
|
+
class SnapshotStore:
|
|
299
|
+
"""The hosted snapshots a sandbox keeps beside its trees, never in one (``$GATEWAY_SNAPSHOT_STORE``,
|
|
300
|
+
``<root>/<batchDigest>/<snapshot name>`` with its sidecar), where the platform lays the one a
|
|
301
|
+
sandbox's version serves. A pinned tree that holds no snapshot of its own reads its rows from the
|
|
302
|
+
copy of its batch; ``root`` is None where no store is configured or its directory does not exist
|
|
303
|
+
(a sandbox the platform lays no snapshot into)."""
|
|
304
|
+
|
|
305
|
+
root: Path | None
|
|
306
|
+
|
|
307
|
+
@classmethod
|
|
308
|
+
def configured(cls) -> "SnapshotStore":
|
|
309
|
+
root = os.environ.get(STORE_ENV)
|
|
310
|
+
return cls(Path(root) if root and Path(root).is_dir() else None)
|
|
311
|
+
|
|
312
|
+
def held(self, pin: SnapshotPin, contract: Contract) -> tuple[Path, dict] | None:
|
|
313
|
+
"""The copy of the pinned batch and its sidecar, opened under ``contract``: the copy itself when
|
|
314
|
+
its rows were stored under this contract's storage, else the copy carried to it (kept beside the
|
|
315
|
+
original) when the contract only adds to their domain. None without a copy that is the bytes its
|
|
316
|
+
sidecar names and, when the pin names them, the pin's own bytes."""
|
|
317
|
+
if self.root is None or not _BATCH_DIGEST.fullmatch(pin.batch_digest):
|
|
318
|
+
return None
|
|
319
|
+
copy = self.root / pin.batch_digest / snapshot_name(contract)
|
|
320
|
+
sidecar = _readable(lambda: _holding(copy, pin.batch_digest))
|
|
321
|
+
if sidecar is None or sidecar["fileSha256"] != file_sha256(copy):
|
|
322
|
+
return None
|
|
323
|
+
if sidecar["fileSha256"] != pin.fields.get("fileSha256", sidecar["fileSha256"]):
|
|
324
|
+
return None
|
|
325
|
+
if storage_refusal(copy, contract) is None:
|
|
326
|
+
return copy, sidecar
|
|
327
|
+
return self._carried(copy, pin, contract)
|
|
328
|
+
|
|
329
|
+
def remedy(self, pin: SnapshotPin) -> str:
|
|
330
|
+
"""How a refusal of a pinned snapshot neither the tree nor this store holds ends."""
|
|
331
|
+
if self.root is None:
|
|
332
|
+
return (
|
|
333
|
+
": run the world on the platform (`gateway worlds session open <slug>`, load_world_to_sandbox), "
|
|
334
|
+
f"which serves that snapshot, or remove {DEFAULT_DIR}/{PIN} to rebuild from the data files"
|
|
335
|
+
)
|
|
336
|
+
return (
|
|
337
|
+
f", nor does this sandbox hold a whole copy at {self.root / pin.batch_digest}: the platform lays there, "
|
|
338
|
+
"before each `gateway worlds sandbox exec`, the snapshot this sandbox's version serves (that exec's "
|
|
339
|
+
f"warnings say why it could not). Restore {DEFAULT_DIR}/{PIN} as the version has it if it was edited, or "
|
|
340
|
+
"remove that directory and exec again; meanwhile `gateway worlds data sql <slug>` reads the rows from your "
|
|
341
|
+
"machine"
|
|
342
|
+
)
|
|
343
|
+
|
|
344
|
+
def _carried(self, copy: Path, pin: SnapshotPin, contract: Contract) -> tuple[Path, dict]:
|
|
345
|
+
carried = copy.parent / contract_hash(contract) / copy.name
|
|
346
|
+
kept = _readable(lambda: holds(carried, contract, pin.batch_digest))
|
|
347
|
+
if kept is not None:
|
|
348
|
+
return carried, kept
|
|
349
|
+
for other in copy.parent.iterdir():
|
|
350
|
+
if other.is_dir() and other.name != carried.parent.name:
|
|
351
|
+
shutil.rmtree(other, ignore_errors=True)
|
|
352
|
+
fresh = carried.with_name(f".{carried.name}.{uuid.uuid4().hex}")
|
|
353
|
+
fresh.parent.mkdir(parents=True, exist_ok=True)
|
|
354
|
+
WorldDatabase.clone_file(copy, fresh)
|
|
355
|
+
try:
|
|
356
|
+
DomainMigration.carry(fresh, contract, pin.world)
|
|
357
|
+
with WorldDatabase.open(fresh, contract, audit=False) as store:
|
|
358
|
+
counts = store.counts()
|
|
359
|
+
except BaseException:
|
|
360
|
+
fresh.unlink(missing_ok=True)
|
|
361
|
+
raise
|
|
362
|
+
_replace(carried, fresh)
|
|
363
|
+
return carried, write_sidecar(carried, contract_hash(contract), pin.batch_digest, counts)
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def _readable(read: Callable[[], dict | None]) -> dict | None:
|
|
367
|
+
"""``read()``, or None when the sidecar it reads cannot be read: a store copy is the platform's to lay
|
|
368
|
+
again and a carried copy the store's to carry again, so an unreadable sidecar there is a missing copy."""
|
|
369
|
+
try:
|
|
370
|
+
return read()
|
|
371
|
+
except ContractError as error:
|
|
372
|
+
if error.detail.get("code") != "snapshot":
|
|
373
|
+
raise
|
|
374
|
+
return None
|
|
128
375
|
|
|
129
376
|
|
|
130
377
|
def expected_default_digest(world: str | Path) -> str:
|
|
131
378
|
"""The batch digest the default snapshot must carry: the pin's for a data-only
|
|
132
379
|
version, the data files' for an authored tree."""
|
|
133
|
-
pin =
|
|
134
|
-
return pin
|
|
380
|
+
pin = SnapshotPin.of(world)
|
|
381
|
+
return pin.batch_digest if pin is not None else default_batch_digest(world)
|
|
135
382
|
|
|
136
383
|
|
|
137
384
|
def scenario_dirs(world: str | Path) -> dict[str, Path]:
|
|
@@ -179,13 +426,16 @@ def scenario_batch_digest(scenario: Path, default_digest: str) -> str:
|
|
|
179
426
|
)
|
|
180
427
|
|
|
181
428
|
|
|
182
|
-
# ---------------------------------------------------------------- sidecars
|
|
183
|
-
|
|
184
|
-
|
|
185
429
|
def sidecar_path(snapshot: Path) -> Path:
|
|
186
430
|
return snapshot.with_name(SIDECAR)
|
|
187
431
|
|
|
188
432
|
|
|
433
|
+
def _shown(path: Path) -> str:
|
|
434
|
+
"""A snapshot file as a refusal names it: inside the world, never where the runtime
|
|
435
|
+
keeps it (a server's temporary directory)."""
|
|
436
|
+
return f"{path.parent.name}/{path.name}"
|
|
437
|
+
|
|
438
|
+
|
|
189
439
|
def read_sidecar(snapshot: Path) -> dict | None:
|
|
190
440
|
path = sidecar_path(snapshot)
|
|
191
441
|
if not path.is_file():
|
|
@@ -193,9 +443,10 @@ def read_sidecar(snapshot: Path) -> dict | None:
|
|
|
193
443
|
try:
|
|
194
444
|
sidecar = json.loads(path.read_text(encoding="utf-8"))
|
|
195
445
|
except (OSError, ValueError) as error:
|
|
196
|
-
|
|
446
|
+
shown = _shown(path)
|
|
447
|
+
raise ContractError("snapshot", f"Cannot read the snapshot sidecar {shown}", schema_path=shown) from error
|
|
197
448
|
if not isinstance(sidecar, dict) or any(key not in sidecar for key in SIDECAR_KEYS):
|
|
198
|
-
fail("snapshot", f"The snapshot sidecar {path
|
|
449
|
+
fail("snapshot", f"The snapshot sidecar {_shown(path)} lacks {', '.join(SIDECAR_KEYS)}", _shown(path))
|
|
199
450
|
return sidecar
|
|
200
451
|
|
|
201
452
|
|
|
@@ -212,39 +463,92 @@ def write_sidecar(snapshot: Path, contract_digest: str, digest: str, counts: dic
|
|
|
212
463
|
return sidecar
|
|
213
464
|
|
|
214
465
|
|
|
215
|
-
def
|
|
216
|
-
"""The sidecar
|
|
466
|
+
def _verified_for(snapshot: Path, sidecar: dict, contract: Contract) -> dict:
|
|
467
|
+
"""The sidecar, naming ``contract`` as the lock this snapshot was last verified under."""
|
|
468
|
+
if sidecar["contractHash"] == contract_hash(contract):
|
|
469
|
+
return sidecar
|
|
470
|
+
verified = {**sidecar, "contractHash": contract_hash(contract)}
|
|
471
|
+
sidecar_path(snapshot).write_text(json.dumps(verified, indent=1) + "\n", encoding="utf-8")
|
|
472
|
+
return verified
|
|
473
|
+
|
|
474
|
+
|
|
475
|
+
def _holding(snapshot: Path, digest: str) -> dict | None:
|
|
217
476
|
if not snapshot.is_file():
|
|
218
477
|
return None
|
|
219
478
|
sidecar = read_sidecar(snapshot)
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
479
|
+
return sidecar if sidecar is not None and sidecar["batchDigest"] == digest else None
|
|
480
|
+
|
|
481
|
+
|
|
482
|
+
def holds(snapshot: Path, contract: Contract, digest: str) -> dict | None:
|
|
483
|
+
"""The sidecar when this snapshot holds batch ``digest``, its bytes are the ones the
|
|
484
|
+
sidecar names, and ``contract`` may open its rows (stored under its storage contract),
|
|
485
|
+
whatever lock the sidecar names: the one validity rule, which :func:`verify` explains.
|
|
486
|
+
The bytes are checked before the file is opened."""
|
|
487
|
+
sidecar = _holding(snapshot, digest)
|
|
488
|
+
if sidecar is None or sidecar["fileSha256"] != file_sha256(snapshot):
|
|
489
|
+
return None
|
|
490
|
+
return sidecar if storage_refusal(snapshot, contract) is None else None
|
|
223
491
|
|
|
224
492
|
|
|
225
|
-
def
|
|
226
|
-
"""
|
|
493
|
+
def storage_refusal(snapshot: Path, contract: Contract) -> ContractError | None:
|
|
494
|
+
"""Why ``contract`` may not open the rows in ``snapshot`` (the store's own check,
|
|
495
|
+
without the row audit), or None when it may."""
|
|
496
|
+
engine = WorldDatabase.for_contract(contract)
|
|
497
|
+
try:
|
|
498
|
+
with engine.open(snapshot, contract, audit=False):
|
|
499
|
+
return None
|
|
500
|
+
except engine.native_error as error:
|
|
501
|
+
return ContractError("state_contract", f"{snapshot.name} is not a world store ({error})")
|
|
502
|
+
except ContractError as error:
|
|
503
|
+
if error.detail["code"] != "state_contract":
|
|
504
|
+
raise
|
|
505
|
+
return error
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
def verify(snapshot: Path, contract: Contract, remedy: str = REBUILD_REMEDY) -> dict:
|
|
509
|
+
"""The sidecar of a snapshot whose bytes it names and whose rows ``contract`` may
|
|
510
|
+
open: anything else is refused, saying what a recompile does about it."""
|
|
511
|
+
shown = _shown(snapshot)
|
|
227
512
|
sidecar = read_sidecar(snapshot)
|
|
228
513
|
if sidecar is None:
|
|
229
|
-
fail("snapshot", f"{
|
|
230
|
-
if sidecar["contractHash"] != contract_hash(contract):
|
|
231
|
-
fail("snapshot", f"{snapshot} was built for a different contract; recompile", str(snapshot))
|
|
514
|
+
fail("snapshot", f"{shown} has no {SIDECAR} sidecar; recompile", shown)
|
|
232
515
|
if not snapshot.is_file():
|
|
233
|
-
fail("snapshot", f"{
|
|
516
|
+
fail("snapshot", f"{shown} is missing; recompile", shown)
|
|
234
517
|
if sidecar["fileSha256"] != file_sha256(snapshot):
|
|
235
|
-
fail("snapshot", f"{
|
|
518
|
+
fail("snapshot", f"{shown} differs from its sidecar's fileSha256; recompile", shown)
|
|
519
|
+
refusal = storage_refusal(snapshot, contract)
|
|
520
|
+
if refusal is not None:
|
|
521
|
+
fail(
|
|
522
|
+
"snapshot",
|
|
523
|
+
f"{shown} holds rows stored under a different contract ({refusal.detail['message']}); {remedy}",
|
|
524
|
+
shown,
|
|
525
|
+
changed=refusal.detail.get("changed", []),
|
|
526
|
+
)
|
|
236
527
|
return sidecar
|
|
237
528
|
|
|
238
529
|
|
|
239
|
-
|
|
530
|
+
def snapshot_name(contract: Contract) -> str:
|
|
531
|
+
"""The file name the contract's engine writes a snapshot to."""
|
|
532
|
+
return WorldDatabase.for_contract(contract).snapshot_name
|
|
240
533
|
|
|
241
534
|
|
|
242
|
-
def default_snapshot(world: str | Path) -> Path:
|
|
243
|
-
return Path(world) / DEFAULT_DIR /
|
|
535
|
+
def default_snapshot(world: str | Path, contract: Contract) -> Path:
|
|
536
|
+
return Path(world) / DEFAULT_DIR / snapshot_name(contract)
|
|
244
537
|
|
|
245
538
|
|
|
246
|
-
def scenario_snapshot(world: str | Path, name: str) -> Path:
|
|
247
|
-
return Path(world) / SCENARIOS / name /
|
|
539
|
+
def scenario_snapshot(world: str | Path, name: str, contract: Contract) -> Path:
|
|
540
|
+
return Path(world) / SCENARIOS / name / snapshot_name(contract)
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
def has_default_snapshot(world: str | Path) -> bool:
|
|
544
|
+
"""Whether any engine's default snapshot is in the tree."""
|
|
545
|
+
return any((Path(world) / DEFAULT_DIR / engine.snapshot_name).is_file() for engine in WorldDatabase.engines())
|
|
546
|
+
|
|
547
|
+
|
|
548
|
+
def build_names() -> tuple[str, ...]:
|
|
549
|
+
"""Every engine's snapshot and sidecar names and their in-progress files: never authored, never copied."""
|
|
550
|
+
names = [engine.snapshot_name for engine in WorldDatabase.engines()]
|
|
551
|
+
return (*names, SIDECAR, *(f".{name}.*" for name in names))
|
|
248
552
|
|
|
249
553
|
|
|
250
554
|
def _replace(snapshot: Path, fresh: Path) -> None:
|
|
@@ -255,32 +559,23 @@ def _replace(snapshot: Path, fresh: Path) -> None:
|
|
|
255
559
|
|
|
256
560
|
|
|
257
561
|
def build_default(world: str | Path, contract: Contract, *, force: bool = False) -> dict:
|
|
258
|
-
"""``db/state.sqlite`` for the tree's data files; a snapshot already
|
|
259
|
-
kept. A pinned tree's default is the hosted snapshot,
|
|
260
|
-
reproduce: it is kept when
|
|
562
|
+
"""``db/state.sqlite`` for the tree's data files; a snapshot already holding them in
|
|
563
|
+
rows this contract may open is kept. A pinned tree's default is the hosted snapshot,
|
|
564
|
+
which the data files cannot reproduce: it is kept when held and refused when not."""
|
|
261
565
|
world = Path(world)
|
|
262
|
-
snapshot = default_snapshot(world)
|
|
263
|
-
|
|
264
|
-
digest =
|
|
265
|
-
kept =
|
|
266
|
-
if
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
f"run the world on the platform (`gateway worlds session open <slug>`, "
|
|
272
|
-
f"load_world_to_sandbox), which serves that snapshot, or remove {DEFAULT_DIR}/{PIN} "
|
|
273
|
-
f"to rebuild from the data files",
|
|
274
|
-
f"/{DEFAULT_DIR}/{SNAPSHOT}",
|
|
275
|
-
)
|
|
276
|
-
return kept
|
|
277
|
-
if not force and kept is not None:
|
|
278
|
-
return kept
|
|
279
|
-
fresh = snapshot.with_name(f".{SNAPSHOT}.building")
|
|
566
|
+
snapshot = default_snapshot(world, contract)
|
|
567
|
+
pin = SnapshotPin.of(world)
|
|
568
|
+
digest = pin.batch_digest if pin is not None else default_batch_digest(world)
|
|
569
|
+
kept = holds(snapshot, contract, digest)
|
|
570
|
+
if kept is None:
|
|
571
|
+
refuse_unheld_pin(world, contract)
|
|
572
|
+
if pin is not None or (kept is not None and not force):
|
|
573
|
+
return _verified_for(snapshot, kept, contract)
|
|
574
|
+
fresh = snapshot.with_name(f".{snapshot.name}.building")
|
|
280
575
|
fresh.parent.mkdir(parents=True, exist_ok=True)
|
|
281
576
|
fresh.unlink(missing_ok=True)
|
|
282
577
|
try:
|
|
283
|
-
with
|
|
578
|
+
with WorldDatabase.create(fresh, contract) as store:
|
|
284
579
|
for path in _data_files(world):
|
|
285
580
|
store.apply_batch(read_json(path))
|
|
286
581
|
counts = store.counts()
|
|
@@ -288,7 +583,7 @@ def build_default(world: str | Path, contract: Contract, *, force: bool = False)
|
|
|
288
583
|
fresh.unlink(missing_ok=True)
|
|
289
584
|
raise
|
|
290
585
|
_replace(snapshot, fresh)
|
|
291
|
-
return write_sidecar(snapshot,
|
|
586
|
+
return write_sidecar(snapshot, contract_hash(contract), digest, counts)
|
|
292
587
|
|
|
293
588
|
|
|
294
589
|
def build_scenario(world: str | Path, name: str, contract: Contract, default: dict, *, force: bool = False) -> dict:
|
|
@@ -297,20 +592,18 @@ def build_scenario(world: str | Path, name: str, contract: Contract, default: di
|
|
|
297
592
|
scenario = scenario_dirs(world).get(name)
|
|
298
593
|
if scenario is None:
|
|
299
594
|
fail("scenario", f"No scenario {name}", f"/{SCENARIOS}/{name}")
|
|
300
|
-
snapshot = scenario_snapshot(world, name)
|
|
301
|
-
contract_digest = contract_hash(contract)
|
|
595
|
+
snapshot = scenario_snapshot(world, name, contract)
|
|
302
596
|
digest = scenario_batch_digest(scenario, default["batchDigest"])
|
|
303
|
-
if
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
return kept
|
|
597
|
+
kept = None if force else holds(snapshot, contract, digest)
|
|
598
|
+
if kept is not None:
|
|
599
|
+
return _verified_for(snapshot, kept, contract)
|
|
307
600
|
from .data import seed_batch
|
|
308
601
|
|
|
309
|
-
fresh = snapshot.with_name(f".{
|
|
602
|
+
fresh = snapshot.with_name(f".{snapshot.name}.building")
|
|
310
603
|
fresh.unlink(missing_ok=True)
|
|
311
|
-
|
|
604
|
+
WorldDatabase.clone_file(default_snapshot(world, contract), fresh)
|
|
312
605
|
try:
|
|
313
|
-
with
|
|
606
|
+
with WorldDatabase.open(fresh, contract, audit=False) as store:
|
|
314
607
|
overlay = scenario_overlay(scenario)
|
|
315
608
|
if overlay:
|
|
316
609
|
store.apply_batch(seed_batch(store, contract, overlay, "replace", batch_id=f"scenario:{name}"))
|
|
@@ -319,94 +612,164 @@ def build_scenario(world: str | Path, name: str, contract: Contract, default: di
|
|
|
319
612
|
fresh.unlink(missing_ok=True)
|
|
320
613
|
raise
|
|
321
614
|
_replace(snapshot, fresh)
|
|
322
|
-
return write_sidecar(snapshot,
|
|
615
|
+
return write_sidecar(snapshot, contract_hash(contract), digest, counts)
|
|
323
616
|
|
|
324
617
|
|
|
325
618
|
def build_all(world: str | Path, contract: Contract, *, force: bool = False) -> dict:
|
|
326
|
-
"""The default snapshot and one per scenario; ``{"default": sidecar, "scenarios": {name: sidecar}}``.
|
|
619
|
+
"""The default snapshot and one per scenario; ``{"default": sidecar, "scenarios": {name: sidecar}}``.
|
|
620
|
+
A pinned tree that does not hold its hosted snapshot builds none (its data lives on the
|
|
621
|
+
platform) and keeps none: the snapshots its data files built claim data that no longer
|
|
622
|
+
applies. The default is the pin's view, once the pin proves the rows' domain."""
|
|
327
623
|
world = Path(world)
|
|
624
|
+
pin = SnapshotPin.of(world)
|
|
625
|
+
if pin is not None and not pin.held(contract):
|
|
626
|
+
view = pin.view(contract)
|
|
627
|
+
discard_all(world)
|
|
628
|
+
return {"default": view, "scenarios": {}}
|
|
629
|
+
if pin is not None:
|
|
630
|
+
carry_held(world, contract)
|
|
631
|
+
if pin is not None and holds(default_snapshot(world, contract), contract, pin.batch_digest) is None:
|
|
632
|
+
return {"default": pin.view(contract), "scenarios": {}}
|
|
328
633
|
default = build_default(world, contract, force=force)
|
|
329
634
|
scenarios = {name: build_scenario(world, name, contract, default, force=force) for name in scenario_dirs(world)}
|
|
330
635
|
return {"default": default, "scenarios": scenarios}
|
|
331
636
|
|
|
332
637
|
|
|
333
|
-
def
|
|
334
|
-
"""Remove every snapshot in the tree: nothing may claim data that no longer applies."""
|
|
335
|
-
world = Path(world)
|
|
336
|
-
targets = [default_snapshot(world)]
|
|
638
|
+
def _snapshot_paths(world: Path) -> list[Path]:
|
|
337
639
|
scenarios = world / SCENARIOS
|
|
640
|
+
directories = [world / DEFAULT_DIR]
|
|
338
641
|
if scenarios.is_dir():
|
|
339
|
-
|
|
340
|
-
for
|
|
642
|
+
directories.extend(path for path in scenarios.iterdir() if path.is_dir())
|
|
643
|
+
return [directory / engine.snapshot_name for directory in directories for engine in WorldDatabase.engines()]
|
|
644
|
+
|
|
645
|
+
|
|
646
|
+
def carry_held(world: str | Path, contract: Contract) -> None:
|
|
647
|
+
"""Every snapshot a pinned tree holds, carried to ``contract`` when its rows were stored
|
|
648
|
+
under a domain the contract only adds to, so ``check`` opens it as compile left it. A
|
|
649
|
+
snapshot the contract cannot take is left for ``check`` to refuse, naming why."""
|
|
650
|
+
world = Path(world)
|
|
651
|
+
engine = WorldDatabase.for_contract(contract)
|
|
652
|
+
for snapshot in _snapshot_paths(world):
|
|
653
|
+
if snapshot.name != engine.snapshot_name:
|
|
654
|
+
continue
|
|
655
|
+
sidecar = read_sidecar(snapshot) if snapshot.is_file() else None
|
|
656
|
+
if sidecar is None or sidecar["fileSha256"] != file_sha256(snapshot):
|
|
657
|
+
continue
|
|
658
|
+
if storage_refusal(snapshot, contract) is None:
|
|
659
|
+
continue
|
|
660
|
+
migration = DomainMigration.of_lock(engine.stored_lock(snapshot), contract, DomainMigration.recorded(world))
|
|
661
|
+
if not (migration.needed and migration.carries):
|
|
662
|
+
continue
|
|
663
|
+
counts = DomainMigration.carry(snapshot, contract, world)
|
|
664
|
+
write_sidecar(snapshot, contract_hash(contract), sidecar["batchDigest"], counts)
|
|
665
|
+
|
|
666
|
+
|
|
667
|
+
def discard_all(world: str | Path) -> None:
|
|
668
|
+
"""Remove every snapshot in the tree: nothing may claim data that no longer applies."""
|
|
669
|
+
for snapshot in _snapshot_paths(Path(world)):
|
|
341
670
|
snapshot.unlink(missing_ok=True)
|
|
342
671
|
sidecar_path(snapshot).unlink(missing_ok=True)
|
|
343
672
|
|
|
344
673
|
|
|
345
674
|
def check_all(world: str | Path, contract: Contract) -> dict:
|
|
346
|
-
"""Every snapshot present in the tree, verified against its sidecar; absent ones are not required.
|
|
675
|
+
"""Every snapshot present in the tree, verified against its sidecar; absent ones are not required.
|
|
676
|
+
A pinned tree that does not hold its hosted snapshot has none to verify: nothing opens the
|
|
677
|
+
snapshots its data files built (compile removes them), so they are never refused."""
|
|
347
678
|
world = Path(world)
|
|
348
679
|
found: dict[str, Any] = {"default": None, "scenarios": {}}
|
|
349
|
-
|
|
680
|
+
pin = SnapshotPin.of(world)
|
|
681
|
+
if pin is not None and not pin.held(contract):
|
|
682
|
+
return found
|
|
683
|
+
remedy = PINNED_REMEDY if pin is not None else REBUILD_REMEDY
|
|
684
|
+
snapshot = default_snapshot(world, contract)
|
|
350
685
|
if snapshot.is_file() or sidecar_path(snapshot).is_file():
|
|
351
|
-
found["default"] = verify(snapshot, contract)
|
|
352
|
-
#
|
|
353
|
-
|
|
354
|
-
# replays the journals past a stale snapshot (WorldState opens only a
|
|
355
|
-
# current one). `stale` is in every check report so it never passes unseen.
|
|
356
|
-
found["default"]["stale"] = pinned_default(world) is None and found["default"][
|
|
357
|
-
"batchDigest"
|
|
358
|
-
] != default_batch_digest(world)
|
|
686
|
+
found["default"] = verify(snapshot, contract, remedy)
|
|
687
|
+
# Reported, not refused: a served world replays the journals past a stale snapshot.
|
|
688
|
+
found["default"]["stale"] = pin is None and found["default"]["batchDigest"] != default_batch_digest(world)
|
|
359
689
|
for name in scenario_dirs(world):
|
|
360
|
-
snapshot = scenario_snapshot(world, name)
|
|
690
|
+
snapshot = scenario_snapshot(world, name, contract)
|
|
361
691
|
if snapshot.is_file() or sidecar_path(snapshot).is_file():
|
|
362
|
-
found["scenarios"][name] = verify(snapshot, contract)
|
|
692
|
+
found["scenarios"][name] = verify(snapshot, contract, remedy)
|
|
363
693
|
return found
|
|
364
694
|
|
|
365
695
|
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
the legacy path (replay the JSON) still serves the world."""
|
|
373
|
-
if not snapshot.is_file():
|
|
696
|
+
def pinned_counts(world: str | Path, contract: Contract) -> dict[str, int] | None:
|
|
697
|
+
"""For validation: None for a tree whose own data stands (no pin, or the hosted snapshot
|
|
698
|
+
held beside it), else the hosted snapshot's entity counts. Either way a pin whose rows
|
|
699
|
+
were stored under another domain, or whose domain this tree cannot prove, is refused."""
|
|
700
|
+
pin = SnapshotPin.of(world)
|
|
701
|
+
if pin is None:
|
|
374
702
|
return None
|
|
375
|
-
|
|
376
|
-
if
|
|
703
|
+
pin.require_storage(contract)
|
|
704
|
+
if current_default(world, contract) is not None:
|
|
377
705
|
return None
|
|
378
|
-
return
|
|
706
|
+
return pin.counts(contract)
|
|
707
|
+
|
|
708
|
+
|
|
709
|
+
def migration_view(world: str | Path, contract: Contract) -> dict | None:
|
|
710
|
+
"""What the platform must migrate before a version of this tree can open its hosted
|
|
711
|
+
rows: the lock they were stored under and what the change adds; None when nothing."""
|
|
712
|
+
pin = SnapshotPin.of(world)
|
|
713
|
+
migration = None if pin is None else pin.migration(contract)
|
|
714
|
+
return None if migration is None else migration.view()
|
|
379
715
|
|
|
380
716
|
|
|
381
717
|
def current_default(world: str | Path, contract: Contract) -> dict | None:
|
|
382
|
-
"""The default snapshot's sidecar when it
|
|
383
|
-
|
|
718
|
+
"""The default snapshot's sidecar when it holds exactly the tree's data (or the pin's)
|
|
719
|
+
in rows this contract may open; None when absent or stale."""
|
|
384
720
|
world = Path(world)
|
|
385
|
-
snapshot = default_snapshot(world)
|
|
721
|
+
snapshot = default_snapshot(world, contract)
|
|
386
722
|
if not snapshot.is_file() or not (world / "data/initial.json").is_file():
|
|
387
723
|
return None
|
|
388
|
-
return
|
|
724
|
+
return holds(snapshot, contract, expected_default_digest(world))
|
|
725
|
+
|
|
726
|
+
|
|
727
|
+
def held_default(world: str | Path, contract: Contract) -> tuple[Path, dict] | None:
|
|
728
|
+
"""The default snapshot a reader opens, and its sidecar: the tree's own when it holds the tree's
|
|
729
|
+
data (or the pin's), else, for a pinned tree, the snapshot store's copy of the pinned batch."""
|
|
730
|
+
world = Path(world)
|
|
731
|
+
sidecar = current_default(world, contract)
|
|
732
|
+
if sidecar is not None:
|
|
733
|
+
return default_snapshot(world, contract), sidecar
|
|
734
|
+
pin = SnapshotPin.of(world)
|
|
735
|
+
if pin is None:
|
|
736
|
+
return None
|
|
737
|
+
return SnapshotStore.configured().held(pin, contract)
|
|
389
738
|
|
|
390
739
|
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
740
|
+
@dataclass(frozen=True)
|
|
741
|
+
class ServedSnapshot:
|
|
742
|
+
"""What a served world opens: its contract, the tree checked against it, and the
|
|
743
|
+
default snapshot (``path`` and its ``sidecar``) when one holds the tree's data (or the
|
|
744
|
+
pin's) in rows the contract may open, None on the legacy path. A pinned tree that holds
|
|
745
|
+
no snapshot, and finds none in the snapshot store, is refused. Checking hashes the
|
|
746
|
+
snapshot, so a version cache checks its immutable tree once and hands the answer to
|
|
747
|
+
every session of the version."""
|
|
397
748
|
|
|
749
|
+
contract: Contract
|
|
750
|
+
sidecar: dict | None
|
|
751
|
+
path: Path | None = None
|
|
398
752
|
|
|
399
|
-
|
|
753
|
+
@classmethod
|
|
754
|
+
def of(cls, world: str | Path) -> "ServedSnapshot":
|
|
755
|
+
contract = check_bundle(world)
|
|
756
|
+
held = held_default(world, contract)
|
|
757
|
+
if held is None:
|
|
758
|
+
refuse_unheld_pin(world, contract)
|
|
759
|
+
return cls(contract, None)
|
|
760
|
+
return cls(contract, held[1], held[0])
|
|
400
761
|
|
|
401
762
|
|
|
402
763
|
def fetch(world: str | Path, contract: Contract, url: str, sha256: str, digest: str) -> dict:
|
|
403
|
-
"""Download the pod's snapshot to
|
|
404
|
-
``.gz`` or the response says so, verify the sha256 of the file bytes,
|
|
405
|
-
|
|
764
|
+
"""Download the pod's snapshot to the default snapshot: gunzip when the URL ends in
|
|
765
|
+
``.gz`` or the response says so, verify the sha256 of the file bytes, open it under
|
|
766
|
+
the version's contract (refused when its rows were stored under another domain),
|
|
767
|
+
write the sidecar. A mismatch is a refusal naming the digest; nothing usable is left
|
|
768
|
+
behind."""
|
|
406
769
|
world = Path(world)
|
|
407
|
-
snapshot = default_snapshot(world)
|
|
770
|
+
snapshot = default_snapshot(world, contract)
|
|
408
771
|
snapshot.parent.mkdir(parents=True, exist_ok=True)
|
|
409
|
-
partial = snapshot.with_name(f".{
|
|
772
|
+
partial = snapshot.with_name(f".{snapshot.name}.download")
|
|
410
773
|
hasher = hashlib.sha256()
|
|
411
774
|
try:
|
|
412
775
|
with urllib.request.urlopen(urllib.request.Request(url), timeout=300) as response:
|
|
@@ -428,7 +791,7 @@ def fetch(world: str | Path, contract: Contract, url: str, sha256: str, digest:
|
|
|
428
791
|
f"Snapshot {digest} downloaded with sha256 {hasher.hexdigest()}, expected {sha256}",
|
|
429
792
|
)
|
|
430
793
|
try:
|
|
431
|
-
with
|
|
794
|
+
with WorldDatabase.open(partial, contract, audit=False) as store:
|
|
432
795
|
counts = store.counts()
|
|
433
796
|
except ContractError:
|
|
434
797
|
partial.unlink(missing_ok=True)
|
|
@@ -446,15 +809,20 @@ def fetch(world: str | Path, contract: Contract, url: str, sha256: str, digest:
|
|
|
446
809
|
|
|
447
810
|
__all__ = [
|
|
448
811
|
"EMPTY_DIGEST",
|
|
812
|
+
"PIN_REFUSALS",
|
|
449
813
|
"ENV_DIGEST",
|
|
450
814
|
"ENV_SHA256",
|
|
451
815
|
"ENV_URL",
|
|
452
816
|
"SIDECAR",
|
|
453
|
-
"
|
|
817
|
+
"ServedSnapshot",
|
|
818
|
+
"SnapshotPin",
|
|
819
|
+
"SnapshotStore",
|
|
820
|
+
"STORE_ENV",
|
|
454
821
|
"batch_digest",
|
|
455
822
|
"build_all",
|
|
456
823
|
"build_default",
|
|
457
824
|
"build_scenario",
|
|
825
|
+
"carry_held",
|
|
458
826
|
"check_all",
|
|
459
827
|
"contract_hash",
|
|
460
828
|
"current_default",
|
|
@@ -464,13 +832,21 @@ __all__ = [
|
|
|
464
832
|
"expected_default_digest",
|
|
465
833
|
"fetch",
|
|
466
834
|
"file_sha256",
|
|
467
|
-
"
|
|
468
|
-
"
|
|
835
|
+
"held_default",
|
|
836
|
+
"holds",
|
|
837
|
+
"lock_hash",
|
|
838
|
+
"migration_view",
|
|
839
|
+
"has_default_snapshot",
|
|
840
|
+
"build_names",
|
|
841
|
+
"snapshot_name",
|
|
842
|
+
"pinned_counts",
|
|
469
843
|
"read_sidecar",
|
|
844
|
+
"refuse_pinned_rows",
|
|
845
|
+
"refuse_unheld_pin",
|
|
470
846
|
"scenario_dirs",
|
|
471
847
|
"scenario_overlay",
|
|
472
848
|
"scenario_snapshot",
|
|
473
|
-
"
|
|
849
|
+
"storage_refusal",
|
|
474
850
|
"verify",
|
|
475
851
|
"write_sidecar",
|
|
476
852
|
]
|