@withgateway/sdk 0.6.0 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/index.d.mts +581 -0
- package/dist/agents/index.d.ts +581 -0
- package/dist/agents/index.js +1473 -0
- package/dist/agents/index.js.map +1 -0
- package/dist/agents/index.mjs +1424 -0
- package/dist/agents/index.mjs.map +1 -0
- package/dist/apiError-DHpfbmdx.d.mts +17 -0
- package/dist/apiError-DHpfbmdx.d.ts +17 -0
- package/dist/cli.js +11611 -6845
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +10090 -5325
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +3 -1
- package/dist/files/index.d.ts +3 -1
- package/dist/files/index.js +371 -102
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +371 -102
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.js +346 -136
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +346 -136
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-LA-PMRJT.d.ts → index-DIN14Lhg.d.ts} +1 -1
- package/dist/{index-Di9nNqOY.d.mts → index-DuB-eQ0Z.d.mts} +1 -1
- package/dist/index.d.mts +7 -4
- package/dist/index.d.ts +7 -4
- package/dist/index.js +3177 -748
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2987 -590
- package/dist/index.mjs.map +1 -1
- package/dist/media-CQUC7TkS.d.mts +39 -0
- package/dist/media-DQnBloaX.d.ts +39 -0
- package/dist/traces/index.d.mts +3 -1
- package/dist/traces/index.d.ts +3 -1
- package/dist/traces/index.js +371 -102
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +371 -102
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +3 -2
- package/dist/tracing/index.d.ts +3 -2
- package/dist/tracing/index.js +257 -16
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +257 -16
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/transport-CpRe7zgd.d.mts +123 -0
- package/dist/transport-CpRe7zgd.d.ts +123 -0
- package/dist/worlds/index.d.mts +1986 -1746
- package/dist/worlds/index.d.ts +1986 -1746
- package/dist/worlds/index.js +1470 -290
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1465 -286
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +6 -1
- package/runtime/gatewaysdk/environment/bundle_manifest.py +531 -0
- package/runtime/gatewaysdk/environment/factory_documents.py +670 -0
- package/runtime/gatewaysdk/environment/factory_rules.py +973 -0
- package/runtime/gatewaysdk/environment/factory_shapes.py +253 -0
- package/runtime/gatewaysdk/environment/factory_spine.py +1155 -0
- package/runtime/gatewaysdk/environment/factory_spine_shapes.py +480 -0
- package/runtime/gatewaysdk/environment/schema/__main__.py +2 -2
- package/runtime/gatewaysdk/environment/schema/_slack_fidelity.py +8 -23
- package/runtime/gatewaysdk/environment/schema/admission.py +61 -59
- package/runtime/gatewaysdk/environment/schema/api.py +3 -4
- package/runtime/gatewaysdk/environment/schema/assertions.py +7 -1
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +20 -0
- package/runtime/gatewaysdk/environment/schema/cast.py +133 -67
- package/runtime/gatewaysdk/environment/schema/compiler.py +16 -17
- package/runtime/gatewaysdk/environment/schema/database/base.py +2 -2
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +58 -4
- package/runtime/gatewaysdk/environment/schema/host.py +94 -12
- package/runtime/gatewaysdk/environment/schema/host_browser.py +48 -0
- package/runtime/gatewaysdk/environment/schema/host_control.py +35 -2
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +4 -1
- package/runtime/gatewaysdk/environment/schema/host_loops.py +27 -4
- package/runtime/gatewaysdk/environment/schema/host_metrics.py +69 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +35 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +20 -2
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +100 -19
- package/runtime/gatewaysdk/environment/schema/host_workers.py +172 -50
- package/runtime/gatewaysdk/environment/schema/ingest.py +12 -3
- package/runtime/gatewaysdk/environment/schema/journal.py +7 -2
- package/runtime/gatewaysdk/environment/schema/local.py +32 -7
- package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
- package/runtime/gatewaysdk/environment/schema/measured_tests.py +187 -0
- package/runtime/gatewaysdk/environment/schema/page_checks.py +123 -0
- package/runtime/gatewaysdk/environment/schema/platform.py +80 -270
- package/runtime/gatewaysdk/environment/schema/platform_query.py +4 -3
- package/runtime/gatewaysdk/environment/schema/platform_templates.py +420 -0
- package/runtime/gatewaysdk/environment/schema/process_memory.py +76 -0
- package/runtime/gatewaysdk/environment/schema/redaction.py +13 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +4 -1
- package/runtime/gatewaysdk/environment/schema/run_parameters.py +25 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +57 -5
- package/runtime/gatewaysdk/environment/schema/scaffold.py +386 -219
- package/runtime/gatewaysdk/environment/schema/session_files.py +13 -2
- package/runtime/gatewaysdk/environment/schema/shape_catalog.py +493 -0
- package/runtime/gatewaysdk/environment/schema/shape_declarations.py +219 -0
- package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +1 -1
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +2 -2
- package/runtime/gatewaysdk/environment/schema/skills/factory-orchestrator/SKILL.md +147 -0
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +8 -6
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +9 -9
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +33 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +6 -6
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +17 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/new-world-sandbox.md +1 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/shape-catalog.md +59 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/ui-map.md +3 -2
- package/runtime/gatewaysdk/environment/schema/slack.py +55 -10
- package/runtime/gatewaysdk/environment/schema/snapshot.py +28 -2
- package/runtime/gatewaysdk/environment/schema/tasks.py +25 -2
- package/runtime/gatewaysdk/environment/schema/template_record.py +131 -0
- package/runtime/gatewaysdk/environment/schema/tools_world.py +7 -5
- package/runtime/gatewaysdk/environment/schema/ui.py +54 -33
- package/runtime/gatewaysdk/environment/schema/ui_interactions.py +419 -0
- package/runtime/gatewaysdk/environment/schema/ui_recorder.js +895 -0
- package/runtime/gatewaysdk/environment/schema/ui_replay.py +366 -0
- package/runtime/gatewaysdk/environment/schema/ui_replay_record.LICENSE +25 -0
- package/runtime/gatewaysdk/environment/schema/ui_replay_record.js +109 -0
- package/runtime/gatewaysdk/environment/schema/validation.py +24 -1
- package/runtime/gatewaysdk/environment/schema/worker.py +12 -5
- package/runtime/gatewaysdk/environment/schema/worker_key.py +38 -0
- package/runtime/gatewaysdk/environment/schema/worker_peaks.py +38 -0
- package/runtime/gatewaysdk/environment/schema/worker_watch.py +50 -0
- package/runtime/gatewaysdk/environment/schema/workers.py +245 -69
- package/runtime/gatewaysdk/environment/schema/world_host.py +96 -73
- package/runtime/gatewaysdk/environment/schema/world_tests.py +3 -1
- package/runtime/gatewaysdk/environment/secret_values.py +1 -1
- package/runtime/gatewaysdk/environment/world.py +34 -43
- package/runtime/gatewaysdk/environment/world_session_contract.py +7 -34
- package/skills/benchmarks-getting-started/SKILL.md +1 -1
- package/skills/connector-schema-authoring/SKILL.md +2 -2
- package/skills/connector-schema-authoring/connector-worlds.md +3 -2
- package/skills/connector-schema-authoring/world-schema.md +36 -23
- package/skills/factory-orchestrator/SKILL.md +147 -0
- package/skills/world-data-ingestion/ROWS.md +5 -1
- package/skills/world-data-ingestion/SKILL.md +8 -6
- package/skills/worlds-getting-started/SKILL.md +9 -9
- package/skills/worlds-getting-started/references/cli-mcp-map.md +33 -2
- package/skills/worlds-getting-started/references/gotchas.md +6 -6
- package/skills/worlds-getting-started/references/journal-checks.md +17 -2
- package/skills/worlds-getting-started/references/new-world-sandbox.md +1 -1
- package/skills/worlds-getting-started/references/shape-catalog.md +59 -0
- package/skills/worlds-getting-started/references/ui-map.md +3 -2
- package/dist/media-BIApiPzS.d.mts +0 -102
- package/dist/media-BIApiPzS.d.ts +0 -102
|
@@ -28,6 +28,37 @@ class DependencyRef:
|
|
|
28
28
|
snapshot: dict[str, str] | None
|
|
29
29
|
raw: dict[str, Any]
|
|
30
30
|
|
|
31
|
+
@property
|
|
32
|
+
def for_worker(self) -> dict[str, Any]:
|
|
33
|
+
"""The entry a version worker is handed: its snapshot without the url, a tree the supervisor built."""
|
|
34
|
+
return {**self.raw, "snapshot": worker_snapshot(self.snapshot)}
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def worker_snapshot(snapshot: dict[str, str] | None) -> dict[str, str] | None:
|
|
38
|
+
"""A snapshot as a version worker sees it: its sha256 and digest, never the presigned url."""
|
|
39
|
+
return {key: value for key, value in snapshot.items() if key != "url"} if snapshot else None
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
MAX_MEMORY_PEAK_MB = 1 << 20
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def parse_memory_peak(value: Any) -> int | None:
|
|
46
|
+
"""An open's ``memoryPeakMb``: the most memory the platform measured the session's world to need, a whole
|
|
47
|
+
MiB from 1 to 1 TiB; None when it has not measured one."""
|
|
48
|
+
if value is None:
|
|
49
|
+
return None
|
|
50
|
+
if isinstance(value, bool) or not isinstance(value, int) or not 0 < value <= MAX_MEMORY_PEAK_MB:
|
|
51
|
+
fail("host", "memoryPeakMb is a whole number of MiB from 1 to 1048576")
|
|
52
|
+
return value
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def parse_world_state(value: Any) -> bool | None:
|
|
56
|
+
"""An open's ``worldState``: whether the session is on one of its world's states, None when the open names
|
|
57
|
+
none (a platform from before a version's states had a worker of their own)."""
|
|
58
|
+
if value is None or isinstance(value, bool):
|
|
59
|
+
return value
|
|
60
|
+
fail("host", "worldState is true or false")
|
|
61
|
+
|
|
31
62
|
|
|
32
63
|
def parse_dependency_refs(dependencies: Any) -> list[DependencyRef]:
|
|
33
64
|
if dependencies is None:
|
|
@@ -77,6 +108,10 @@ class BrowserRelay:
|
|
|
77
108
|
raise AssertionError # fail() never returns
|
|
78
109
|
return cls(url.rstrip("/"), token)
|
|
79
110
|
|
|
111
|
+
@property
|
|
112
|
+
def http_url(self) -> str:
|
|
113
|
+
return "http://" + self.url.removeprefix("ws://")
|
|
114
|
+
|
|
80
115
|
|
|
81
116
|
def restore_sha256(restore: object) -> str | None:
|
|
82
117
|
"""The sha256 of the export an open restores from, or None for an open that restores nothing."""
|
|
@@ -10,9 +10,10 @@ from pathlib import Path
|
|
|
10
10
|
from typing import TYPE_CHECKING
|
|
11
11
|
|
|
12
12
|
from .door import OPEN_MODE, VENDOR_MODE, Door, same
|
|
13
|
-
from .host_surfaces import SessionHost, capability_host
|
|
13
|
+
from .host_surfaces import SessionHost, capability_host, session_host
|
|
14
14
|
from .request_ledger import LedgerLife
|
|
15
15
|
from .session_move import SessionMove
|
|
16
|
+
from .worker_key import WorkerKey
|
|
16
17
|
|
|
17
18
|
if TYPE_CHECKING:
|
|
18
19
|
from .api import ApiSettings, WorldApp
|
|
@@ -75,6 +76,8 @@ class Session:
|
|
|
75
76
|
placement: dict | None = None
|
|
76
77
|
settings: ApiSettings | None = None
|
|
77
78
|
restore_sha256: str | None = None
|
|
79
|
+
snapshot_sha256: str | None = None
|
|
80
|
+
parameters: dict[str, str] = field(default_factory=dict)
|
|
78
81
|
pending: InitVar[bool] = False
|
|
79
82
|
move: SessionMove = field(init=False, repr=False, compare=False)
|
|
80
83
|
auth: str = VENDOR_MODE
|
|
@@ -83,6 +86,7 @@ class Session:
|
|
|
83
86
|
recoveries: int = field(default=0, init=False)
|
|
84
87
|
close_reason: str = field(default="closed", init=False)
|
|
85
88
|
unrecovered: bool = field(default=False, init=False)
|
|
89
|
+
world_state: bool = False
|
|
86
90
|
|
|
87
91
|
def __post_init__(self, pending: bool) -> None:
|
|
88
92
|
self.move = SessionMove(self.session_id, pending=pending)
|
|
@@ -92,6 +96,11 @@ class Session:
|
|
|
92
96
|
world serves browsers: from the world's declared settings, never its worker."""
|
|
93
97
|
return None if self.settings is None else self.settings.cors.preflight(method, headers)
|
|
94
98
|
|
|
99
|
+
@property
|
|
100
|
+
def key(self) -> WorkerKey:
|
|
101
|
+
"""Which of its version's lanes serves the session: its turns, and its worker when it has one."""
|
|
102
|
+
return WorkerKey.of(self.version_id, self.world_state)
|
|
103
|
+
|
|
95
104
|
@property
|
|
96
105
|
def app(self) -> WorldApp | None:
|
|
97
106
|
"""The world's routes over this session's store, when this process
|
|
@@ -119,8 +128,17 @@ class Session:
|
|
|
119
128
|
capability = self.door.capability if self.door is not None else None
|
|
120
129
|
return self.auth == OPEN_MODE and capability is not None and same(named.capability, capability)
|
|
121
130
|
|
|
131
|
+
def browser_hosts(self, domain: str) -> list[str]:
|
|
132
|
+
"""The only hosts this session's browser may load: its ui and api, and their capability
|
|
133
|
+
pair when the session is open."""
|
|
134
|
+
hosts = [session_host(self.session_id, surface, domain) for surface in ("ui", "api")]
|
|
135
|
+
capability = self.door.capability if self.door is not None else None
|
|
136
|
+
if self.auth == OPEN_MODE and capability is not None:
|
|
137
|
+
hosts += [capability_host(self.session_id, capability, surface, domain) for surface in ("ui", "api")]
|
|
138
|
+
return hosts
|
|
139
|
+
|
|
122
140
|
def api_host(self, domain: str, opened: bool) -> str:
|
|
123
141
|
"""The host a link the world writes points at when a request names none."""
|
|
124
142
|
if opened and self.door is not None and self.door.capability is not None:
|
|
125
143
|
return capability_host(self.session_id, self.door.capability, "api", domain)
|
|
126
|
-
return
|
|
144
|
+
return session_host(self.session_id, "api", domain)
|
|
@@ -260,6 +260,12 @@ def _capability_host(rest: str, surface: str) -> SessionHost | None:
|
|
|
260
260
|
return SessionHost(session_id, surface, capability)
|
|
261
261
|
|
|
262
262
|
|
|
263
|
+
def session_host(session_id: str, surface: str, domain: str) -> str:
|
|
264
|
+
"""The session's own hostname for ``surface`` (``api`` or ``ui``)."""
|
|
265
|
+
suffix = "-api" if surface == "api" else ""
|
|
266
|
+
return f"w-{session_id}{suffix}.{domain}"
|
|
267
|
+
|
|
268
|
+
|
|
263
269
|
def capability_host(session_id: str, capability: str, surface: str, domain: str) -> str:
|
|
264
270
|
"""The capability hostname of an open session's ``surface`` (``api`` or ``ui``)."""
|
|
265
271
|
suffix = "-api" if surface == "api" else ""
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
from __future__ import annotations
|
|
5
5
|
|
|
6
6
|
import os
|
|
7
|
+
import re
|
|
7
8
|
import shutil
|
|
8
9
|
import tarfile
|
|
9
10
|
import threading
|
|
@@ -21,6 +22,7 @@ from .api import ApiSettings, Handler, Router, load_api_settings, load_handlers
|
|
|
21
22
|
from .compiler import check_bundle
|
|
22
23
|
from .connector import Spec, load_spec
|
|
23
24
|
from .database import WorldDatabase
|
|
25
|
+
from .database.base import reflink
|
|
24
26
|
from .dependencies import Dependency
|
|
25
27
|
from .door import Door
|
|
26
28
|
from .host_surfaces import UiSurface, load_ui_surface
|
|
@@ -33,6 +35,8 @@ from .validation import fail
|
|
|
33
35
|
|
|
34
36
|
BUNDLED_WORLD = "server/world"
|
|
35
37
|
WORLD_SPEC = "schema/world.json"
|
|
38
|
+
SNAPSHOTS_DIR = "snapshots"
|
|
39
|
+
_SHA256 = re.compile(r"[0-9a-f]{64}")
|
|
36
40
|
#: Bounded because the cache is one shared disk and a build's memory is the supervisor's.
|
|
37
41
|
DEFAULT_MAX_BUNDLE_MB = 2048
|
|
38
42
|
MAX_BUNDLE_MEMBERS = 200_000
|
|
@@ -53,6 +57,41 @@ def world_tree(bundle: Path) -> Path:
|
|
|
53
57
|
)
|
|
54
58
|
|
|
55
59
|
|
|
60
|
+
def clone_mode(directory: Path) -> str:
|
|
61
|
+
"""``reflink`` when session copies on ``directory`` share blocks until written, ``copy`` when each one
|
|
62
|
+
copies its file: the disk decides, so the host probes it once and reports it in its stats."""
|
|
63
|
+
probe = directory / f".clone-probe-{uuid.uuid4().hex}"
|
|
64
|
+
clone = probe.with_name(f"{probe.name}.clone")
|
|
65
|
+
try:
|
|
66
|
+
probe.write_bytes(b"probe")
|
|
67
|
+
return "reflink" if reflink(probe, clone) else "copy"
|
|
68
|
+
finally:
|
|
69
|
+
probe.unlink(missing_ok=True)
|
|
70
|
+
clone.unlink(missing_ok=True)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def data_tree(bundle: Path) -> Path:
|
|
74
|
+
return bundle if is_tools_bundle(bundle) else world_tree(bundle)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@dataclass(frozen=True)
|
|
78
|
+
class DataKey:
|
|
79
|
+
"""A version as a session opens it: on the version's own data, or on one snapshot of it, named by the
|
|
80
|
+
sha256 of the snapshot file. Two snapshots of one version never share a tree, a template or a session."""
|
|
81
|
+
|
|
82
|
+
version_id: str
|
|
83
|
+
sha256: str | None
|
|
84
|
+
|
|
85
|
+
@classmethod
|
|
86
|
+
def of(cls, version_id: str, snapshot: dict[str, str] | None) -> DataKey:
|
|
87
|
+
if not snapshot:
|
|
88
|
+
return cls(version_id, None)
|
|
89
|
+
sha256 = str(snapshot.get("sha256") or "").lower()
|
|
90
|
+
if not _SHA256.fullmatch(sha256):
|
|
91
|
+
fail("host", "a snapshot names the sha256 of its file (64 hex digits)")
|
|
92
|
+
return cls(version_id, sha256)
|
|
93
|
+
|
|
94
|
+
|
|
56
95
|
@dataclass
|
|
57
96
|
class CachedVersion:
|
|
58
97
|
"""One cached version, of either shape the host serves: an api bundle (a
|
|
@@ -103,6 +142,10 @@ class CachedVersion:
|
|
|
103
142
|
def needs_worker(self) -> bool:
|
|
104
143
|
return self.authored_code is not None
|
|
105
144
|
|
|
145
|
+
@property
|
|
146
|
+
def world(self) -> str:
|
|
147
|
+
return self.runtime_class.world_name(self)
|
|
148
|
+
|
|
106
149
|
def served_names(self, dependencies: dict[str, DependencyWorld]) -> ServedNames:
|
|
107
150
|
"""The tools a session task may allow, beside the routes it may not."""
|
|
108
151
|
return ServedNames.of(self.runtime_class.served_by(self, dependencies), self.spec)
|
|
@@ -124,8 +167,9 @@ class VersionCache:
|
|
|
124
167
|
#: Off in the supervisor: only a version worker imports world-authored code.
|
|
125
168
|
self.run_handlers = run_handlers
|
|
126
169
|
self.max_bundle_bytes = max_bundle_mb * (1 << 20)
|
|
170
|
+
self.clone_mode = clone_mode(self.root)
|
|
127
171
|
self._lock = threading.Lock()
|
|
128
|
-
self._versions: dict[
|
|
172
|
+
self._versions: dict[DataKey, CachedVersion] = {}
|
|
129
173
|
self._builds = Lanes()
|
|
130
174
|
|
|
131
175
|
def ensure(
|
|
@@ -134,37 +178,76 @@ class VersionCache:
|
|
|
134
178
|
bundle_url: str,
|
|
135
179
|
snapshot: dict[str, str] | None,
|
|
136
180
|
) -> CachedVersion:
|
|
137
|
-
|
|
181
|
+
key = DataKey.of(version_id, snapshot)
|
|
182
|
+
hit = self._cached(key)
|
|
138
183
|
if hit is not None:
|
|
139
184
|
return hit
|
|
140
185
|
with self._builds.turn(version_id):
|
|
141
|
-
return self._cached(
|
|
186
|
+
return self._cached(key) or self._store(key, self._build(key, bundle_url, snapshot))
|
|
142
187
|
|
|
143
|
-
def _cached(self,
|
|
188
|
+
def _cached(self, key: DataKey) -> CachedVersion | None:
|
|
144
189
|
with self._lock:
|
|
145
|
-
return self._versions.get(
|
|
190
|
+
return self._versions.get(key)
|
|
146
191
|
|
|
147
|
-
def _store(self, cached: CachedVersion) -> CachedVersion:
|
|
192
|
+
def _store(self, key: DataKey, cached: CachedVersion) -> CachedVersion:
|
|
148
193
|
with self._lock:
|
|
149
|
-
self._versions[
|
|
194
|
+
self._versions[key] = cached
|
|
150
195
|
return cached
|
|
151
196
|
|
|
152
|
-
def has(self, version_id: str) -> bool:
|
|
197
|
+
def has(self, version_id: str, snapshot: dict[str, str] | None = None) -> bool:
|
|
198
|
+
key = DataKey.of(version_id, snapshot)
|
|
153
199
|
with self._lock:
|
|
154
|
-
return
|
|
200
|
+
return key in self._versions
|
|
155
201
|
|
|
156
202
|
def version_ids(self) -> list[str]:
|
|
157
203
|
with self._lock:
|
|
158
|
-
return sorted(self._versions)
|
|
204
|
+
return sorted({key.version_id for key in self._versions})
|
|
159
205
|
|
|
160
|
-
def _build(self,
|
|
161
|
-
|
|
162
|
-
if not
|
|
163
|
-
self._download(bundle_url,
|
|
164
|
-
if
|
|
165
|
-
|
|
166
|
-
|
|
206
|
+
def _build(self, key: DataKey, bundle_url: str, snapshot: dict[str, str] | None) -> CachedVersion:
|
|
207
|
+
base = self.root / key.version_id / "bundle"
|
|
208
|
+
if not base.is_dir():
|
|
209
|
+
self._download(bundle_url, base)
|
|
210
|
+
if key.sha256 is None or snapshot is None:
|
|
211
|
+
return self._load(key.version_id, base, self.root / key.version_id / "template.db", fetched=False)
|
|
212
|
+
bundle = self._with_snapshot(key, base, snapshot)
|
|
213
|
+
return self._load(key.version_id, bundle, bundle.parent / "template.db", fetched=True)
|
|
214
|
+
|
|
215
|
+
def _with_snapshot(self, key: DataKey, base: Path, snapshot: dict[str, str]) -> Path:
|
|
216
|
+
"""The version's bundle with the snapshot's rows as its data, under ``snapshots/<sha256>``: cloned
|
|
217
|
+
from the downloaded bundle (never linked, so nothing written here reaches it) without the data the
|
|
218
|
+
snapshot replaces, then fetched and verified. Renamed into place only once complete; one built
|
|
219
|
+
before is reused without a url, which is how a version worker finds it."""
|
|
220
|
+
home = self.root / key.version_id / SNAPSHOTS_DIR / str(key.sha256)
|
|
221
|
+
if (home / "bundle").is_dir():
|
|
222
|
+
return home / "bundle"
|
|
223
|
+
url, digest = snapshot.get("url"), snapshot.get("digest")
|
|
224
|
+
if not url or not isinstance(digest, str):
|
|
225
|
+
fail(
|
|
226
|
+
"host",
|
|
227
|
+
f"snapshot {key.sha256} of version {key.version_id} is not cached on this host, "
|
|
228
|
+
"and the open names no url and digest to fetch it",
|
|
167
229
|
)
|
|
230
|
+
tree = data_tree(base)
|
|
231
|
+
contract = check_bundle(tree)
|
|
232
|
+
replaced = snapshots.default_snapshot(tree, contract)
|
|
233
|
+
staging = home.parent / f".{key.sha256}.{uuid.uuid4().hex}"
|
|
234
|
+
try:
|
|
235
|
+
for path in sorted(base.rglob("*")):
|
|
236
|
+
target = staging / "bundle" / path.relative_to(base)
|
|
237
|
+
if path.is_dir():
|
|
238
|
+
target.mkdir(parents=True, exist_ok=True)
|
|
239
|
+
elif path != replaced:
|
|
240
|
+
WorldDatabase.clone_file(path, target)
|
|
241
|
+
staged = staging / "bundle" / tree.relative_to(base)
|
|
242
|
+
snapshots.fetch(staged, contract, url, str(key.sha256), digest)
|
|
243
|
+
os.replace(staging, home)
|
|
244
|
+
finally:
|
|
245
|
+
shutil.rmtree(staging, ignore_errors=True)
|
|
246
|
+
return home / "bundle"
|
|
247
|
+
|
|
248
|
+
def _load(self, version_id: str, bundle: Path, template: Path, *, fetched: bool) -> CachedVersion:
|
|
249
|
+
if is_tools_bundle(bundle):
|
|
250
|
+
tools = load_tools_version(bundle, template, fetched=fetched, run_handlers=self.run_handlers)
|
|
168
251
|
return CachedVersion(
|
|
169
252
|
version_id=version_id,
|
|
170
253
|
bundle=bundle,
|
|
@@ -179,8 +262,6 @@ class VersionCache:
|
|
|
179
262
|
)
|
|
180
263
|
tree = world_tree(bundle)
|
|
181
264
|
contract = check_bundle(tree)
|
|
182
|
-
if snapshot:
|
|
183
|
-
snapshots.fetch(tree, contract, snapshot["url"], snapshot["sha256"], snapshot["digest"])
|
|
184
265
|
served = snapshots.ServedSnapshot.of(tree)
|
|
185
266
|
spec = load_spec(tree)
|
|
186
267
|
settings = load_api_settings(tree, spec)
|