@withgateway/sdk 0.4.0 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/index.d.mts +573 -0
- package/dist/agents/index.d.ts +573 -0
- package/dist/agents/index.js +1443 -0
- package/dist/agents/index.js.map +1 -0
- package/dist/agents/index.mjs +1394 -0
- package/dist/agents/index.mjs.map +1 -0
- package/dist/apiError-DHpfbmdx.d.mts +17 -0
- package/dist/apiError-DHpfbmdx.d.ts +17 -0
- package/dist/cli.js +5655 -1890
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +5769 -2005
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +3 -1
- package/dist/files/index.d.ts +3 -1
- package/dist/files/index.js +418 -120
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +418 -120
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.js +346 -136
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +346 -136
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-DUHmS74p.d.ts → index-DIN14Lhg.d.ts} +27 -4
- package/dist/{index-mJhG3vZU.d.mts → index-DuB-eQ0Z.d.mts} +27 -4
- package/dist/index.d.mts +7 -4
- package/dist/index.d.ts +7 -4
- package/dist/index.js +2862 -438
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2820 -431
- package/dist/index.mjs.map +1 -1
- package/dist/media-CQUC7TkS.d.mts +39 -0
- package/dist/media-DQnBloaX.d.ts +39 -0
- package/dist/replay/index.d.mts +1 -1
- package/dist/replay/index.d.ts +1 -1
- package/dist/replay/index.js.map +1 -1
- package/dist/replay/index.mjs.map +1 -1
- package/dist/traces/index.d.mts +3 -1
- package/dist/traces/index.d.ts +3 -1
- package/dist/traces/index.js +418 -120
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +418 -120
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +3 -2
- package/dist/tracing/index.d.ts +3 -2
- package/dist/tracing/index.js +257 -16
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +257 -16
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/transport-CpRe7zgd.d.mts +123 -0
- package/dist/transport-CpRe7zgd.d.ts +123 -0
- package/dist/worlds/index.d.mts +2086 -1749
- package/dist/worlds/index.d.ts +2086 -1749
- package/dist/worlds/index.js +1471 -269
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1466 -265
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +6 -1
- package/runtime/gatewaysdk/environment/_bundle.py +4 -6
- package/runtime/gatewaysdk/environment/_world.py +24 -1
- package/runtime/gatewaysdk/environment/bundle_manifest.py +243 -0
- package/runtime/gatewaysdk/environment/runtime.py +45 -2
- package/runtime/gatewaysdk/environment/schema/__main__.py +2 -2
- package/runtime/gatewaysdk/environment/schema/_slack_fidelity.py +8 -23
- package/runtime/gatewaysdk/environment/schema/api.py +27 -7
- package/runtime/gatewaysdk/environment/schema/assertions.py +7 -1
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +2 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +13 -1
- package/runtime/gatewaysdk/environment/schema/call_log.py +66 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +6 -0
- package/runtime/gatewaysdk/environment/schema/compiler.py +4 -1
- package/runtime/gatewaysdk/environment/schema/conform.py +32 -8
- package/runtime/gatewaysdk/environment/schema/connector.py +5 -0
- package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +2 -1
- package/runtime/gatewaysdk/environment/schema/connectors/github/api/handlers.py +217 -58
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +9 -19
- package/runtime/gatewaysdk/environment/schema/connectors/github/scenarios/default/seed/github_webhooks.json +10 -2
- package/runtime/gatewaysdk/environment/schema/connectors/github/tests/webhooks.json +47 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +21 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/api/handlers.py +185 -18
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/connector.toml +29 -14
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +43 -12
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scenarios/default/seed/jamf_webhooks.json +2 -2
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +4 -17
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/tests/webhooks.json +107 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/vendor-openapi.json +1 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/api/handlers.py +85 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/connector.toml +19 -31
- package/runtime/gatewaysdk/environment/schema/connectors/linear/provenance.json +18 -8
- package/runtime/gatewaysdk/environment/schema/connectors/linear/scenarios/default/seed/webhooks.json +19 -3
- package/runtime/gatewaysdk/environment/schema/connectors/linear/tests/test_webhooks.py +71 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +2 -2
- package/runtime/gatewaysdk/environment/schema/database/guard.py +2 -1
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +27 -1
- package/runtime/gatewaysdk/environment/schema/host.py +74 -11
- package/runtime/gatewaysdk/environment/schema/host_browser.py +48 -0
- package/runtime/gatewaysdk/environment/schema/host_control.py +33 -2
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +4 -1
- package/runtime/gatewaysdk/environment/schema/host_requests.py +14 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +13 -2
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +96 -19
- package/runtime/gatewaysdk/environment/schema/journal.py +10 -4
- package/runtime/gatewaysdk/environment/schema/local.py +33 -7
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +20 -5
- package/runtime/gatewaysdk/environment/schema/page_checks.py +123 -0
- package/runtime/gatewaysdk/environment/schema/platform.py +33 -480
- package/runtime/gatewaysdk/environment/schema/platform_templates.py +332 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +4 -1
- package/runtime/gatewaysdk/environment/schema/run_parameters.py +25 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +49 -4
- package/runtime/gatewaysdk/environment/schema/scaffold.py +369 -218
- package/runtime/gatewaysdk/environment/schema/session_files.py +13 -2
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +1 -1
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +20 -24
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +11 -11
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +9 -9
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +45 -12
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +40 -9
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +17 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/new-world-sandbox.md +1 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/ui-map.md +3 -2
- package/runtime/gatewaysdk/environment/schema/slack.py +55 -10
- package/runtime/gatewaysdk/environment/schema/snapshot.py +30 -4
- package/runtime/gatewaysdk/environment/schema/tasks.py +25 -2
- package/runtime/gatewaysdk/environment/schema/template_record.py +131 -0
- package/runtime/gatewaysdk/environment/schema/tools_world.py +26 -5
- package/runtime/gatewaysdk/environment/schema/ui.py +54 -33
- package/runtime/gatewaysdk/environment/schema/ui_interactions.py +419 -0
- package/runtime/gatewaysdk/environment/schema/ui_recorder.js +895 -0
- package/runtime/gatewaysdk/environment/schema/ui_replay.py +366 -0
- package/runtime/gatewaysdk/environment/schema/ui_replay_record.LICENSE +25 -0
- package/runtime/gatewaysdk/environment/schema/ui_replay_record.js +109 -0
- package/runtime/gatewaysdk/environment/schema/validation.py +10 -1
- package/runtime/gatewaysdk/environment/schema/verbs.py +6 -0
- package/runtime/gatewaysdk/environment/schema/webhooks.py +687 -0
- package/runtime/gatewaysdk/environment/schema/worker.py +5 -1
- package/runtime/gatewaysdk/environment/schema/workers.py +8 -1
- package/runtime/gatewaysdk/environment/schema/world_host.py +64 -18
- package/runtime/gatewaysdk/environment/schema/world_tests.py +74 -0
- package/runtime/gatewaysdk/environment/secret_values.py +1 -1
- package/runtime/gatewaysdk/environment/world.py +34 -43
- package/runtime/gatewaysdk/environment/world_session_contract.py +7 -34
- package/skills/benchmarks-getting-started/SKILL.md +1 -1
- package/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/skills/connector-schema-authoring/SKILL.md +20 -24
- package/skills/connector-schema-authoring/connector-worlds.md +155 -13
- package/skills/connector-schema-authoring/world-schema.md +36 -29
- package/skills/world-data-ingestion/SKILL.md +11 -11
- package/skills/worlds-getting-started/SKILL.md +9 -9
- package/skills/worlds-getting-started/references/cli-mcp-map.md +45 -12
- package/skills/worlds-getting-started/references/gotchas.md +40 -9
- package/skills/worlds-getting-started/references/journal-checks.md +17 -2
- package/skills/worlds-getting-started/references/new-world-sandbox.md +1 -1
- package/skills/worlds-getting-started/references/ui-map.md +3 -2
- package/dist/media-BIApiPzS.d.mts +0 -102
- package/dist/media-BIApiPzS.d.ts +0 -102
|
@@ -50,15 +50,18 @@ import signal
|
|
|
50
50
|
import sys
|
|
51
51
|
import threading
|
|
52
52
|
import urllib.parse
|
|
53
|
+
from functools import partial
|
|
53
54
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
54
55
|
from pathlib import Path
|
|
55
56
|
from typing import Any
|
|
56
57
|
|
|
57
58
|
from .api import Via, with_query
|
|
58
59
|
from .call_capture import JSON_TYPE, send_response
|
|
60
|
+
from .call_redaction import CallRedaction
|
|
59
61
|
from .credentials import BearerAuth
|
|
60
62
|
from .door import PLATFORM_HEADER, Entry
|
|
61
63
|
from .handler_errors import log_request_failure, request_failure_body
|
|
64
|
+
from .host_browser import RelayNavigation, log_relay
|
|
62
65
|
from .host_control import ControlPlane
|
|
63
66
|
from .host_handoff import DrainHandoff, HandoffDoor, PlatformHandoff
|
|
64
67
|
from .host_loops import HEARTBEAT_S, Heartbeat, start_lease_sweeper
|
|
@@ -90,9 +93,12 @@ from .host_versions import DEFAULT_MAX_BUNDLE_MB
|
|
|
90
93
|
from .host_workers import UNKNOWN_SESSION, refusal
|
|
91
94
|
from .request_ledger import LedgerFlusher, LedgerSpill, NoLedger, PlatformLedger
|
|
92
95
|
from .session_move import SessionMoved
|
|
96
|
+
from .ui_interactions import RECORDER_PATH, UiRecorder
|
|
97
|
+
from .ui_replay import REPLAY_PATH, REPLAY_SCRIPT_PATH, REPLAY_STOP_S, ReplayShipper, UiReplay
|
|
93
98
|
from .validation import ContractError
|
|
94
99
|
from .workers import (
|
|
95
100
|
DEFAULT_WORKER_BUDGET_MB,
|
|
101
|
+
DEFAULT_WORKER_FILE_MB,
|
|
96
102
|
DEFAULT_WORKER_FOOTPRINT_MB,
|
|
97
103
|
DEFAULT_WORKER_MEMORY_MB,
|
|
98
104
|
WorkerPool,
|
|
@@ -127,13 +133,15 @@ def make_host_handler(
|
|
|
127
133
|
*,
|
|
128
134
|
max_body_mb: int = DEFAULT_MAX_BODY_MB,
|
|
129
135
|
secure_cookies: bool = True,
|
|
136
|
+
navigation: RelayNavigation | None = None,
|
|
137
|
+
replays: ReplayShipper | None = None,
|
|
130
138
|
):
|
|
131
139
|
"""The request handler class for one host; its control plane is :class:`ControlPlane`, which
|
|
132
140
|
refuses an empty ``secret`` or ``worlds_domain``. ``secure_cookies`` is off
|
|
133
141
|
only for a host whose public surface is reached over plain http (a dev
|
|
134
142
|
host); it is the deployment's word, not the request's, since a client
|
|
135
143
|
writes the forwarding headers a request arrives with."""
|
|
136
|
-
control = ControlPlane(host, secret, HOSTED_OPERATIONS, worlds_domain)
|
|
144
|
+
control = ControlPlane(host, secret, HOSTED_OPERATIONS, worlds_domain, navigation)
|
|
137
145
|
if max_body_mb <= 0:
|
|
138
146
|
raise ValueError(f"the request body cap must be positive (WORLDS_HOST_MAX_BODY_MB={max_body_mb})")
|
|
139
147
|
max_body_bytes = max_body_mb * (1 << 20)
|
|
@@ -318,12 +326,23 @@ def make_host_handler(
|
|
|
318
326
|
):
|
|
319
327
|
return self._reply(401, {"error": "invalid session token"})
|
|
320
328
|
route = api_target(ui, url.path)
|
|
321
|
-
if
|
|
322
|
-
return self.
|
|
329
|
+
if url.path == RECORDER_PATH:
|
|
330
|
+
return self._serve_file(method, opened, UiRecorder.script)
|
|
331
|
+
if url.path == REPLAY_SCRIPT_PATH:
|
|
332
|
+
return self._serve_file(method, opened, partial(UiReplay.script, replays))
|
|
333
|
+
static = route is None and not UiRecorder.serves(url.path)
|
|
334
|
+
if static and session.worker is not None and UiRecorder.is_page(url.path):
|
|
335
|
+
return self._on_worker(session, method, headers)
|
|
336
|
+
if static:
|
|
337
|
+
return self._serve_static(session, ui, method, url, opened)
|
|
323
338
|
if cross_site(headers, lambda origin: urllib.parse.urlsplit(origin).netloc == headers.get("host")):
|
|
324
339
|
return self._reply(403, {"error": "the ui's api prefix answers its own page, not another site"})
|
|
340
|
+
if url.path == REPLAY_PATH:
|
|
341
|
+
return self._record_replay(session, method)
|
|
325
342
|
if session.worker is not None:
|
|
326
343
|
return self._on_worker(session, method, headers)
|
|
344
|
+
if route is None:
|
|
345
|
+
return self._record_interactions(session, method, url, headers)
|
|
327
346
|
target = urllib.parse.urlunsplit(("", "", route, url.query, ""))
|
|
328
347
|
api_headers = {k: v for k, v in headers.items() if k not in ("cookie", "authorization")}
|
|
329
348
|
api_headers["host"] = session.api_host(worlds_domain, opened)
|
|
@@ -334,20 +353,49 @@ def make_host_handler(
|
|
|
334
353
|
cookie = session_cookie(token, secure=secure_cookies, max_age_s=host.lease_s)
|
|
335
354
|
self._reply(303, {"location": landing}, {"Location": landing, "Set-Cookie": cookie})
|
|
336
355
|
|
|
337
|
-
def _serve_static(self, ui: UiSurface, method: str, url, opened: bool) -> None:
|
|
338
|
-
"""A file of the ui;
|
|
339
|
-
|
|
356
|
+
def _serve_static(self, session: Session, ui: UiSurface, method: str, url, opened: bool) -> None:
|
|
357
|
+
"""A file of the ui; a page carries the recorder and is noted as served, a note for the
|
|
358
|
+
ledger alone (no grader reads it), so a page is never held by the move fence."""
|
|
340
359
|
if method != "GET":
|
|
341
360
|
return self._reply(405, {"error": "the static ui answers GET only"})
|
|
342
|
-
status,
|
|
343
|
-
|
|
361
|
+
status, file_headers, payload = static_file(ui, url.path)
|
|
362
|
+
page = UiRecorder.page(status, file_headers, payload)
|
|
363
|
+
if page is not None and session.runtime is not None:
|
|
364
|
+
UiRecorder.served(url.path, session.runtime.calls)
|
|
365
|
+
self._send_file(opened, status, file_headers, payload if page is None else page)
|
|
366
|
+
|
|
367
|
+
def _serve_file(self, method: str, opened: bool, answer) -> None:
|
|
368
|
+
if method != "GET":
|
|
369
|
+
return self._reply(405, {"error": "the static ui answers GET only"})
|
|
370
|
+
self._send_file(opened, *answer())
|
|
371
|
+
|
|
372
|
+
def _send_file(self, opened: bool, status: int, headers: dict[str, str], payload: bytes) -> None:
|
|
373
|
+
"""On a capability host with no ``Referer``, so a page that loads another site's
|
|
374
|
+
resource never hands it the capability."""
|
|
375
|
+
sealed = {**headers, **(NO_REFERRER if opened else {})}
|
|
344
376
|
self._relay(status, sealed.items(), payload, drop=FRAMING_HEADERS)
|
|
345
377
|
|
|
378
|
+
def _record_interactions(self, session: Session, method: str, url, headers: dict[str, str]) -> None:
|
|
379
|
+
if session.runtime is None:
|
|
380
|
+
return self._reply(404, {"error": "this session's world is not served here"})
|
|
381
|
+
raw = self._read()
|
|
382
|
+
with host.admitted(session, HOP_HEADER in headers):
|
|
383
|
+
answer = UiRecorder.record(method, url.path, raw, session.runtime.calls)
|
|
384
|
+
self._reply(*answer)
|
|
385
|
+
|
|
386
|
+
def _record_replay(self, session: Session, method: str) -> None:
|
|
387
|
+
"""The host that holds the session keeps its page's recording, a worker's session too, scrubbed
|
|
388
|
+
as the call log scrubs (:class:`CallRedaction`)."""
|
|
389
|
+
scrub = CallRedaction.of(session.spec, session.token).scrub
|
|
390
|
+
self._reply(*UiReplay.take(method, self._read(), session.session_id, scrub, replays))
|
|
391
|
+
|
|
346
392
|
def _serve_browser(self, session: Session, url, headers: dict[str, str]) -> None:
|
|
347
393
|
"""``/cdp/<session token>`` and ``/live/<grant>`` on the session's
|
|
348
394
|
cdp host: check the credential against the SESSION, then tunnel
|
|
349
395
|
the websocket to the leased relay with the POOL credential the
|
|
350
|
-
relay knows. The relay's token never leaves the host.
|
|
396
|
+
relay knows. The relay's token never leaves the host. The relay is
|
|
397
|
+
bound to the session's hosts first, so one that restarted and lost
|
|
398
|
+
its binding is never reached unbound."""
|
|
351
399
|
relay = session.browser
|
|
352
400
|
if relay is None:
|
|
353
401
|
return self._reply(404, {"error": "this session has no browser surface"})
|
|
@@ -368,11 +416,16 @@ def make_host_handler(
|
|
|
368
416
|
upstream_path = f"/live/{mint_live_grant(relay.token, LIVE_RELAY_GRANT_MS, scope)}"
|
|
369
417
|
if headers.get("upgrade", "").lower() != "websocket":
|
|
370
418
|
return self._reply(400, {"error": "the browser surface is a websocket"})
|
|
419
|
+
try:
|
|
420
|
+
control.navigation.bind(relay, session.browser_hosts(control.domain))
|
|
421
|
+
except ContractError as error:
|
|
422
|
+
return self._reply(502, {"error": "browser unbound", "detail": str(error)})
|
|
371
423
|
self.close_connection = True
|
|
372
424
|
try:
|
|
373
425
|
tunnel(self.connection, relay.url, upstream_path, headers)
|
|
374
426
|
except OSError as error:
|
|
375
|
-
|
|
427
|
+
log_relay("browser relay unreachable", relay, error=str(error))
|
|
428
|
+
self._reply(502, {"error": "browser unreachable"})
|
|
376
429
|
|
|
377
430
|
return Handler
|
|
378
431
|
|
|
@@ -397,12 +450,14 @@ class HostServer(ThreadingHTTPServer):
|
|
|
397
450
|
drain_s: float = DEFAULT_DRAIN_S,
|
|
398
451
|
ledger: LedgerFlusher | None = None,
|
|
399
452
|
handoff: HandoffDoor | None = None,
|
|
453
|
+
replays: ReplayShipper | None = None,
|
|
400
454
|
):
|
|
401
455
|
super().__init__(address, handler)
|
|
402
456
|
self.world_host = world_host
|
|
403
457
|
self.lease_sweeper = start_lease_sweeper(world_host)
|
|
404
458
|
self.heartbeat = Heartbeat(world_host, *heartbeat, interval_s=heartbeat_s).start() if heartbeat else None
|
|
405
459
|
world_host.ledger = ledger.start() if ledger else NoLedger()
|
|
460
|
+
self.replays = replays.start() if replays else None
|
|
406
461
|
self.handoff = DrainHandoff(world_host, handoff, drain_s)
|
|
407
462
|
self._answering = 0
|
|
408
463
|
self._draining = False
|
|
@@ -457,6 +512,7 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
|
|
|
457
512
|
``WORLDS_WORKER_MEMORY_MB`` caps each version worker's address space, ``WORLDS_WORKER_BUDGET_MB`` is the memory
|
|
458
513
|
all of them together are held to and ``WORLDS_WORKER_FOOTPRINT_MB`` what each is counted at against it (unset,
|
|
459
514
|
its whole address-space cap; their count is the quotient, and one more version answers 503 ``at_capacity``),
|
|
515
|
+
``WORLDS_WORKER_FILE_MB`` is the largest file a worker may write (a session's database, so a world state's data),
|
|
460
516
|
``WORLDS_HOST_OBJECT_STORE`` is where every presigned URL it is handed may point (required; ``any`` for a dev
|
|
461
517
|
host), ``WORLDS_HOST_MAX_BUNDLE_MB`` is the largest version bundle the cache accepts,
|
|
462
518
|
``WORLDS_HOST_MAX_BODY_MB`` the largest request body it reads, ``WORLDS_HOST_LEDGER_SPILL_DIR`` (under the cache
|
|
@@ -491,6 +547,7 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
|
|
|
491
547
|
memory_mb=int(os.environ.get("WORLDS_WORKER_MEMORY_MB") or DEFAULT_WORKER_MEMORY_MB),
|
|
492
548
|
budget_mb=int(os.environ.get("WORLDS_WORKER_BUDGET_MB") or DEFAULT_WORKER_BUDGET_MB),
|
|
493
549
|
footprint_mb=int(footprint) if footprint else DEFAULT_WORKER_FOOTPRINT_MB,
|
|
550
|
+
file_mb=int(os.environ.get("WORLDS_WORKER_FILE_MB") or DEFAULT_WORKER_FILE_MB),
|
|
494
551
|
)
|
|
495
552
|
owners = PlatformOwners(platform_url, secret, advertise) if platform_url else None
|
|
496
553
|
world_host = WorldHost(
|
|
@@ -504,6 +561,7 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
|
|
|
504
561
|
build=os.environ.get("WORLDS_HOST_BUILD") or None,
|
|
505
562
|
owners=owners,
|
|
506
563
|
)
|
|
564
|
+
replays = ReplayShipper(platform_url, secret) if platform_url else None
|
|
507
565
|
handler = make_host_handler(
|
|
508
566
|
world_host,
|
|
509
567
|
secret=secret,
|
|
@@ -511,6 +569,7 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
|
|
|
511
569
|
owner_lookup=owners,
|
|
512
570
|
max_body_mb=int(os.environ.get("WORLDS_HOST_MAX_BODY_MB") or DEFAULT_MAX_BODY_MB),
|
|
513
571
|
secure_cookies=os.environ.get("WORLDS_HOST_PLAIN_HTTP", "0") != "1",
|
|
572
|
+
replays=replays,
|
|
514
573
|
)
|
|
515
574
|
sink = PlatformLedger(platform_url, secret, host_url=advertise) if platform_url else None
|
|
516
575
|
return HostServer(
|
|
@@ -522,6 +581,7 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
|
|
|
522
581
|
heartbeat_s=float(os.environ.get("WORLDS_HOST_HEARTBEAT_S") or HEARTBEAT_S),
|
|
523
582
|
drain_s=float(os.environ.get("WORLDS_HOST_DRAIN_S") or DEFAULT_DRAIN_S),
|
|
524
583
|
handoff=PlatformHandoff(platform_url, secret) if platform_url and advertise else None,
|
|
584
|
+
replays=replays,
|
|
525
585
|
)
|
|
526
586
|
|
|
527
587
|
|
|
@@ -534,7 +594,8 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
534
594
|
args = parser.parse_args(argv)
|
|
535
595
|
server = serve(args.cache_root, port=args.port)
|
|
536
596
|
print(
|
|
537
|
-
f"world host listening on :{server.server_address[1]}, cache {args.cache_root}"
|
|
597
|
+
f"world host listening on :{server.server_address[1]}, cache {args.cache_root} "
|
|
598
|
+
f"(session copies: {server.world_host.cache.clone_mode})",
|
|
538
599
|
file=sys.stderr,
|
|
539
600
|
flush=True,
|
|
540
601
|
)
|
|
@@ -543,6 +604,8 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
543
604
|
server.serve_forever()
|
|
544
605
|
finally:
|
|
545
606
|
server.world_host.ledger.stop(settle=server.world_host.finishing.settle)
|
|
607
|
+
if server.replays is not None:
|
|
608
|
+
server.replays.stop(REPLAY_STOP_S)
|
|
546
609
|
server.lease_sweeper.set()
|
|
547
610
|
if server.heartbeat is not None:
|
|
548
611
|
server.heartbeat.stop()
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Binds a leased pool browser to the one session it serves: the relay's egress proxy then
|
|
2
|
+
loads that session's hosts and nothing else (deploy/world-browser/navigation.mjs)."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import json
|
|
7
|
+
import sys
|
|
8
|
+
import time
|
|
9
|
+
import urllib.error
|
|
10
|
+
import urllib.request
|
|
11
|
+
from collections.abc import Callable
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
from .host_requests import BrowserRelay
|
|
16
|
+
from .validation import fail
|
|
17
|
+
|
|
18
|
+
GUARD_REQUIRED_FROM = 1794787200.0
|
|
19
|
+
BIND_TIMEOUT_S = 5.0
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True)
|
|
23
|
+
class RelayNavigation:
|
|
24
|
+
opener: Callable[..., Any] = field(default=urllib.request.urlopen)
|
|
25
|
+
clock: Callable[[], float] = field(default=time.time)
|
|
26
|
+
|
|
27
|
+
def bind(self, relay: BrowserRelay, hosts: list[str]) -> str:
|
|
28
|
+
request = urllib.request.Request(
|
|
29
|
+
f"{relay.http_url}/navigation",
|
|
30
|
+
data=json.dumps({"hosts": hosts}).encode(),
|
|
31
|
+
method="PUT",
|
|
32
|
+
headers={"authorization": f"Bearer {relay.token}", "content-type": "application/json"},
|
|
33
|
+
)
|
|
34
|
+
try:
|
|
35
|
+
with self.opener(request, timeout=BIND_TIMEOUT_S) as answer:
|
|
36
|
+
return str(json.loads(answer.read() or b"{}").get("navigation", "bound"))
|
|
37
|
+
except urllib.error.HTTPError as error:
|
|
38
|
+
if error.code == 404 and self.clock() < GUARD_REQUIRED_FROM:
|
|
39
|
+
log_relay("browser relay predates the navigation guard", relay)
|
|
40
|
+
return "unguarded"
|
|
41
|
+
fail("host", f"the browser could not be bound to this session (HTTP {error.code})")
|
|
42
|
+
except (OSError, ValueError) as error:
|
|
43
|
+
log_relay("browser relay could not be bound", relay, error=str(error))
|
|
44
|
+
fail("host", "the browser could not be bound to this session: its relay did not answer as expected")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def log_relay(msg: str, relay: BrowserRelay, **extra: str) -> None:
|
|
48
|
+
print(json.dumps({"msg": msg, "relay": relay.url, **extra}), file=sys.stderr, flush=True)
|
|
@@ -15,6 +15,8 @@ from email.message import Message
|
|
|
15
15
|
from typing import TYPE_CHECKING, Any
|
|
16
16
|
|
|
17
17
|
from .handler_errors import carried
|
|
18
|
+
from .host_browser import RelayNavigation
|
|
19
|
+
from .host_requests import BrowserRelay
|
|
18
20
|
from .host_surfaces import public_host
|
|
19
21
|
from .host_workers import WORKER_EXITED, refusal
|
|
20
22
|
from .validation import ContractError, fail
|
|
@@ -35,10 +37,18 @@ class ControlPlane:
|
|
|
35
37
|
"""One host's control plane. A host with no secret has no control plane at all rather than an
|
|
36
38
|
open one, and one with no worlds domain could not tell its own address from a session's."""
|
|
37
39
|
|
|
38
|
-
def __init__(
|
|
40
|
+
def __init__(
|
|
41
|
+
self,
|
|
42
|
+
host: WorldHost,
|
|
43
|
+
secret: str,
|
|
44
|
+
operations: Mapping[str, Callable[..., dict]],
|
|
45
|
+
worlds_domain: str,
|
|
46
|
+
navigation: RelayNavigation | None = None,
|
|
47
|
+
):
|
|
39
48
|
if not secret:
|
|
40
49
|
raise ValueError("the world host needs a control-plane secret (WORLDS_HOST_SECRET)")
|
|
41
50
|
self.host = host
|
|
51
|
+
self.navigation = navigation or RelayNavigation()
|
|
42
52
|
self.operations = operations
|
|
43
53
|
self.domain = self.domain_of(worlds_domain)
|
|
44
54
|
self._secret = secret.encode()
|
|
@@ -109,6 +119,20 @@ class ControlPlane:
|
|
|
109
119
|
return 404, {"error": "unknown control route"}
|
|
110
120
|
|
|
111
121
|
def _open(self, body: dict) -> dict:
|
|
122
|
+
answer = self._open_session(body)
|
|
123
|
+
relay = BrowserRelay.parse(body.get("browser"))
|
|
124
|
+
if relay is not None:
|
|
125
|
+
self._bind_or_close(body["sessionId"], relay)
|
|
126
|
+
return answer
|
|
127
|
+
|
|
128
|
+
def _bind_or_close(self, session_id: str, relay: BrowserRelay) -> None:
|
|
129
|
+
try:
|
|
130
|
+
self.navigation.bind(relay, self.host.browser_hosts(session_id, self.domain))
|
|
131
|
+
except Exception:
|
|
132
|
+
self.host.close(session_id, "browser_unbound")
|
|
133
|
+
raise
|
|
134
|
+
|
|
135
|
+
def _open_session(self, body: dict) -> dict:
|
|
112
136
|
return self.host.open(
|
|
113
137
|
session_id=body["sessionId"],
|
|
114
138
|
version_id=body["versionId"],
|
|
@@ -123,14 +147,21 @@ class ControlPlane:
|
|
|
123
147
|
pending=body.get("pending") is True,
|
|
124
148
|
auth=body.get("auth"),
|
|
125
149
|
ledger_source=body.get("ledgerSource"),
|
|
150
|
+
parameters=body.get("parameters"),
|
|
126
151
|
)
|
|
127
152
|
|
|
153
|
+
def _swap_browser(self, session_id: str, browser: Any) -> dict:
|
|
154
|
+
relay = BrowserRelay.parse(browser)
|
|
155
|
+
if relay is not None:
|
|
156
|
+
self.navigation.bind(relay, self.host.browser_hosts(session_id, self.domain))
|
|
157
|
+
return self.host.set_browser(session_id, browser)
|
|
158
|
+
|
|
128
159
|
def _session(self, method: str, parts: list[str], body: dict) -> Answer:
|
|
129
160
|
sid = parts[2]
|
|
130
161
|
self.host.forwarding.refuse(sid)
|
|
131
162
|
verb = parts[3] if len(parts) > 3 else None
|
|
132
163
|
if method == "PUT" and verb == "browser" and len(parts) == 4:
|
|
133
|
-
return 200, self.
|
|
164
|
+
return 200, self._swap_browser(sid, body.get("browser"))
|
|
134
165
|
if method == "POST" and verb == "export" and len(parts) == 5:
|
|
135
166
|
return self._upload(sid, parts[4], body.get("url"))
|
|
136
167
|
row = session_verb(method, verb)
|
|
@@ -31,6 +31,7 @@ from .handler_errors import log_line
|
|
|
31
31
|
from .host_owners import PROXY_TIMEOUT_S
|
|
32
32
|
from .request_ledger import LEDGER_STOP_S
|
|
33
33
|
from .session_move import same_host
|
|
34
|
+
from .ui_replay import REPLAY_STOP_S
|
|
34
35
|
|
|
35
36
|
if TYPE_CHECKING:
|
|
36
37
|
from .host_session import Session
|
|
@@ -47,7 +48,9 @@ ENLIST_S = 1.0
|
|
|
47
48
|
LINGER_S = 5.0
|
|
48
49
|
IN_FLIGHT_S = PROXY_TIMEOUT_S
|
|
49
50
|
EXIT_S = 110.0
|
|
50
|
-
assert
|
|
51
|
+
assert (
|
|
52
|
+
LINGER_S + IN_FLIGHT_S + LEDGER_STOP_S + REPLAY_STOP_S + 5.0 <= EXIT_S
|
|
53
|
+
), "a stopping host's exit fits the grace past its drain"
|
|
51
54
|
|
|
52
55
|
MOVED = "moved"
|
|
53
56
|
STAYED = "stayed"
|
|
@@ -28,6 +28,16 @@ class DependencyRef:
|
|
|
28
28
|
snapshot: dict[str, str] | None
|
|
29
29
|
raw: dict[str, Any]
|
|
30
30
|
|
|
31
|
+
@property
|
|
32
|
+
def for_worker(self) -> dict[str, Any]:
|
|
33
|
+
"""The entry a version worker is handed: its snapshot without the url, a tree the supervisor built."""
|
|
34
|
+
return {**self.raw, "snapshot": worker_snapshot(self.snapshot)}
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def worker_snapshot(snapshot: dict[str, str] | None) -> dict[str, str] | None:
|
|
38
|
+
"""A snapshot as a version worker sees it: its sha256 and digest, never the presigned url."""
|
|
39
|
+
return {key: value for key, value in snapshot.items() if key != "url"} if snapshot else None
|
|
40
|
+
|
|
31
41
|
|
|
32
42
|
def parse_dependency_refs(dependencies: Any) -> list[DependencyRef]:
|
|
33
43
|
if dependencies is None:
|
|
@@ -77,6 +87,10 @@ class BrowserRelay:
|
|
|
77
87
|
raise AssertionError # fail() never returns
|
|
78
88
|
return cls(url.rstrip("/"), token)
|
|
79
89
|
|
|
90
|
+
@property
|
|
91
|
+
def http_url(self) -> str:
|
|
92
|
+
return "http://" + self.url.removeprefix("ws://")
|
|
93
|
+
|
|
80
94
|
|
|
81
95
|
def restore_sha256(restore: object) -> str | None:
|
|
82
96
|
"""The sha256 of the export an open restores from, or None for an open that restores nothing."""
|
|
@@ -10,7 +10,7 @@ from pathlib import Path
|
|
|
10
10
|
from typing import TYPE_CHECKING
|
|
11
11
|
|
|
12
12
|
from .door import OPEN_MODE, VENDOR_MODE, Door, same
|
|
13
|
-
from .host_surfaces import SessionHost, capability_host
|
|
13
|
+
from .host_surfaces import SessionHost, capability_host, session_host
|
|
14
14
|
from .request_ledger import LedgerLife
|
|
15
15
|
from .session_move import SessionMove
|
|
16
16
|
|
|
@@ -75,6 +75,8 @@ class Session:
|
|
|
75
75
|
placement: dict | None = None
|
|
76
76
|
settings: ApiSettings | None = None
|
|
77
77
|
restore_sha256: str | None = None
|
|
78
|
+
snapshot_sha256: str | None = None
|
|
79
|
+
parameters: dict[str, str] = field(default_factory=dict)
|
|
78
80
|
pending: InitVar[bool] = False
|
|
79
81
|
move: SessionMove = field(init=False, repr=False, compare=False)
|
|
80
82
|
auth: str = VENDOR_MODE
|
|
@@ -119,8 +121,17 @@ class Session:
|
|
|
119
121
|
capability = self.door.capability if self.door is not None else None
|
|
120
122
|
return self.auth == OPEN_MODE and capability is not None and same(named.capability, capability)
|
|
121
123
|
|
|
124
|
+
def browser_hosts(self, domain: str) -> list[str]:
|
|
125
|
+
"""The only hosts this session's browser may load: its ui and api, and their capability
|
|
126
|
+
pair when the session is open."""
|
|
127
|
+
hosts = [session_host(self.session_id, surface, domain) for surface in ("ui", "api")]
|
|
128
|
+
capability = self.door.capability if self.door is not None else None
|
|
129
|
+
if self.auth == OPEN_MODE and capability is not None:
|
|
130
|
+
hosts += [capability_host(self.session_id, capability, surface, domain) for surface in ("ui", "api")]
|
|
131
|
+
return hosts
|
|
132
|
+
|
|
122
133
|
def api_host(self, domain: str, opened: bool) -> str:
|
|
123
134
|
"""The host a link the world writes points at when a request names none."""
|
|
124
135
|
if opened and self.door is not None and self.door.capability is not None:
|
|
125
136
|
return capability_host(self.session_id, self.door.capability, "api", domain)
|
|
126
|
-
return
|
|
137
|
+
return session_host(self.session_id, "api", domain)
|
|
@@ -260,6 +260,12 @@ def _capability_host(rest: str, surface: str) -> SessionHost | None:
|
|
|
260
260
|
return SessionHost(session_id, surface, capability)
|
|
261
261
|
|
|
262
262
|
|
|
263
|
+
def session_host(session_id: str, surface: str, domain: str) -> str:
|
|
264
|
+
"""The session's own hostname for ``surface`` (``api`` or ``ui``)."""
|
|
265
|
+
suffix = "-api" if surface == "api" else ""
|
|
266
|
+
return f"w-{session_id}{suffix}.{domain}"
|
|
267
|
+
|
|
268
|
+
|
|
263
269
|
def capability_host(session_id: str, capability: str, surface: str, domain: str) -> str:
|
|
264
270
|
"""The capability hostname of an open session's ``surface`` (``api`` or ``ui``)."""
|
|
265
271
|
suffix = "-api" if surface == "api" else ""
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
from __future__ import annotations
|
|
5
5
|
|
|
6
6
|
import os
|
|
7
|
+
import re
|
|
7
8
|
import shutil
|
|
8
9
|
import tarfile
|
|
9
10
|
import threading
|
|
@@ -21,6 +22,7 @@ from .api import ApiSettings, Handler, Router, load_api_settings, load_handlers
|
|
|
21
22
|
from .compiler import check_bundle
|
|
22
23
|
from .connector import Spec, load_spec
|
|
23
24
|
from .database import WorldDatabase
|
|
25
|
+
from .database.base import reflink
|
|
24
26
|
from .dependencies import Dependency
|
|
25
27
|
from .door import Door
|
|
26
28
|
from .host_surfaces import UiSurface, load_ui_surface
|
|
@@ -33,6 +35,8 @@ from .validation import fail
|
|
|
33
35
|
|
|
34
36
|
BUNDLED_WORLD = "server/world"
|
|
35
37
|
WORLD_SPEC = "schema/world.json"
|
|
38
|
+
SNAPSHOTS_DIR = "snapshots"
|
|
39
|
+
_SHA256 = re.compile(r"[0-9a-f]{64}")
|
|
36
40
|
#: Bounded because the cache is one shared disk and a build's memory is the supervisor's.
|
|
37
41
|
DEFAULT_MAX_BUNDLE_MB = 2048
|
|
38
42
|
MAX_BUNDLE_MEMBERS = 200_000
|
|
@@ -53,6 +57,41 @@ def world_tree(bundle: Path) -> Path:
|
|
|
53
57
|
)
|
|
54
58
|
|
|
55
59
|
|
|
60
|
+
def clone_mode(directory: Path) -> str:
|
|
61
|
+
"""``reflink`` when session copies on ``directory`` share blocks until written, ``copy`` when each one
|
|
62
|
+
copies its file: the disk decides, so the host probes it once and reports it in its stats."""
|
|
63
|
+
probe = directory / f".clone-probe-{uuid.uuid4().hex}"
|
|
64
|
+
clone = probe.with_name(f"{probe.name}.clone")
|
|
65
|
+
try:
|
|
66
|
+
probe.write_bytes(b"probe")
|
|
67
|
+
return "reflink" if reflink(probe, clone) else "copy"
|
|
68
|
+
finally:
|
|
69
|
+
probe.unlink(missing_ok=True)
|
|
70
|
+
clone.unlink(missing_ok=True)
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def data_tree(bundle: Path) -> Path:
|
|
74
|
+
return bundle if is_tools_bundle(bundle) else world_tree(bundle)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@dataclass(frozen=True)
|
|
78
|
+
class DataKey:
|
|
79
|
+
"""A version as a session opens it: on the version's own data, or on one snapshot of it, named by the
|
|
80
|
+
sha256 of the snapshot file. Two snapshots of one version never share a tree, a template or a session."""
|
|
81
|
+
|
|
82
|
+
version_id: str
|
|
83
|
+
sha256: str | None
|
|
84
|
+
|
|
85
|
+
@classmethod
|
|
86
|
+
def of(cls, version_id: str, snapshot: dict[str, str] | None) -> DataKey:
|
|
87
|
+
if not snapshot:
|
|
88
|
+
return cls(version_id, None)
|
|
89
|
+
sha256 = str(snapshot.get("sha256") or "").lower()
|
|
90
|
+
if not _SHA256.fullmatch(sha256):
|
|
91
|
+
fail("host", "a snapshot names the sha256 of its file (64 hex digits)")
|
|
92
|
+
return cls(version_id, sha256)
|
|
93
|
+
|
|
94
|
+
|
|
56
95
|
@dataclass
|
|
57
96
|
class CachedVersion:
|
|
58
97
|
"""One cached version, of either shape the host serves: an api bundle (a
|
|
@@ -124,8 +163,9 @@ class VersionCache:
|
|
|
124
163
|
#: Off in the supervisor: only a version worker imports world-authored code.
|
|
125
164
|
self.run_handlers = run_handlers
|
|
126
165
|
self.max_bundle_bytes = max_bundle_mb * (1 << 20)
|
|
166
|
+
self.clone_mode = clone_mode(self.root)
|
|
127
167
|
self._lock = threading.Lock()
|
|
128
|
-
self._versions: dict[
|
|
168
|
+
self._versions: dict[DataKey, CachedVersion] = {}
|
|
129
169
|
self._builds = Lanes()
|
|
130
170
|
|
|
131
171
|
def ensure(
|
|
@@ -134,37 +174,76 @@ class VersionCache:
|
|
|
134
174
|
bundle_url: str,
|
|
135
175
|
snapshot: dict[str, str] | None,
|
|
136
176
|
) -> CachedVersion:
|
|
137
|
-
|
|
177
|
+
key = DataKey.of(version_id, snapshot)
|
|
178
|
+
hit = self._cached(key)
|
|
138
179
|
if hit is not None:
|
|
139
180
|
return hit
|
|
140
181
|
with self._builds.turn(version_id):
|
|
141
|
-
return self._cached(
|
|
182
|
+
return self._cached(key) or self._store(key, self._build(key, bundle_url, snapshot))
|
|
142
183
|
|
|
143
|
-
def _cached(self,
|
|
184
|
+
def _cached(self, key: DataKey) -> CachedVersion | None:
|
|
144
185
|
with self._lock:
|
|
145
|
-
return self._versions.get(
|
|
186
|
+
return self._versions.get(key)
|
|
146
187
|
|
|
147
|
-
def _store(self, cached: CachedVersion) -> CachedVersion:
|
|
188
|
+
def _store(self, key: DataKey, cached: CachedVersion) -> CachedVersion:
|
|
148
189
|
with self._lock:
|
|
149
|
-
self._versions[
|
|
190
|
+
self._versions[key] = cached
|
|
150
191
|
return cached
|
|
151
192
|
|
|
152
|
-
def has(self, version_id: str) -> bool:
|
|
193
|
+
def has(self, version_id: str, snapshot: dict[str, str] | None = None) -> bool:
|
|
194
|
+
key = DataKey.of(version_id, snapshot)
|
|
153
195
|
with self._lock:
|
|
154
|
-
return
|
|
196
|
+
return key in self._versions
|
|
155
197
|
|
|
156
198
|
def version_ids(self) -> list[str]:
|
|
157
199
|
with self._lock:
|
|
158
|
-
return sorted(self._versions)
|
|
200
|
+
return sorted({key.version_id for key in self._versions})
|
|
159
201
|
|
|
160
|
-
def _build(self,
|
|
161
|
-
|
|
162
|
-
if not
|
|
163
|
-
self._download(bundle_url,
|
|
164
|
-
if
|
|
165
|
-
|
|
166
|
-
|
|
202
|
+
def _build(self, key: DataKey, bundle_url: str, snapshot: dict[str, str] | None) -> CachedVersion:
|
|
203
|
+
base = self.root / key.version_id / "bundle"
|
|
204
|
+
if not base.is_dir():
|
|
205
|
+
self._download(bundle_url, base)
|
|
206
|
+
if key.sha256 is None or snapshot is None:
|
|
207
|
+
return self._load(key.version_id, base, self.root / key.version_id / "template.db", fetched=False)
|
|
208
|
+
bundle = self._with_snapshot(key, base, snapshot)
|
|
209
|
+
return self._load(key.version_id, bundle, bundle.parent / "template.db", fetched=True)
|
|
210
|
+
|
|
211
|
+
def _with_snapshot(self, key: DataKey, base: Path, snapshot: dict[str, str]) -> Path:
|
|
212
|
+
"""The version's bundle with the snapshot's rows as its data, under ``snapshots/<sha256>``: cloned
|
|
213
|
+
from the downloaded bundle (never linked, so nothing written here reaches it) without the data the
|
|
214
|
+
snapshot replaces, then fetched and verified. Renamed into place only once complete; one built
|
|
215
|
+
before is reused without a url, which is how a version worker finds it."""
|
|
216
|
+
home = self.root / key.version_id / SNAPSHOTS_DIR / str(key.sha256)
|
|
217
|
+
if (home / "bundle").is_dir():
|
|
218
|
+
return home / "bundle"
|
|
219
|
+
url, digest = snapshot.get("url"), snapshot.get("digest")
|
|
220
|
+
if not url or not isinstance(digest, str):
|
|
221
|
+
fail(
|
|
222
|
+
"host",
|
|
223
|
+
f"snapshot {key.sha256} of version {key.version_id} is not cached on this host, "
|
|
224
|
+
"and the open names no url and digest to fetch it",
|
|
167
225
|
)
|
|
226
|
+
tree = data_tree(base)
|
|
227
|
+
contract = check_bundle(tree)
|
|
228
|
+
replaced = snapshots.default_snapshot(tree, contract)
|
|
229
|
+
staging = home.parent / f".{key.sha256}.{uuid.uuid4().hex}"
|
|
230
|
+
try:
|
|
231
|
+
for path in sorted(base.rglob("*")):
|
|
232
|
+
target = staging / "bundle" / path.relative_to(base)
|
|
233
|
+
if path.is_dir():
|
|
234
|
+
target.mkdir(parents=True, exist_ok=True)
|
|
235
|
+
elif path != replaced:
|
|
236
|
+
WorldDatabase.clone_file(path, target)
|
|
237
|
+
staged = staging / "bundle" / tree.relative_to(base)
|
|
238
|
+
snapshots.fetch(staged, contract, url, str(key.sha256), digest)
|
|
239
|
+
os.replace(staging, home)
|
|
240
|
+
finally:
|
|
241
|
+
shutil.rmtree(staging, ignore_errors=True)
|
|
242
|
+
return home / "bundle"
|
|
243
|
+
|
|
244
|
+
def _load(self, version_id: str, bundle: Path, template: Path, *, fetched: bool) -> CachedVersion:
|
|
245
|
+
if is_tools_bundle(bundle):
|
|
246
|
+
tools = load_tools_version(bundle, template, fetched=fetched, run_handlers=self.run_handlers)
|
|
168
247
|
return CachedVersion(
|
|
169
248
|
version_id=version_id,
|
|
170
249
|
bundle=bundle,
|
|
@@ -179,8 +258,6 @@ class VersionCache:
|
|
|
179
258
|
)
|
|
180
259
|
tree = world_tree(bundle)
|
|
181
260
|
contract = check_bundle(tree)
|
|
182
|
-
if snapshot:
|
|
183
|
-
snapshots.fetch(tree, contract, snapshot["url"], snapshot["sha256"], snapshot["digest"])
|
|
184
261
|
served = snapshots.ServedSnapshot.of(tree)
|
|
185
262
|
spec = load_spec(tree)
|
|
186
263
|
settings = load_api_settings(tree, spec)
|