@withgateway/sdk 0.4.0 → 0.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/dist/agents/index.d.mts +573 -0
  2. package/dist/agents/index.d.ts +573 -0
  3. package/dist/agents/index.js +1443 -0
  4. package/dist/agents/index.js.map +1 -0
  5. package/dist/agents/index.mjs +1394 -0
  6. package/dist/agents/index.mjs.map +1 -0
  7. package/dist/apiError-DHpfbmdx.d.mts +17 -0
  8. package/dist/apiError-DHpfbmdx.d.ts +17 -0
  9. package/dist/cli.js +5655 -1890
  10. package/dist/cli.js.map +1 -1
  11. package/dist/cli.mjs +5769 -2005
  12. package/dist/cli.mjs.map +1 -1
  13. package/dist/files/index.d.mts +3 -1
  14. package/dist/files/index.d.ts +3 -1
  15. package/dist/files/index.js +418 -120
  16. package/dist/files/index.js.map +1 -1
  17. package/dist/files/index.mjs +418 -120
  18. package/dist/files/index.mjs.map +1 -1
  19. package/dist/hooks/index.js +346 -136
  20. package/dist/hooks/index.js.map +1 -1
  21. package/dist/hooks/index.mjs +346 -136
  22. package/dist/hooks/index.mjs.map +1 -1
  23. package/dist/{index-DUHmS74p.d.ts → index-DIN14Lhg.d.ts} +27 -4
  24. package/dist/{index-mJhG3vZU.d.mts → index-DuB-eQ0Z.d.mts} +27 -4
  25. package/dist/index.d.mts +7 -4
  26. package/dist/index.d.ts +7 -4
  27. package/dist/index.js +2862 -438
  28. package/dist/index.js.map +1 -1
  29. package/dist/index.mjs +2820 -431
  30. package/dist/index.mjs.map +1 -1
  31. package/dist/media-CQUC7TkS.d.mts +39 -0
  32. package/dist/media-DQnBloaX.d.ts +39 -0
  33. package/dist/replay/index.d.mts +1 -1
  34. package/dist/replay/index.d.ts +1 -1
  35. package/dist/replay/index.js.map +1 -1
  36. package/dist/replay/index.mjs.map +1 -1
  37. package/dist/traces/index.d.mts +3 -1
  38. package/dist/traces/index.d.ts +3 -1
  39. package/dist/traces/index.js +418 -120
  40. package/dist/traces/index.js.map +1 -1
  41. package/dist/traces/index.mjs +418 -120
  42. package/dist/traces/index.mjs.map +1 -1
  43. package/dist/tracing/index.d.mts +3 -2
  44. package/dist/tracing/index.d.ts +3 -2
  45. package/dist/tracing/index.js +257 -16
  46. package/dist/tracing/index.js.map +1 -1
  47. package/dist/tracing/index.mjs +257 -16
  48. package/dist/tracing/index.mjs.map +1 -1
  49. package/dist/transport-CpRe7zgd.d.mts +123 -0
  50. package/dist/transport-CpRe7zgd.d.ts +123 -0
  51. package/dist/worlds/index.d.mts +2086 -1749
  52. package/dist/worlds/index.d.ts +2086 -1749
  53. package/dist/worlds/index.js +1471 -269
  54. package/dist/worlds/index.js.map +1 -1
  55. package/dist/worlds/index.mjs +1466 -265
  56. package/dist/worlds/index.mjs.map +1 -1
  57. package/package.json +6 -1
  58. package/runtime/gatewaysdk/environment/_bundle.py +4 -6
  59. package/runtime/gatewaysdk/environment/_world.py +24 -1
  60. package/runtime/gatewaysdk/environment/bundle_manifest.py +243 -0
  61. package/runtime/gatewaysdk/environment/runtime.py +45 -2
  62. package/runtime/gatewaysdk/environment/schema/__main__.py +2 -2
  63. package/runtime/gatewaysdk/environment/schema/_slack_fidelity.py +8 -23
  64. package/runtime/gatewaysdk/environment/schema/api.py +27 -7
  65. package/runtime/gatewaysdk/environment/schema/assertions.py +7 -1
  66. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +2 -0
  67. package/runtime/gatewaysdk/environment/schema/call_capture.py +13 -1
  68. package/runtime/gatewaysdk/environment/schema/call_log.py +66 -0
  69. package/runtime/gatewaysdk/environment/schema/call_redaction.py +6 -0
  70. package/runtime/gatewaysdk/environment/schema/compiler.py +4 -1
  71. package/runtime/gatewaysdk/environment/schema/conform.py +32 -8
  72. package/runtime/gatewaysdk/environment/schema/connector.py +5 -0
  73. package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +2 -1
  74. package/runtime/gatewaysdk/environment/schema/connectors/github/api/handlers.py +217 -58
  75. package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +9 -19
  76. package/runtime/gatewaysdk/environment/schema/connectors/github/scenarios/default/seed/github_webhooks.json +10 -2
  77. package/runtime/gatewaysdk/environment/schema/connectors/github/tests/webhooks.json +47 -0
  78. package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +21 -0
  79. package/runtime/gatewaysdk/environment/schema/connectors/jamf/api/handlers.py +185 -18
  80. package/runtime/gatewaysdk/environment/schema/connectors/jamf/connector.toml +29 -14
  81. package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +43 -12
  82. package/runtime/gatewaysdk/environment/schema/connectors/jamf/scenarios/default/seed/jamf_webhooks.json +2 -2
  83. package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +4 -17
  84. package/runtime/gatewaysdk/environment/schema/connectors/jamf/tests/webhooks.json +107 -0
  85. package/runtime/gatewaysdk/environment/schema/connectors/jamf/vendor-openapi.json +1 -1
  86. package/runtime/gatewaysdk/environment/schema/connectors/linear/api/handlers.py +85 -1
  87. package/runtime/gatewaysdk/environment/schema/connectors/linear/connector.toml +19 -31
  88. package/runtime/gatewaysdk/environment/schema/connectors/linear/provenance.json +18 -8
  89. package/runtime/gatewaysdk/environment/schema/connectors/linear/scenarios/default/seed/webhooks.json +19 -3
  90. package/runtime/gatewaysdk/environment/schema/connectors/linear/tests/test_webhooks.py +71 -0
  91. package/runtime/gatewaysdk/environment/schema/database/base.py +2 -2
  92. package/runtime/gatewaysdk/environment/schema/database/guard.py +2 -1
  93. package/runtime/gatewaysdk/environment/schema/handler_errors.py +27 -1
  94. package/runtime/gatewaysdk/environment/schema/host.py +74 -11
  95. package/runtime/gatewaysdk/environment/schema/host_browser.py +48 -0
  96. package/runtime/gatewaysdk/environment/schema/host_control.py +33 -2
  97. package/runtime/gatewaysdk/environment/schema/host_handoff.py +4 -1
  98. package/runtime/gatewaysdk/environment/schema/host_requests.py +14 -0
  99. package/runtime/gatewaysdk/environment/schema/host_session.py +13 -2
  100. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +6 -0
  101. package/runtime/gatewaysdk/environment/schema/host_versions.py +96 -19
  102. package/runtime/gatewaysdk/environment/schema/journal.py +10 -4
  103. package/runtime/gatewaysdk/environment/schema/local.py +33 -7
  104. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +20 -5
  105. package/runtime/gatewaysdk/environment/schema/page_checks.py +123 -0
  106. package/runtime/gatewaysdk/environment/schema/platform.py +33 -480
  107. package/runtime/gatewaysdk/environment/schema/platform_templates.py +332 -0
  108. package/runtime/gatewaysdk/environment/schema/request_ledger.py +4 -1
  109. package/runtime/gatewaysdk/environment/schema/run_parameters.py +25 -0
  110. package/runtime/gatewaysdk/environment/schema/runtime.py +49 -4
  111. package/runtime/gatewaysdk/environment/schema/scaffold.py +369 -218
  112. package/runtime/gatewaysdk/environment/schema/session_files.py +13 -2
  113. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +1 -1
  114. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +3 -3
  115. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +20 -24
  116. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +11 -11
  117. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +9 -9
  118. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +45 -12
  119. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +40 -9
  120. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +17 -2
  121. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/new-world-sandbox.md +1 -1
  122. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/ui-map.md +3 -2
  123. package/runtime/gatewaysdk/environment/schema/slack.py +55 -10
  124. package/runtime/gatewaysdk/environment/schema/snapshot.py +30 -4
  125. package/runtime/gatewaysdk/environment/schema/tasks.py +25 -2
  126. package/runtime/gatewaysdk/environment/schema/template_record.py +131 -0
  127. package/runtime/gatewaysdk/environment/schema/tools_world.py +26 -5
  128. package/runtime/gatewaysdk/environment/schema/ui.py +54 -33
  129. package/runtime/gatewaysdk/environment/schema/ui_interactions.py +419 -0
  130. package/runtime/gatewaysdk/environment/schema/ui_recorder.js +895 -0
  131. package/runtime/gatewaysdk/environment/schema/ui_replay.py +366 -0
  132. package/runtime/gatewaysdk/environment/schema/ui_replay_record.LICENSE +25 -0
  133. package/runtime/gatewaysdk/environment/schema/ui_replay_record.js +109 -0
  134. package/runtime/gatewaysdk/environment/schema/validation.py +10 -1
  135. package/runtime/gatewaysdk/environment/schema/verbs.py +6 -0
  136. package/runtime/gatewaysdk/environment/schema/webhooks.py +687 -0
  137. package/runtime/gatewaysdk/environment/schema/worker.py +5 -1
  138. package/runtime/gatewaysdk/environment/schema/workers.py +8 -1
  139. package/runtime/gatewaysdk/environment/schema/world_host.py +64 -18
  140. package/runtime/gatewaysdk/environment/schema/world_tests.py +74 -0
  141. package/runtime/gatewaysdk/environment/secret_values.py +1 -1
  142. package/runtime/gatewaysdk/environment/world.py +34 -43
  143. package/runtime/gatewaysdk/environment/world_session_contract.py +7 -34
  144. package/skills/benchmarks-getting-started/SKILL.md +1 -1
  145. package/skills/connector-schema-authoring/CONNECTORS.md +3 -3
  146. package/skills/connector-schema-authoring/SKILL.md +20 -24
  147. package/skills/connector-schema-authoring/connector-worlds.md +155 -13
  148. package/skills/connector-schema-authoring/world-schema.md +36 -29
  149. package/skills/world-data-ingestion/SKILL.md +11 -11
  150. package/skills/worlds-getting-started/SKILL.md +9 -9
  151. package/skills/worlds-getting-started/references/cli-mcp-map.md +45 -12
  152. package/skills/worlds-getting-started/references/gotchas.md +40 -9
  153. package/skills/worlds-getting-started/references/journal-checks.md +17 -2
  154. package/skills/worlds-getting-started/references/new-world-sandbox.md +1 -1
  155. package/skills/worlds-getting-started/references/ui-map.md +3 -2
  156. package/dist/media-BIApiPzS.d.mts +0 -102
  157. package/dist/media-BIApiPzS.d.ts +0 -102
@@ -50,15 +50,18 @@ import signal
50
50
  import sys
51
51
  import threading
52
52
  import urllib.parse
53
+ from functools import partial
53
54
  from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
54
55
  from pathlib import Path
55
56
  from typing import Any
56
57
 
57
58
  from .api import Via, with_query
58
59
  from .call_capture import JSON_TYPE, send_response
60
+ from .call_redaction import CallRedaction
59
61
  from .credentials import BearerAuth
60
62
  from .door import PLATFORM_HEADER, Entry
61
63
  from .handler_errors import log_request_failure, request_failure_body
64
+ from .host_browser import RelayNavigation, log_relay
62
65
  from .host_control import ControlPlane
63
66
  from .host_handoff import DrainHandoff, HandoffDoor, PlatformHandoff
64
67
  from .host_loops import HEARTBEAT_S, Heartbeat, start_lease_sweeper
@@ -90,9 +93,12 @@ from .host_versions import DEFAULT_MAX_BUNDLE_MB
90
93
  from .host_workers import UNKNOWN_SESSION, refusal
91
94
  from .request_ledger import LedgerFlusher, LedgerSpill, NoLedger, PlatformLedger
92
95
  from .session_move import SessionMoved
96
+ from .ui_interactions import RECORDER_PATH, UiRecorder
97
+ from .ui_replay import REPLAY_PATH, REPLAY_SCRIPT_PATH, REPLAY_STOP_S, ReplayShipper, UiReplay
93
98
  from .validation import ContractError
94
99
  from .workers import (
95
100
  DEFAULT_WORKER_BUDGET_MB,
101
+ DEFAULT_WORKER_FILE_MB,
96
102
  DEFAULT_WORKER_FOOTPRINT_MB,
97
103
  DEFAULT_WORKER_MEMORY_MB,
98
104
  WorkerPool,
@@ -127,13 +133,15 @@ def make_host_handler(
127
133
  *,
128
134
  max_body_mb: int = DEFAULT_MAX_BODY_MB,
129
135
  secure_cookies: bool = True,
136
+ navigation: RelayNavigation | None = None,
137
+ replays: ReplayShipper | None = None,
130
138
  ):
131
139
  """The request handler class for one host; its control plane is :class:`ControlPlane`, which
132
140
  refuses an empty ``secret`` or ``worlds_domain``. ``secure_cookies`` is off
133
141
  only for a host whose public surface is reached over plain http (a dev
134
142
  host); it is the deployment's word, not the request's, since a client
135
143
  writes the forwarding headers a request arrives with."""
136
- control = ControlPlane(host, secret, HOSTED_OPERATIONS, worlds_domain)
144
+ control = ControlPlane(host, secret, HOSTED_OPERATIONS, worlds_domain, navigation)
137
145
  if max_body_mb <= 0:
138
146
  raise ValueError(f"the request body cap must be positive (WORLDS_HOST_MAX_BODY_MB={max_body_mb})")
139
147
  max_body_bytes = max_body_mb * (1 << 20)
@@ -318,12 +326,23 @@ def make_host_handler(
318
326
  ):
319
327
  return self._reply(401, {"error": "invalid session token"})
320
328
  route = api_target(ui, url.path)
321
- if route is None:
322
- return self._serve_static(ui, method, url, opened)
329
+ if url.path == RECORDER_PATH:
330
+ return self._serve_file(method, opened, UiRecorder.script)
331
+ if url.path == REPLAY_SCRIPT_PATH:
332
+ return self._serve_file(method, opened, partial(UiReplay.script, replays))
333
+ static = route is None and not UiRecorder.serves(url.path)
334
+ if static and session.worker is not None and UiRecorder.is_page(url.path):
335
+ return self._on_worker(session, method, headers)
336
+ if static:
337
+ return self._serve_static(session, ui, method, url, opened)
323
338
  if cross_site(headers, lambda origin: urllib.parse.urlsplit(origin).netloc == headers.get("host")):
324
339
  return self._reply(403, {"error": "the ui's api prefix answers its own page, not another site"})
340
+ if url.path == REPLAY_PATH:
341
+ return self._record_replay(session, method)
325
342
  if session.worker is not None:
326
343
  return self._on_worker(session, method, headers)
344
+ if route is None:
345
+ return self._record_interactions(session, method, url, headers)
327
346
  target = urllib.parse.urlunsplit(("", "", route, url.query, ""))
328
347
  api_headers = {k: v for k, v in headers.items() if k not in ("cookie", "authorization")}
329
348
  api_headers["host"] = session.api_host(worlds_domain, opened)
@@ -334,20 +353,49 @@ def make_host_handler(
334
353
  cookie = session_cookie(token, secure=secure_cookies, max_age_s=host.lease_s)
335
354
  self._reply(303, {"location": landing}, {"Location": landing, "Set-Cookie": cookie})
336
355
 
337
- def _serve_static(self, ui: UiSurface, method: str, url, opened: bool) -> None:
338
- """A file of the ui; on a capability host with no ``Referer``, so a page that
339
- loads another site's resource never hands it the capability."""
356
+ def _serve_static(self, session: Session, ui: UiSurface, method: str, url, opened: bool) -> None:
357
+ """A file of the ui; a page carries the recorder and is noted as served, a note for the
358
+ ledger alone (no grader reads it), so a page is never held by the move fence."""
340
359
  if method != "GET":
341
360
  return self._reply(405, {"error": "the static ui answers GET only"})
342
- status, static_headers, payload = static_file(ui, url.path)
343
- sealed = {**static_headers, **(NO_REFERRER if opened else {})}
361
+ status, file_headers, payload = static_file(ui, url.path)
362
+ page = UiRecorder.page(status, file_headers, payload)
363
+ if page is not None and session.runtime is not None:
364
+ UiRecorder.served(url.path, session.runtime.calls)
365
+ self._send_file(opened, status, file_headers, payload if page is None else page)
366
+
367
+ def _serve_file(self, method: str, opened: bool, answer) -> None:
368
+ if method != "GET":
369
+ return self._reply(405, {"error": "the static ui answers GET only"})
370
+ self._send_file(opened, *answer())
371
+
372
+ def _send_file(self, opened: bool, status: int, headers: dict[str, str], payload: bytes) -> None:
373
+ """On a capability host with no ``Referer``, so a page that loads another site's
374
+ resource never hands it the capability."""
375
+ sealed = {**headers, **(NO_REFERRER if opened else {})}
344
376
  self._relay(status, sealed.items(), payload, drop=FRAMING_HEADERS)
345
377
 
378
+ def _record_interactions(self, session: Session, method: str, url, headers: dict[str, str]) -> None:
379
+ if session.runtime is None:
380
+ return self._reply(404, {"error": "this session's world is not served here"})
381
+ raw = self._read()
382
+ with host.admitted(session, HOP_HEADER in headers):
383
+ answer = UiRecorder.record(method, url.path, raw, session.runtime.calls)
384
+ self._reply(*answer)
385
+
386
+ def _record_replay(self, session: Session, method: str) -> None:
387
+ """The host that holds the session keeps its page's recording, a worker's session too, scrubbed
388
+ as the call log scrubs (:class:`CallRedaction`)."""
389
+ scrub = CallRedaction.of(session.spec, session.token).scrub
390
+ self._reply(*UiReplay.take(method, self._read(), session.session_id, scrub, replays))
391
+
346
392
  def _serve_browser(self, session: Session, url, headers: dict[str, str]) -> None:
347
393
  """``/cdp/<session token>`` and ``/live/<grant>`` on the session's
348
394
  cdp host: check the credential against the SESSION, then tunnel
349
395
  the websocket to the leased relay with the POOL credential the
350
- relay knows. The relay's token never leaves the host."""
396
+ relay knows. The relay's token never leaves the host. The relay is
397
+ bound to the session's hosts first, so one that restarted and lost
398
+ its binding is never reached unbound."""
351
399
  relay = session.browser
352
400
  if relay is None:
353
401
  return self._reply(404, {"error": "this session has no browser surface"})
@@ -368,11 +416,16 @@ def make_host_handler(
368
416
  upstream_path = f"/live/{mint_live_grant(relay.token, LIVE_RELAY_GRANT_MS, scope)}"
369
417
  if headers.get("upgrade", "").lower() != "websocket":
370
418
  return self._reply(400, {"error": "the browser surface is a websocket"})
419
+ try:
420
+ control.navigation.bind(relay, session.browser_hosts(control.domain))
421
+ except ContractError as error:
422
+ return self._reply(502, {"error": "browser unbound", "detail": str(error)})
371
423
  self.close_connection = True
372
424
  try:
373
425
  tunnel(self.connection, relay.url, upstream_path, headers)
374
426
  except OSError as error:
375
- self._reply(502, {"error": "browser unreachable", "detail": str(error)})
427
+ log_relay("browser relay unreachable", relay, error=str(error))
428
+ self._reply(502, {"error": "browser unreachable"})
376
429
 
377
430
  return Handler
378
431
 
@@ -397,12 +450,14 @@ class HostServer(ThreadingHTTPServer):
397
450
  drain_s: float = DEFAULT_DRAIN_S,
398
451
  ledger: LedgerFlusher | None = None,
399
452
  handoff: HandoffDoor | None = None,
453
+ replays: ReplayShipper | None = None,
400
454
  ):
401
455
  super().__init__(address, handler)
402
456
  self.world_host = world_host
403
457
  self.lease_sweeper = start_lease_sweeper(world_host)
404
458
  self.heartbeat = Heartbeat(world_host, *heartbeat, interval_s=heartbeat_s).start() if heartbeat else None
405
459
  world_host.ledger = ledger.start() if ledger else NoLedger()
460
+ self.replays = replays.start() if replays else None
406
461
  self.handoff = DrainHandoff(world_host, handoff, drain_s)
407
462
  self._answering = 0
408
463
  self._draining = False
@@ -457,6 +512,7 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
457
512
  ``WORLDS_WORKER_MEMORY_MB`` caps each version worker's address space, ``WORLDS_WORKER_BUDGET_MB`` is the memory
458
513
  all of them together are held to and ``WORLDS_WORKER_FOOTPRINT_MB`` what each is counted at against it (unset,
459
514
  its whole address-space cap; their count is the quotient, and one more version answers 503 ``at_capacity``),
515
+ ``WORLDS_WORKER_FILE_MB`` is the largest file a worker may write (a session's database, so a world state's data),
460
516
  ``WORLDS_HOST_OBJECT_STORE`` is where every presigned URL it is handed may point (required; ``any`` for a dev
461
517
  host), ``WORLDS_HOST_MAX_BUNDLE_MB`` is the largest version bundle the cache accepts,
462
518
  ``WORLDS_HOST_MAX_BODY_MB`` the largest request body it reads, ``WORLDS_HOST_LEDGER_SPILL_DIR`` (under the cache
@@ -491,6 +547,7 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
491
547
  memory_mb=int(os.environ.get("WORLDS_WORKER_MEMORY_MB") or DEFAULT_WORKER_MEMORY_MB),
492
548
  budget_mb=int(os.environ.get("WORLDS_WORKER_BUDGET_MB") or DEFAULT_WORKER_BUDGET_MB),
493
549
  footprint_mb=int(footprint) if footprint else DEFAULT_WORKER_FOOTPRINT_MB,
550
+ file_mb=int(os.environ.get("WORLDS_WORKER_FILE_MB") or DEFAULT_WORKER_FILE_MB),
494
551
  )
495
552
  owners = PlatformOwners(platform_url, secret, advertise) if platform_url else None
496
553
  world_host = WorldHost(
@@ -504,6 +561,7 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
504
561
  build=os.environ.get("WORLDS_HOST_BUILD") or None,
505
562
  owners=owners,
506
563
  )
564
+ replays = ReplayShipper(platform_url, secret) if platform_url else None
507
565
  handler = make_host_handler(
508
566
  world_host,
509
567
  secret=secret,
@@ -511,6 +569,7 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
511
569
  owner_lookup=owners,
512
570
  max_body_mb=int(os.environ.get("WORLDS_HOST_MAX_BODY_MB") or DEFAULT_MAX_BODY_MB),
513
571
  secure_cookies=os.environ.get("WORLDS_HOST_PLAIN_HTTP", "0") != "1",
572
+ replays=replays,
514
573
  )
515
574
  sink = PlatformLedger(platform_url, secret, host_url=advertise) if platform_url else None
516
575
  return HostServer(
@@ -522,6 +581,7 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
522
581
  heartbeat_s=float(os.environ.get("WORLDS_HOST_HEARTBEAT_S") or HEARTBEAT_S),
523
582
  drain_s=float(os.environ.get("WORLDS_HOST_DRAIN_S") or DEFAULT_DRAIN_S),
524
583
  handoff=PlatformHandoff(platform_url, secret) if platform_url and advertise else None,
584
+ replays=replays,
525
585
  )
526
586
 
527
587
 
@@ -534,7 +594,8 @@ def main(argv: list[str] | None = None) -> int:
534
594
  args = parser.parse_args(argv)
535
595
  server = serve(args.cache_root, port=args.port)
536
596
  print(
537
- f"world host listening on :{server.server_address[1]}, cache {args.cache_root}",
597
+ f"world host listening on :{server.server_address[1]}, cache {args.cache_root} "
598
+ f"(session copies: {server.world_host.cache.clone_mode})",
538
599
  file=sys.stderr,
539
600
  flush=True,
540
601
  )
@@ -543,6 +604,8 @@ def main(argv: list[str] | None = None) -> int:
543
604
  server.serve_forever()
544
605
  finally:
545
606
  server.world_host.ledger.stop(settle=server.world_host.finishing.settle)
607
+ if server.replays is not None:
608
+ server.replays.stop(REPLAY_STOP_S)
546
609
  server.lease_sweeper.set()
547
610
  if server.heartbeat is not None:
548
611
  server.heartbeat.stop()
@@ -0,0 +1,48 @@
1
+ """Binds a leased pool browser to the one session it serves: the relay's egress proxy then
2
+ loads that session's hosts and nothing else (deploy/world-browser/navigation.mjs)."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import json
7
+ import sys
8
+ import time
9
+ import urllib.error
10
+ import urllib.request
11
+ from collections.abc import Callable
12
+ from dataclasses import dataclass, field
13
+ from typing import Any
14
+
15
+ from .host_requests import BrowserRelay
16
+ from .validation import fail
17
+
18
+ GUARD_REQUIRED_FROM = 1794787200.0
19
+ BIND_TIMEOUT_S = 5.0
20
+
21
+
22
+ @dataclass(frozen=True)
23
+ class RelayNavigation:
24
+ opener: Callable[..., Any] = field(default=urllib.request.urlopen)
25
+ clock: Callable[[], float] = field(default=time.time)
26
+
27
+ def bind(self, relay: BrowserRelay, hosts: list[str]) -> str:
28
+ request = urllib.request.Request(
29
+ f"{relay.http_url}/navigation",
30
+ data=json.dumps({"hosts": hosts}).encode(),
31
+ method="PUT",
32
+ headers={"authorization": f"Bearer {relay.token}", "content-type": "application/json"},
33
+ )
34
+ try:
35
+ with self.opener(request, timeout=BIND_TIMEOUT_S) as answer:
36
+ return str(json.loads(answer.read() or b"{}").get("navigation", "bound"))
37
+ except urllib.error.HTTPError as error:
38
+ if error.code == 404 and self.clock() < GUARD_REQUIRED_FROM:
39
+ log_relay("browser relay predates the navigation guard", relay)
40
+ return "unguarded"
41
+ fail("host", f"the browser could not be bound to this session (HTTP {error.code})")
42
+ except (OSError, ValueError) as error:
43
+ log_relay("browser relay could not be bound", relay, error=str(error))
44
+ fail("host", "the browser could not be bound to this session: its relay did not answer as expected")
45
+
46
+
47
+ def log_relay(msg: str, relay: BrowserRelay, **extra: str) -> None:
48
+ print(json.dumps({"msg": msg, "relay": relay.url, **extra}), file=sys.stderr, flush=True)
@@ -15,6 +15,8 @@ from email.message import Message
15
15
  from typing import TYPE_CHECKING, Any
16
16
 
17
17
  from .handler_errors import carried
18
+ from .host_browser import RelayNavigation
19
+ from .host_requests import BrowserRelay
18
20
  from .host_surfaces import public_host
19
21
  from .host_workers import WORKER_EXITED, refusal
20
22
  from .validation import ContractError, fail
@@ -35,10 +37,18 @@ class ControlPlane:
35
37
  """One host's control plane. A host with no secret has no control plane at all rather than an
36
38
  open one, and one with no worlds domain could not tell its own address from a session's."""
37
39
 
38
- def __init__(self, host: WorldHost, secret: str, operations: Mapping[str, Callable[..., dict]], worlds_domain: str):
40
+ def __init__(
41
+ self,
42
+ host: WorldHost,
43
+ secret: str,
44
+ operations: Mapping[str, Callable[..., dict]],
45
+ worlds_domain: str,
46
+ navigation: RelayNavigation | None = None,
47
+ ):
39
48
  if not secret:
40
49
  raise ValueError("the world host needs a control-plane secret (WORLDS_HOST_SECRET)")
41
50
  self.host = host
51
+ self.navigation = navigation or RelayNavigation()
42
52
  self.operations = operations
43
53
  self.domain = self.domain_of(worlds_domain)
44
54
  self._secret = secret.encode()
@@ -109,6 +119,20 @@ class ControlPlane:
109
119
  return 404, {"error": "unknown control route"}
110
120
 
111
121
  def _open(self, body: dict) -> dict:
122
+ answer = self._open_session(body)
123
+ relay = BrowserRelay.parse(body.get("browser"))
124
+ if relay is not None:
125
+ self._bind_or_close(body["sessionId"], relay)
126
+ return answer
127
+
128
+ def _bind_or_close(self, session_id: str, relay: BrowserRelay) -> None:
129
+ try:
130
+ self.navigation.bind(relay, self.host.browser_hosts(session_id, self.domain))
131
+ except Exception:
132
+ self.host.close(session_id, "browser_unbound")
133
+ raise
134
+
135
+ def _open_session(self, body: dict) -> dict:
112
136
  return self.host.open(
113
137
  session_id=body["sessionId"],
114
138
  version_id=body["versionId"],
@@ -123,14 +147,21 @@ class ControlPlane:
123
147
  pending=body.get("pending") is True,
124
148
  auth=body.get("auth"),
125
149
  ledger_source=body.get("ledgerSource"),
150
+ parameters=body.get("parameters"),
126
151
  )
127
152
 
153
+ def _swap_browser(self, session_id: str, browser: Any) -> dict:
154
+ relay = BrowserRelay.parse(browser)
155
+ if relay is not None:
156
+ self.navigation.bind(relay, self.host.browser_hosts(session_id, self.domain))
157
+ return self.host.set_browser(session_id, browser)
158
+
128
159
  def _session(self, method: str, parts: list[str], body: dict) -> Answer:
129
160
  sid = parts[2]
130
161
  self.host.forwarding.refuse(sid)
131
162
  verb = parts[3] if len(parts) > 3 else None
132
163
  if method == "PUT" and verb == "browser" and len(parts) == 4:
133
- return 200, self.host.set_browser(sid, body.get("browser"))
164
+ return 200, self._swap_browser(sid, body.get("browser"))
134
165
  if method == "POST" and verb == "export" and len(parts) == 5:
135
166
  return self._upload(sid, parts[4], body.get("url"))
136
167
  row = session_verb(method, verb)
@@ -31,6 +31,7 @@ from .handler_errors import log_line
31
31
  from .host_owners import PROXY_TIMEOUT_S
32
32
  from .request_ledger import LEDGER_STOP_S
33
33
  from .session_move import same_host
34
+ from .ui_replay import REPLAY_STOP_S
34
35
 
35
36
  if TYPE_CHECKING:
36
37
  from .host_session import Session
@@ -47,7 +48,9 @@ ENLIST_S = 1.0
47
48
  LINGER_S = 5.0
48
49
  IN_FLIGHT_S = PROXY_TIMEOUT_S
49
50
  EXIT_S = 110.0
50
- assert LINGER_S + IN_FLIGHT_S + LEDGER_STOP_S + 5.0 <= EXIT_S, "a stopping host's exit fits the grace past its drain"
51
+ assert (
52
+ LINGER_S + IN_FLIGHT_S + LEDGER_STOP_S + REPLAY_STOP_S + 5.0 <= EXIT_S
53
+ ), "a stopping host's exit fits the grace past its drain"
51
54
 
52
55
  MOVED = "moved"
53
56
  STAYED = "stayed"
@@ -28,6 +28,16 @@ class DependencyRef:
28
28
  snapshot: dict[str, str] | None
29
29
  raw: dict[str, Any]
30
30
 
31
+ @property
32
+ def for_worker(self) -> dict[str, Any]:
33
+ """The entry a version worker is handed: its snapshot without the url, a tree the supervisor built."""
34
+ return {**self.raw, "snapshot": worker_snapshot(self.snapshot)}
35
+
36
+
37
+ def worker_snapshot(snapshot: dict[str, str] | None) -> dict[str, str] | None:
38
+ """A snapshot as a version worker sees it: its sha256 and digest, never the presigned url."""
39
+ return {key: value for key, value in snapshot.items() if key != "url"} if snapshot else None
40
+
31
41
 
32
42
  def parse_dependency_refs(dependencies: Any) -> list[DependencyRef]:
33
43
  if dependencies is None:
@@ -77,6 +87,10 @@ class BrowserRelay:
77
87
  raise AssertionError # fail() never returns
78
88
  return cls(url.rstrip("/"), token)
79
89
 
90
+ @property
91
+ def http_url(self) -> str:
92
+ return "http://" + self.url.removeprefix("ws://")
93
+
80
94
 
81
95
  def restore_sha256(restore: object) -> str | None:
82
96
  """The sha256 of the export an open restores from, or None for an open that restores nothing."""
@@ -10,7 +10,7 @@ from pathlib import Path
10
10
  from typing import TYPE_CHECKING
11
11
 
12
12
  from .door import OPEN_MODE, VENDOR_MODE, Door, same
13
- from .host_surfaces import SessionHost, capability_host
13
+ from .host_surfaces import SessionHost, capability_host, session_host
14
14
  from .request_ledger import LedgerLife
15
15
  from .session_move import SessionMove
16
16
 
@@ -75,6 +75,8 @@ class Session:
75
75
  placement: dict | None = None
76
76
  settings: ApiSettings | None = None
77
77
  restore_sha256: str | None = None
78
+ snapshot_sha256: str | None = None
79
+ parameters: dict[str, str] = field(default_factory=dict)
78
80
  pending: InitVar[bool] = False
79
81
  move: SessionMove = field(init=False, repr=False, compare=False)
80
82
  auth: str = VENDOR_MODE
@@ -119,8 +121,17 @@ class Session:
119
121
  capability = self.door.capability if self.door is not None else None
120
122
  return self.auth == OPEN_MODE and capability is not None and same(named.capability, capability)
121
123
 
124
+ def browser_hosts(self, domain: str) -> list[str]:
125
+ """The only hosts this session's browser may load: its ui and api, and their capability
126
+ pair when the session is open."""
127
+ hosts = [session_host(self.session_id, surface, domain) for surface in ("ui", "api")]
128
+ capability = self.door.capability if self.door is not None else None
129
+ if self.auth == OPEN_MODE and capability is not None:
130
+ hosts += [capability_host(self.session_id, capability, surface, domain) for surface in ("ui", "api")]
131
+ return hosts
132
+
122
133
  def api_host(self, domain: str, opened: bool) -> str:
123
134
  """The host a link the world writes points at when a request names none."""
124
135
  if opened and self.door is not None and self.door.capability is not None:
125
136
  return capability_host(self.session_id, self.door.capability, "api", domain)
126
- return f"w-{self.session_id}-api.{domain}"
137
+ return session_host(self.session_id, "api", domain)
@@ -260,6 +260,12 @@ def _capability_host(rest: str, surface: str) -> SessionHost | None:
260
260
  return SessionHost(session_id, surface, capability)
261
261
 
262
262
 
263
+ def session_host(session_id: str, surface: str, domain: str) -> str:
264
+ """The session's own hostname for ``surface`` (``api`` or ``ui``)."""
265
+ suffix = "-api" if surface == "api" else ""
266
+ return f"w-{session_id}{suffix}.{domain}"
267
+
268
+
263
269
  def capability_host(session_id: str, capability: str, surface: str, domain: str) -> str:
264
270
  """The capability hostname of an open session's ``surface`` (``api`` or ``ui``)."""
265
271
  suffix = "-api" if surface == "api" else ""
@@ -4,6 +4,7 @@
4
4
  from __future__ import annotations
5
5
 
6
6
  import os
7
+ import re
7
8
  import shutil
8
9
  import tarfile
9
10
  import threading
@@ -21,6 +22,7 @@ from .api import ApiSettings, Handler, Router, load_api_settings, load_handlers
21
22
  from .compiler import check_bundle
22
23
  from .connector import Spec, load_spec
23
24
  from .database import WorldDatabase
25
+ from .database.base import reflink
24
26
  from .dependencies import Dependency
25
27
  from .door import Door
26
28
  from .host_surfaces import UiSurface, load_ui_surface
@@ -33,6 +35,8 @@ from .validation import fail
33
35
 
34
36
  BUNDLED_WORLD = "server/world"
35
37
  WORLD_SPEC = "schema/world.json"
38
+ SNAPSHOTS_DIR = "snapshots"
39
+ _SHA256 = re.compile(r"[0-9a-f]{64}")
36
40
  #: Bounded because the cache is one shared disk and a build's memory is the supervisor's.
37
41
  DEFAULT_MAX_BUNDLE_MB = 2048
38
42
  MAX_BUNDLE_MEMBERS = 200_000
@@ -53,6 +57,41 @@ def world_tree(bundle: Path) -> Path:
53
57
  )
54
58
 
55
59
 
60
+ def clone_mode(directory: Path) -> str:
61
+ """``reflink`` when session copies on ``directory`` share blocks until written, ``copy`` when each one
62
+ copies its file: the disk decides, so the host probes it once and reports it in its stats."""
63
+ probe = directory / f".clone-probe-{uuid.uuid4().hex}"
64
+ clone = probe.with_name(f"{probe.name}.clone")
65
+ try:
66
+ probe.write_bytes(b"probe")
67
+ return "reflink" if reflink(probe, clone) else "copy"
68
+ finally:
69
+ probe.unlink(missing_ok=True)
70
+ clone.unlink(missing_ok=True)
71
+
72
+
73
+ def data_tree(bundle: Path) -> Path:
74
+ return bundle if is_tools_bundle(bundle) else world_tree(bundle)
75
+
76
+
77
+ @dataclass(frozen=True)
78
+ class DataKey:
79
+ """A version as a session opens it: on the version's own data, or on one snapshot of it, named by the
80
+ sha256 of the snapshot file. Two snapshots of one version never share a tree, a template or a session."""
81
+
82
+ version_id: str
83
+ sha256: str | None
84
+
85
+ @classmethod
86
+ def of(cls, version_id: str, snapshot: dict[str, str] | None) -> DataKey:
87
+ if not snapshot:
88
+ return cls(version_id, None)
89
+ sha256 = str(snapshot.get("sha256") or "").lower()
90
+ if not _SHA256.fullmatch(sha256):
91
+ fail("host", "a snapshot names the sha256 of its file (64 hex digits)")
92
+ return cls(version_id, sha256)
93
+
94
+
56
95
  @dataclass
57
96
  class CachedVersion:
58
97
  """One cached version, of either shape the host serves: an api bundle (a
@@ -124,8 +163,9 @@ class VersionCache:
124
163
  #: Off in the supervisor: only a version worker imports world-authored code.
125
164
  self.run_handlers = run_handlers
126
165
  self.max_bundle_bytes = max_bundle_mb * (1 << 20)
166
+ self.clone_mode = clone_mode(self.root)
127
167
  self._lock = threading.Lock()
128
- self._versions: dict[str, CachedVersion] = {}
168
+ self._versions: dict[DataKey, CachedVersion] = {}
129
169
  self._builds = Lanes()
130
170
 
131
171
  def ensure(
@@ -134,37 +174,76 @@ class VersionCache:
134
174
  bundle_url: str,
135
175
  snapshot: dict[str, str] | None,
136
176
  ) -> CachedVersion:
137
- hit = self._cached(version_id)
177
+ key = DataKey.of(version_id, snapshot)
178
+ hit = self._cached(key)
138
179
  if hit is not None:
139
180
  return hit
140
181
  with self._builds.turn(version_id):
141
- return self._cached(version_id) or self._store(self._build(version_id, bundle_url, snapshot))
182
+ return self._cached(key) or self._store(key, self._build(key, bundle_url, snapshot))
142
183
 
143
- def _cached(self, version_id: str) -> CachedVersion | None:
184
+ def _cached(self, key: DataKey) -> CachedVersion | None:
144
185
  with self._lock:
145
- return self._versions.get(version_id)
186
+ return self._versions.get(key)
146
187
 
147
- def _store(self, cached: CachedVersion) -> CachedVersion:
188
+ def _store(self, key: DataKey, cached: CachedVersion) -> CachedVersion:
148
189
  with self._lock:
149
- self._versions[cached.version_id] = cached
190
+ self._versions[key] = cached
150
191
  return cached
151
192
 
152
- def has(self, version_id: str) -> bool:
193
+ def has(self, version_id: str, snapshot: dict[str, str] | None = None) -> bool:
194
+ key = DataKey.of(version_id, snapshot)
153
195
  with self._lock:
154
- return version_id in self._versions
196
+ return key in self._versions
155
197
 
156
198
  def version_ids(self) -> list[str]:
157
199
  with self._lock:
158
- return sorted(self._versions)
200
+ return sorted({key.version_id for key in self._versions})
159
201
 
160
- def _build(self, version_id: str, bundle_url: str, snapshot: dict[str, str] | None) -> CachedVersion:
161
- bundle = self.root / version_id / "bundle"
162
- if not bundle.is_dir():
163
- self._download(bundle_url, bundle)
164
- if is_tools_bundle(bundle):
165
- tools = load_tools_version(
166
- bundle, self.root / version_id / "template.db", snapshot, run_handlers=self.run_handlers
202
+ def _build(self, key: DataKey, bundle_url: str, snapshot: dict[str, str] | None) -> CachedVersion:
203
+ base = self.root / key.version_id / "bundle"
204
+ if not base.is_dir():
205
+ self._download(bundle_url, base)
206
+ if key.sha256 is None or snapshot is None:
207
+ return self._load(key.version_id, base, self.root / key.version_id / "template.db", fetched=False)
208
+ bundle = self._with_snapshot(key, base, snapshot)
209
+ return self._load(key.version_id, bundle, bundle.parent / "template.db", fetched=True)
210
+
211
+ def _with_snapshot(self, key: DataKey, base: Path, snapshot: dict[str, str]) -> Path:
212
+ """The version's bundle with the snapshot's rows as its data, under ``snapshots/<sha256>``: cloned
213
+ from the downloaded bundle (never linked, so nothing written here reaches it) without the data the
214
+ snapshot replaces, then fetched and verified. Renamed into place only once complete; one built
215
+ before is reused without a url, which is how a version worker finds it."""
216
+ home = self.root / key.version_id / SNAPSHOTS_DIR / str(key.sha256)
217
+ if (home / "bundle").is_dir():
218
+ return home / "bundle"
219
+ url, digest = snapshot.get("url"), snapshot.get("digest")
220
+ if not url or not isinstance(digest, str):
221
+ fail(
222
+ "host",
223
+ f"snapshot {key.sha256} of version {key.version_id} is not cached on this host, "
224
+ "and the open names no url and digest to fetch it",
167
225
  )
226
+ tree = data_tree(base)
227
+ contract = check_bundle(tree)
228
+ replaced = snapshots.default_snapshot(tree, contract)
229
+ staging = home.parent / f".{key.sha256}.{uuid.uuid4().hex}"
230
+ try:
231
+ for path in sorted(base.rglob("*")):
232
+ target = staging / "bundle" / path.relative_to(base)
233
+ if path.is_dir():
234
+ target.mkdir(parents=True, exist_ok=True)
235
+ elif path != replaced:
236
+ WorldDatabase.clone_file(path, target)
237
+ staged = staging / "bundle" / tree.relative_to(base)
238
+ snapshots.fetch(staged, contract, url, str(key.sha256), digest)
239
+ os.replace(staging, home)
240
+ finally:
241
+ shutil.rmtree(staging, ignore_errors=True)
242
+ return home / "bundle"
243
+
244
+ def _load(self, version_id: str, bundle: Path, template: Path, *, fetched: bool) -> CachedVersion:
245
+ if is_tools_bundle(bundle):
246
+ tools = load_tools_version(bundle, template, fetched=fetched, run_handlers=self.run_handlers)
168
247
  return CachedVersion(
169
248
  version_id=version_id,
170
249
  bundle=bundle,
@@ -179,8 +258,6 @@ class VersionCache:
179
258
  )
180
259
  tree = world_tree(bundle)
181
260
  contract = check_bundle(tree)
182
- if snapshot:
183
- snapshots.fetch(tree, contract, snapshot["url"], snapshot["sha256"], snapshot["digest"])
184
261
  served = snapshots.ServedSnapshot.of(tree)
185
262
  spec = load_spec(tree)
186
263
  settings = load_api_settings(tree, spec)