@withgateway/sdk 0.6.0 → 0.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/dist/agents/index.d.mts +581 -0
  2. package/dist/agents/index.d.ts +581 -0
  3. package/dist/agents/index.js +1473 -0
  4. package/dist/agents/index.js.map +1 -0
  5. package/dist/agents/index.mjs +1424 -0
  6. package/dist/agents/index.mjs.map +1 -0
  7. package/dist/apiError-DHpfbmdx.d.mts +17 -0
  8. package/dist/apiError-DHpfbmdx.d.ts +17 -0
  9. package/dist/cli.js +11611 -6845
  10. package/dist/cli.js.map +1 -1
  11. package/dist/cli.mjs +10090 -5325
  12. package/dist/cli.mjs.map +1 -1
  13. package/dist/files/index.d.mts +3 -1
  14. package/dist/files/index.d.ts +3 -1
  15. package/dist/files/index.js +371 -102
  16. package/dist/files/index.js.map +1 -1
  17. package/dist/files/index.mjs +371 -102
  18. package/dist/files/index.mjs.map +1 -1
  19. package/dist/hooks/index.js +346 -136
  20. package/dist/hooks/index.js.map +1 -1
  21. package/dist/hooks/index.mjs +346 -136
  22. package/dist/hooks/index.mjs.map +1 -1
  23. package/dist/{index-LA-PMRJT.d.ts → index-DIN14Lhg.d.ts} +1 -1
  24. package/dist/{index-Di9nNqOY.d.mts → index-DuB-eQ0Z.d.mts} +1 -1
  25. package/dist/index.d.mts +7 -4
  26. package/dist/index.d.ts +7 -4
  27. package/dist/index.js +3177 -748
  28. package/dist/index.js.map +1 -1
  29. package/dist/index.mjs +2987 -590
  30. package/dist/index.mjs.map +1 -1
  31. package/dist/media-CQUC7TkS.d.mts +39 -0
  32. package/dist/media-DQnBloaX.d.ts +39 -0
  33. package/dist/traces/index.d.mts +3 -1
  34. package/dist/traces/index.d.ts +3 -1
  35. package/dist/traces/index.js +371 -102
  36. package/dist/traces/index.js.map +1 -1
  37. package/dist/traces/index.mjs +371 -102
  38. package/dist/traces/index.mjs.map +1 -1
  39. package/dist/tracing/index.d.mts +3 -2
  40. package/dist/tracing/index.d.ts +3 -2
  41. package/dist/tracing/index.js +257 -16
  42. package/dist/tracing/index.js.map +1 -1
  43. package/dist/tracing/index.mjs +257 -16
  44. package/dist/tracing/index.mjs.map +1 -1
  45. package/dist/transport-CpRe7zgd.d.mts +123 -0
  46. package/dist/transport-CpRe7zgd.d.ts +123 -0
  47. package/dist/worlds/index.d.mts +1986 -1746
  48. package/dist/worlds/index.d.ts +1986 -1746
  49. package/dist/worlds/index.js +1470 -290
  50. package/dist/worlds/index.js.map +1 -1
  51. package/dist/worlds/index.mjs +1465 -286
  52. package/dist/worlds/index.mjs.map +1 -1
  53. package/package.json +6 -1
  54. package/runtime/gatewaysdk/environment/bundle_manifest.py +531 -0
  55. package/runtime/gatewaysdk/environment/factory_documents.py +670 -0
  56. package/runtime/gatewaysdk/environment/factory_rules.py +973 -0
  57. package/runtime/gatewaysdk/environment/factory_shapes.py +253 -0
  58. package/runtime/gatewaysdk/environment/factory_spine.py +1155 -0
  59. package/runtime/gatewaysdk/environment/factory_spine_shapes.py +480 -0
  60. package/runtime/gatewaysdk/environment/schema/__main__.py +2 -2
  61. package/runtime/gatewaysdk/environment/schema/_slack_fidelity.py +8 -23
  62. package/runtime/gatewaysdk/environment/schema/admission.py +61 -59
  63. package/runtime/gatewaysdk/environment/schema/api.py +3 -4
  64. package/runtime/gatewaysdk/environment/schema/assertions.py +7 -1
  65. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
  66. package/runtime/gatewaysdk/environment/schema/call_log.py +20 -0
  67. package/runtime/gatewaysdk/environment/schema/cast.py +133 -67
  68. package/runtime/gatewaysdk/environment/schema/compiler.py +16 -17
  69. package/runtime/gatewaysdk/environment/schema/database/base.py +2 -2
  70. package/runtime/gatewaysdk/environment/schema/handler_errors.py +58 -4
  71. package/runtime/gatewaysdk/environment/schema/host.py +94 -12
  72. package/runtime/gatewaysdk/environment/schema/host_browser.py +48 -0
  73. package/runtime/gatewaysdk/environment/schema/host_control.py +35 -2
  74. package/runtime/gatewaysdk/environment/schema/host_handoff.py +4 -1
  75. package/runtime/gatewaysdk/environment/schema/host_loops.py +27 -4
  76. package/runtime/gatewaysdk/environment/schema/host_metrics.py +69 -0
  77. package/runtime/gatewaysdk/environment/schema/host_requests.py +35 -0
  78. package/runtime/gatewaysdk/environment/schema/host_session.py +20 -2
  79. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +6 -0
  80. package/runtime/gatewaysdk/environment/schema/host_versions.py +100 -19
  81. package/runtime/gatewaysdk/environment/schema/host_workers.py +172 -50
  82. package/runtime/gatewaysdk/environment/schema/ingest.py +12 -3
  83. package/runtime/gatewaysdk/environment/schema/journal.py +7 -2
  84. package/runtime/gatewaysdk/environment/schema/local.py +32 -7
  85. package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
  86. package/runtime/gatewaysdk/environment/schema/measured_tests.py +187 -0
  87. package/runtime/gatewaysdk/environment/schema/page_checks.py +123 -0
  88. package/runtime/gatewaysdk/environment/schema/platform.py +80 -270
  89. package/runtime/gatewaysdk/environment/schema/platform_query.py +4 -3
  90. package/runtime/gatewaysdk/environment/schema/platform_templates.py +420 -0
  91. package/runtime/gatewaysdk/environment/schema/process_memory.py +76 -0
  92. package/runtime/gatewaysdk/environment/schema/redaction.py +13 -0
  93. package/runtime/gatewaysdk/environment/schema/request_ledger.py +4 -1
  94. package/runtime/gatewaysdk/environment/schema/run_parameters.py +25 -0
  95. package/runtime/gatewaysdk/environment/schema/runtime.py +57 -5
  96. package/runtime/gatewaysdk/environment/schema/scaffold.py +386 -219
  97. package/runtime/gatewaysdk/environment/schema/session_files.py +13 -2
  98. package/runtime/gatewaysdk/environment/schema/shape_catalog.py +493 -0
  99. package/runtime/gatewaysdk/environment/schema/shape_declarations.py +219 -0
  100. package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
  101. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +1 -1
  102. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +2 -2
  103. package/runtime/gatewaysdk/environment/schema/skills/factory-orchestrator/SKILL.md +147 -0
  104. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +5 -1
  105. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +8 -6
  106. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +9 -9
  107. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +33 -2
  108. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +6 -6
  109. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +17 -2
  110. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/new-world-sandbox.md +1 -1
  111. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/shape-catalog.md +59 -0
  112. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/ui-map.md +3 -2
  113. package/runtime/gatewaysdk/environment/schema/slack.py +55 -10
  114. package/runtime/gatewaysdk/environment/schema/snapshot.py +28 -2
  115. package/runtime/gatewaysdk/environment/schema/tasks.py +25 -2
  116. package/runtime/gatewaysdk/environment/schema/template_record.py +131 -0
  117. package/runtime/gatewaysdk/environment/schema/tools_world.py +7 -5
  118. package/runtime/gatewaysdk/environment/schema/ui.py +54 -33
  119. package/runtime/gatewaysdk/environment/schema/ui_interactions.py +419 -0
  120. package/runtime/gatewaysdk/environment/schema/ui_recorder.js +895 -0
  121. package/runtime/gatewaysdk/environment/schema/ui_replay.py +366 -0
  122. package/runtime/gatewaysdk/environment/schema/ui_replay_record.LICENSE +25 -0
  123. package/runtime/gatewaysdk/environment/schema/ui_replay_record.js +109 -0
  124. package/runtime/gatewaysdk/environment/schema/validation.py +24 -1
  125. package/runtime/gatewaysdk/environment/schema/worker.py +12 -5
  126. package/runtime/gatewaysdk/environment/schema/worker_key.py +38 -0
  127. package/runtime/gatewaysdk/environment/schema/worker_peaks.py +38 -0
  128. package/runtime/gatewaysdk/environment/schema/worker_watch.py +50 -0
  129. package/runtime/gatewaysdk/environment/schema/workers.py +245 -69
  130. package/runtime/gatewaysdk/environment/schema/world_host.py +96 -73
  131. package/runtime/gatewaysdk/environment/schema/world_tests.py +3 -1
  132. package/runtime/gatewaysdk/environment/secret_values.py +1 -1
  133. package/runtime/gatewaysdk/environment/world.py +34 -43
  134. package/runtime/gatewaysdk/environment/world_session_contract.py +7 -34
  135. package/skills/benchmarks-getting-started/SKILL.md +1 -1
  136. package/skills/connector-schema-authoring/SKILL.md +2 -2
  137. package/skills/connector-schema-authoring/connector-worlds.md +3 -2
  138. package/skills/connector-schema-authoring/world-schema.md +36 -23
  139. package/skills/factory-orchestrator/SKILL.md +147 -0
  140. package/skills/world-data-ingestion/ROWS.md +5 -1
  141. package/skills/world-data-ingestion/SKILL.md +8 -6
  142. package/skills/worlds-getting-started/SKILL.md +9 -9
  143. package/skills/worlds-getting-started/references/cli-mcp-map.md +33 -2
  144. package/skills/worlds-getting-started/references/gotchas.md +6 -6
  145. package/skills/worlds-getting-started/references/journal-checks.md +17 -2
  146. package/skills/worlds-getting-started/references/new-world-sandbox.md +1 -1
  147. package/skills/worlds-getting-started/references/shape-catalog.md +59 -0
  148. package/skills/worlds-getting-started/references/ui-map.md +3 -2
  149. package/dist/media-BIApiPzS.d.mts +0 -102
  150. package/dist/media-BIApiPzS.d.ts +0 -102
@@ -28,6 +28,9 @@ import json
28
28
  import math
29
29
  import re
30
30
  import sys
31
+ from collections.abc import Iterable, Iterator
32
+ from contextlib import contextmanager
33
+ from contextvars import ContextVar
31
34
  from typing import Any, Callable
32
35
 
33
36
  from .validation import ContractError, fail
@@ -52,6 +55,7 @@ class Refusal(Exception):
52
55
 
53
56
  _DETAIL_LIMIT = 2000
54
57
  _FIELD_LIMIT = 256
58
+ ENGINE = "engine"
55
59
  _SURROGATE = re.compile("[\ud800-\udfff]")
56
60
 
57
61
 
@@ -89,7 +93,45 @@ def log_contract_refusal(world: str, operation: str, error: ContractError) -> No
89
93
 
90
94
  def request_failure_body(error: BaseException) -> dict:
91
95
  """The 500 body for a failure met answering a request, outside any handler."""
92
- return {"error": "engine", "detail": f"{type(error).__name__}: {_detail(error)}"[:_DETAIL_LIMIT]}
96
+ return {"error": ENGINE, "detail": f"{type(error).__name__}: {_detail(error)}"[:_DETAIL_LIMIT]}
97
+
98
+
99
+ class OutOfMemory:
100
+ """A request whose world code raised ``MemoryError``, or ``SystemError`` (what C code raises when an
101
+ allocation fails without setting one), noted in that request's own context where the raise becomes
102
+ its answer (:func:`failure_answer`, the host's request handler) and never read from the answer, so
103
+ no data a world serves can claim one. A worker's front end says so to its supervisor in ``HEADER``
104
+ on that answer, and sends no other header of that name; the supervisor recycles the worker only
105
+ when the kernel counts it near its cap."""
106
+
107
+ HEADER = "X-Gateway-Out-Of-Memory"
108
+ _noted: ContextVar[bool] = ContextVar("gateway_out_of_memory", default=False)
109
+
110
+ @classmethod
111
+ @contextmanager
112
+ def request(cls) -> Iterator[None]:
113
+ token = cls._noted.set(False)
114
+ try:
115
+ yield
116
+ finally:
117
+ cls._noted.reset(token)
118
+
119
+ @classmethod
120
+ def note(cls, error: BaseException) -> None:
121
+ if isinstance(error, (MemoryError, SystemError)):
122
+ cls._noted.set(True)
123
+
124
+ @classmethod
125
+ def noted(cls) -> bool:
126
+ return cls._noted.get()
127
+
128
+ @classmethod
129
+ def named(cls, header: str) -> bool:
130
+ return header.lower() == cls.HEADER.lower()
131
+
132
+ @classmethod
133
+ def in_headers(cls, headers: Iterable[tuple[str, str]]) -> bool:
134
+ return any(cls.named(name) for name, _ in headers)
93
135
 
94
136
 
95
137
  def engine_failure_body(handler: str, error: BaseException) -> dict:
@@ -99,6 +141,7 @@ def engine_failure_body(handler: str, error: BaseException) -> dict:
99
141
 
100
142
  def failure_answer(handler: str, error: Exception) -> tuple[int, dict]:
101
143
  """A handler's raise as its ``(status, body)``: a refusal, its author's bug or the engine's failure."""
144
+ OutOfMemory.note(error)
102
145
  if isinstance(error, Refusal):
103
146
  return refusal_answer(handler, error)
104
147
  if isinstance(error, HANDLER_BUG_ERRORS):
@@ -127,13 +170,24 @@ def unencodable(body: Any) -> Exception | None:
127
170
  return None
128
171
 
129
172
 
173
+ _HEADER_NAME = re.compile(r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+")
174
+ _LINE_BREAK = re.compile(r"[\r\n\x00]")
175
+
176
+
177
+ def header_line(name: str, value: str) -> bool:
178
+ """Whether ``name: value`` is one header line: a token for a name, no CR, LF or NUL in the value, so no
179
+ answer can add a line of its own (a forged ``OutOfMemory.HEADER`` included)."""
180
+ return bool(_HEADER_NAME.fullmatch(name)) and not _LINE_BREAK.search(value)
181
+
182
+
130
183
  def _unsendable_header(headers: dict[str, str] | None) -> Exception | None:
131
184
  try:
132
185
  for name, value in (headers or {}).items():
133
186
  f"{name}: {value}".encode("latin-1")
134
187
  except UnicodeEncodeError as error:
135
- return error
136
- return None
188
+ return ValueError(f"a header is not latin-1: {error}")
189
+ split = next((name for name, value in (headers or {}).items() if not header_line(name, value)), None)
190
+ return None if split is None else ValueError(f"header {split!r} is not one line: a token name, no CR, LF or NUL")
137
191
 
138
192
 
139
193
  def unencodable_answer(handler: str, response: tuple) -> tuple[int, dict] | None:
@@ -143,7 +197,7 @@ def unencodable_answer(handler: str, response: tuple) -> tuple[int, dict] | None
143
197
  return 400, handler_bug_body(handler, ValueError(f"an answer names a status from 200 to 599, not {status!r}"))
144
198
  header = _unsendable_header(headers)
145
199
  if header is not None:
146
- return 400, handler_bug_body(handler, header, "a header is not latin-1: ")
200
+ return 400, handler_bug_body(handler, header)
147
201
  error = unencodable(body)
148
202
  return None if error is None else (400, handler_bug_body(handler, error, "the answer is not JSON: "))
149
203
 
@@ -50,15 +50,18 @@ import signal
50
50
  import sys
51
51
  import threading
52
52
  import urllib.parse
53
+ from functools import partial
53
54
  from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
54
55
  from pathlib import Path
55
56
  from typing import Any
56
57
 
57
58
  from .api import Via, with_query
58
59
  from .call_capture import JSON_TYPE, send_response
60
+ from .call_redaction import CallRedaction
59
61
  from .credentials import BearerAuth
60
62
  from .door import PLATFORM_HEADER, Entry
61
- from .handler_errors import log_request_failure, request_failure_body
63
+ from .handler_errors import OutOfMemory, header_line, log_request_failure, request_failure_body
64
+ from .host_browser import RelayNavigation, log_relay
62
65
  from .host_control import ControlPlane
63
66
  from .host_handoff import DrainHandoff, HandoffDoor, PlatformHandoff
64
67
  from .host_loops import HEARTBEAT_S, Heartbeat, start_lease_sweeper
@@ -90,10 +93,14 @@ from .host_versions import DEFAULT_MAX_BUNDLE_MB
90
93
  from .host_workers import UNKNOWN_SESSION, refusal
91
94
  from .request_ledger import LedgerFlusher, LedgerSpill, NoLedger, PlatformLedger
92
95
  from .session_move import SessionMoved
96
+ from .ui_interactions import RECORDER_PATH, UiRecorder
97
+ from .ui_replay import REPLAY_PATH, REPLAY_SCRIPT_PATH, REPLAY_STOP_S, ReplayShipper, UiReplay
93
98
  from .validation import ContractError
94
99
  from .workers import (
95
100
  DEFAULT_WORKER_BUDGET_MB,
101
+ DEFAULT_WORKER_FILE_MB,
96
102
  DEFAULT_WORKER_FOOTPRINT_MB,
103
+ DEFAULT_WORKER_MEMORY_CEILING_MB,
97
104
  DEFAULT_WORKER_MEMORY_MB,
98
105
  WorkerPool,
99
106
  seal_process,
@@ -127,13 +134,15 @@ def make_host_handler(
127
134
  *,
128
135
  max_body_mb: int = DEFAULT_MAX_BODY_MB,
129
136
  secure_cookies: bool = True,
137
+ navigation: RelayNavigation | None = None,
138
+ replays: ReplayShipper | None = None,
130
139
  ):
131
140
  """The request handler class for one host; its control plane is :class:`ControlPlane`, which
132
141
  refuses an empty ``secret`` or ``worlds_domain``. ``secure_cookies`` is off
133
142
  only for a host whose public surface is reached over plain http (a dev
134
143
  host); it is the deployment's word, not the request's, since a client
135
144
  writes the forwarding headers a request arrives with."""
136
- control = ControlPlane(host, secret, HOSTED_OPERATIONS, worlds_domain)
145
+ control = ControlPlane(host, secret, HOSTED_OPERATIONS, worlds_domain, navigation)
137
146
  if max_body_mb <= 0:
138
147
  raise ValueError(f"the request body cap must be positive (WORLDS_HOST_MAX_BODY_MB={max_body_mb})")
139
148
  max_body_bytes = max_body_mb * (1 << 20)
@@ -172,6 +181,20 @@ def make_host_handler(
172
181
  return {k.lower(): v for k, v in self.headers.items()}
173
182
 
174
183
  def _dispatch(self, method: str) -> None:
184
+ with OutOfMemory.request():
185
+ self._answer(method)
186
+
187
+ def send_header(self, keyword: str, value: str) -> None:
188
+ if OutOfMemory.named(keyword) or not header_line(keyword, str(value)):
189
+ return
190
+ super().send_header(keyword, value)
191
+
192
+ def end_headers(self) -> None:
193
+ if host.run_handlers and OutOfMemory.noted():
194
+ super().send_header(OutOfMemory.HEADER, "1")
195
+ super().end_headers()
196
+
197
+ def _answer(self, method: str) -> None:
175
198
  self._body: bytes | None = None
176
199
  url = urllib.parse.urlsplit(self.path)
177
200
  if url.path == "/healthz" and control.internal(self.path, self.headers):
@@ -195,6 +218,7 @@ def make_host_handler(
195
218
  self.close_connection = True
196
219
  return self._reply(refused.status, {"error": refused.error}, {"Connection": "close"})
197
220
  except Exception as error:
221
+ OutOfMemory.note(error)
198
222
  log_request_failure(error, method, url.path)
199
223
  return self._reply(500, request_failure_body(error))
200
224
 
@@ -318,12 +342,23 @@ def make_host_handler(
318
342
  ):
319
343
  return self._reply(401, {"error": "invalid session token"})
320
344
  route = api_target(ui, url.path)
321
- if route is None:
322
- return self._serve_static(ui, method, url, opened)
345
+ if url.path == RECORDER_PATH:
346
+ return self._serve_file(method, opened, UiRecorder.script)
347
+ if url.path == REPLAY_SCRIPT_PATH:
348
+ return self._serve_file(method, opened, partial(UiReplay.script, replays))
349
+ static = route is None and not UiRecorder.serves(url.path)
350
+ if static and session.worker is not None and UiRecorder.is_page(url.path):
351
+ return self._on_worker(session, method, headers)
352
+ if static:
353
+ return self._serve_static(session, ui, method, url, opened)
323
354
  if cross_site(headers, lambda origin: urllib.parse.urlsplit(origin).netloc == headers.get("host")):
324
355
  return self._reply(403, {"error": "the ui's api prefix answers its own page, not another site"})
356
+ if url.path == REPLAY_PATH:
357
+ return self._record_replay(session, method)
325
358
  if session.worker is not None:
326
359
  return self._on_worker(session, method, headers)
360
+ if route is None:
361
+ return self._record_interactions(session, method, url, headers)
327
362
  target = urllib.parse.urlunsplit(("", "", route, url.query, ""))
328
363
  api_headers = {k: v for k, v in headers.items() if k not in ("cookie", "authorization")}
329
364
  api_headers["host"] = session.api_host(worlds_domain, opened)
@@ -334,20 +369,49 @@ def make_host_handler(
334
369
  cookie = session_cookie(token, secure=secure_cookies, max_age_s=host.lease_s)
335
370
  self._reply(303, {"location": landing}, {"Location": landing, "Set-Cookie": cookie})
336
371
 
337
- def _serve_static(self, ui: UiSurface, method: str, url, opened: bool) -> None:
338
- """A file of the ui; on a capability host with no ``Referer``, so a page that
339
- loads another site's resource never hands it the capability."""
372
+ def _serve_static(self, session: Session, ui: UiSurface, method: str, url, opened: bool) -> None:
373
+ """A file of the ui; a page carries the recorder and is noted as served, a note for the
374
+ ledger alone (no grader reads it), so a page is never held by the move fence."""
340
375
  if method != "GET":
341
376
  return self._reply(405, {"error": "the static ui answers GET only"})
342
- status, static_headers, payload = static_file(ui, url.path)
343
- sealed = {**static_headers, **(NO_REFERRER if opened else {})}
377
+ status, file_headers, payload = static_file(ui, url.path)
378
+ page = UiRecorder.page(status, file_headers, payload)
379
+ if page is not None and session.runtime is not None:
380
+ UiRecorder.served(url.path, session.runtime.calls)
381
+ self._send_file(opened, status, file_headers, payload if page is None else page)
382
+
383
+ def _serve_file(self, method: str, opened: bool, answer) -> None:
384
+ if method != "GET":
385
+ return self._reply(405, {"error": "the static ui answers GET only"})
386
+ self._send_file(opened, *answer())
387
+
388
+ def _send_file(self, opened: bool, status: int, headers: dict[str, str], payload: bytes) -> None:
389
+ """On a capability host with no ``Referer``, so a page that loads another site's
390
+ resource never hands it the capability."""
391
+ sealed = {**headers, **(NO_REFERRER if opened else {})}
344
392
  self._relay(status, sealed.items(), payload, drop=FRAMING_HEADERS)
345
393
 
394
+ def _record_interactions(self, session: Session, method: str, url, headers: dict[str, str]) -> None:
395
+ if session.runtime is None:
396
+ return self._reply(404, {"error": "this session's world is not served here"})
397
+ raw = self._read()
398
+ with host.admitted(session, HOP_HEADER in headers):
399
+ answer = UiRecorder.record(method, url.path, raw, session.runtime.calls)
400
+ self._reply(*answer)
401
+
402
+ def _record_replay(self, session: Session, method: str) -> None:
403
+ """The host that holds the session keeps its page's recording, a worker's session too, scrubbed
404
+ as the call log scrubs (:class:`CallRedaction`)."""
405
+ scrub = CallRedaction.of(session.spec, session.token).scrub
406
+ self._reply(*UiReplay.take(method, self._read(), session.session_id, scrub, replays))
407
+
346
408
  def _serve_browser(self, session: Session, url, headers: dict[str, str]) -> None:
347
409
  """``/cdp/<session token>`` and ``/live/<grant>`` on the session's
348
410
  cdp host: check the credential against the SESSION, then tunnel
349
411
  the websocket to the leased relay with the POOL credential the
350
- relay knows. The relay's token never leaves the host."""
412
+ relay knows. The relay's token never leaves the host. The relay is
413
+ bound to the session's hosts first, so one that restarted and lost
414
+ its binding is never reached unbound."""
351
415
  relay = session.browser
352
416
  if relay is None:
353
417
  return self._reply(404, {"error": "this session has no browser surface"})
@@ -368,11 +432,16 @@ def make_host_handler(
368
432
  upstream_path = f"/live/{mint_live_grant(relay.token, LIVE_RELAY_GRANT_MS, scope)}"
369
433
  if headers.get("upgrade", "").lower() != "websocket":
370
434
  return self._reply(400, {"error": "the browser surface is a websocket"})
435
+ try:
436
+ control.navigation.bind(relay, session.browser_hosts(control.domain))
437
+ except ContractError as error:
438
+ return self._reply(502, {"error": "browser unbound", "detail": str(error)})
371
439
  self.close_connection = True
372
440
  try:
373
441
  tunnel(self.connection, relay.url, upstream_path, headers)
374
442
  except OSError as error:
375
- self._reply(502, {"error": "browser unreachable", "detail": str(error)})
443
+ log_relay("browser relay unreachable", relay, error=str(error))
444
+ self._reply(502, {"error": "browser unreachable"})
376
445
 
377
446
  return Handler
378
447
 
@@ -397,12 +466,14 @@ class HostServer(ThreadingHTTPServer):
397
466
  drain_s: float = DEFAULT_DRAIN_S,
398
467
  ledger: LedgerFlusher | None = None,
399
468
  handoff: HandoffDoor | None = None,
469
+ replays: ReplayShipper | None = None,
400
470
  ):
401
471
  super().__init__(address, handler)
402
472
  self.world_host = world_host
403
473
  self.lease_sweeper = start_lease_sweeper(world_host)
404
474
  self.heartbeat = Heartbeat(world_host, *heartbeat, interval_s=heartbeat_s).start() if heartbeat else None
405
475
  world_host.ledger = ledger.start() if ledger else NoLedger()
476
+ self.replays = replays.start() if replays else None
406
477
  self.handoff = DrainHandoff(world_host, handoff, drain_s)
407
478
  self._answering = 0
408
479
  self._draining = False
@@ -457,6 +528,8 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
457
528
  ``WORLDS_WORKER_MEMORY_MB`` caps each version worker's address space, ``WORLDS_WORKER_BUDGET_MB`` is the memory
458
529
  all of them together are held to and ``WORLDS_WORKER_FOOTPRINT_MB`` what each is counted at against it (unset,
459
530
  its whole address-space cap; their count is the quotient, and one more version answers 503 ``at_capacity``),
531
+ ``WORLDS_WORKER_MEMORY_CEILING_MB`` the most address space a worker measured to need more is given (unset,
532
+ the cap), ``WORLDS_WORKER_FILE_MB`` is the largest file a worker may write (a session's database, so a world state's data),
460
533
  ``WORLDS_HOST_OBJECT_STORE`` is where every presigned URL it is handed may point (required; ``any`` for a dev
461
534
  host), ``WORLDS_HOST_MAX_BUNDLE_MB`` is the largest version bundle the cache accepts,
462
535
  ``WORLDS_HOST_MAX_BODY_MB`` the largest request body it reads, ``WORLDS_HOST_LEDGER_SPILL_DIR`` (under the cache
@@ -484,6 +557,7 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
484
557
  lease_s = float(os.environ.get("WORLDS_HOST_LEASE_S") or DEFAULT_LEASE_S)
485
558
  workers = None
486
559
  footprint = os.environ.get("WORLDS_WORKER_FOOTPRINT_MB")
560
+ ceiling = os.environ.get("WORLDS_WORKER_MEMORY_CEILING_MB")
487
561
  if os.environ.get("WORLDS_HOST_WORKERS", "1") != "0":
488
562
  workers = WorkerPool(
489
563
  Path(cache_root),
@@ -491,6 +565,8 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
491
565
  memory_mb=int(os.environ.get("WORLDS_WORKER_MEMORY_MB") or DEFAULT_WORKER_MEMORY_MB),
492
566
  budget_mb=int(os.environ.get("WORLDS_WORKER_BUDGET_MB") or DEFAULT_WORKER_BUDGET_MB),
493
567
  footprint_mb=int(footprint) if footprint else DEFAULT_WORKER_FOOTPRINT_MB,
568
+ file_mb=int(os.environ.get("WORLDS_WORKER_FILE_MB") or DEFAULT_WORKER_FILE_MB),
569
+ ceiling_mb=int(ceiling) if ceiling else DEFAULT_WORKER_MEMORY_CEILING_MB,
494
570
  )
495
571
  owners = PlatformOwners(platform_url, secret, advertise) if platform_url else None
496
572
  world_host = WorldHost(
@@ -504,6 +580,7 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
504
580
  build=os.environ.get("WORLDS_HOST_BUILD") or None,
505
581
  owners=owners,
506
582
  )
583
+ replays = ReplayShipper(platform_url, secret) if platform_url else None
507
584
  handler = make_host_handler(
508
585
  world_host,
509
586
  secret=secret,
@@ -511,6 +588,7 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
511
588
  owner_lookup=owners,
512
589
  max_body_mb=int(os.environ.get("WORLDS_HOST_MAX_BODY_MB") or DEFAULT_MAX_BODY_MB),
513
590
  secure_cookies=os.environ.get("WORLDS_HOST_PLAIN_HTTP", "0") != "1",
591
+ replays=replays,
514
592
  )
515
593
  sink = PlatformLedger(platform_url, secret, host_url=advertise) if platform_url else None
516
594
  return HostServer(
@@ -522,6 +600,7 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
522
600
  heartbeat_s=float(os.environ.get("WORLDS_HOST_HEARTBEAT_S") or HEARTBEAT_S),
523
601
  drain_s=float(os.environ.get("WORLDS_HOST_DRAIN_S") or DEFAULT_DRAIN_S),
524
602
  handoff=PlatformHandoff(platform_url, secret) if platform_url and advertise else None,
603
+ replays=replays,
525
604
  )
526
605
 
527
606
 
@@ -534,7 +613,8 @@ def main(argv: list[str] | None = None) -> int:
534
613
  args = parser.parse_args(argv)
535
614
  server = serve(args.cache_root, port=args.port)
536
615
  print(
537
- f"world host listening on :{server.server_address[1]}, cache {args.cache_root}",
616
+ f"world host listening on :{server.server_address[1]}, cache {args.cache_root} "
617
+ f"(session copies: {server.world_host.cache.clone_mode})",
538
618
  file=sys.stderr,
539
619
  flush=True,
540
620
  )
@@ -543,6 +623,8 @@ def main(argv: list[str] | None = None) -> int:
543
623
  server.serve_forever()
544
624
  finally:
545
625
  server.world_host.ledger.stop(settle=server.world_host.finishing.settle)
626
+ if server.replays is not None:
627
+ server.replays.stop(REPLAY_STOP_S)
546
628
  server.lease_sweeper.set()
547
629
  if server.heartbeat is not None:
548
630
  server.heartbeat.stop()
@@ -0,0 +1,48 @@
1
+ """Binds a leased pool browser to the one session it serves: the relay's egress proxy then
2
+ loads that session's hosts and nothing else (deploy/world-browser/navigation.mjs)."""
3
+
4
+ from __future__ import annotations
5
+
6
+ import json
7
+ import sys
8
+ import time
9
+ import urllib.error
10
+ import urllib.request
11
+ from collections.abc import Callable
12
+ from dataclasses import dataclass, field
13
+ from typing import Any
14
+
15
+ from .host_requests import BrowserRelay
16
+ from .validation import fail
17
+
18
+ GUARD_REQUIRED_FROM = 1794787200.0
19
+ BIND_TIMEOUT_S = 5.0
20
+
21
+
22
+ @dataclass(frozen=True)
23
+ class RelayNavigation:
24
+ opener: Callable[..., Any] = field(default=urllib.request.urlopen)
25
+ clock: Callable[[], float] = field(default=time.time)
26
+
27
+ def bind(self, relay: BrowserRelay, hosts: list[str]) -> str:
28
+ request = urllib.request.Request(
29
+ f"{relay.http_url}/navigation",
30
+ data=json.dumps({"hosts": hosts}).encode(),
31
+ method="PUT",
32
+ headers={"authorization": f"Bearer {relay.token}", "content-type": "application/json"},
33
+ )
34
+ try:
35
+ with self.opener(request, timeout=BIND_TIMEOUT_S) as answer:
36
+ return str(json.loads(answer.read() or b"{}").get("navigation", "bound"))
37
+ except urllib.error.HTTPError as error:
38
+ if error.code == 404 and self.clock() < GUARD_REQUIRED_FROM:
39
+ log_relay("browser relay predates the navigation guard", relay)
40
+ return "unguarded"
41
+ fail("host", f"the browser could not be bound to this session (HTTP {error.code})")
42
+ except (OSError, ValueError) as error:
43
+ log_relay("browser relay could not be bound", relay, error=str(error))
44
+ fail("host", "the browser could not be bound to this session: its relay did not answer as expected")
45
+
46
+
47
+ def log_relay(msg: str, relay: BrowserRelay, **extra: str) -> None:
48
+ print(json.dumps({"msg": msg, "relay": relay.url, **extra}), file=sys.stderr, flush=True)
@@ -15,6 +15,8 @@ from email.message import Message
15
15
  from typing import TYPE_CHECKING, Any
16
16
 
17
17
  from .handler_errors import carried
18
+ from .host_browser import RelayNavigation
19
+ from .host_requests import BrowserRelay, parse_memory_peak, parse_world_state
18
20
  from .host_surfaces import public_host
19
21
  from .host_workers import WORKER_EXITED, refusal
20
22
  from .validation import ContractError, fail
@@ -35,10 +37,18 @@ class ControlPlane:
35
37
  """One host's control plane. A host with no secret has no control plane at all rather than an
36
38
  open one, and one with no worlds domain could not tell its own address from a session's."""
37
39
 
38
- def __init__(self, host: WorldHost, secret: str, operations: Mapping[str, Callable[..., dict]], worlds_domain: str):
40
+ def __init__(
41
+ self,
42
+ host: WorldHost,
43
+ secret: str,
44
+ operations: Mapping[str, Callable[..., dict]],
45
+ worlds_domain: str,
46
+ navigation: RelayNavigation | None = None,
47
+ ):
39
48
  if not secret:
40
49
  raise ValueError("the world host needs a control-plane secret (WORLDS_HOST_SECRET)")
41
50
  self.host = host
51
+ self.navigation = navigation or RelayNavigation()
42
52
  self.operations = operations
43
53
  self.domain = self.domain_of(worlds_domain)
44
54
  self._secret = secret.encode()
@@ -109,6 +119,20 @@ class ControlPlane:
109
119
  return 404, {"error": "unknown control route"}
110
120
 
111
121
  def _open(self, body: dict) -> dict:
122
+ answer = self._open_session(body)
123
+ relay = BrowserRelay.parse(body.get("browser"))
124
+ if relay is not None:
125
+ self._bind_or_close(body["sessionId"], relay)
126
+ return answer
127
+
128
+ def _bind_or_close(self, session_id: str, relay: BrowserRelay) -> None:
129
+ try:
130
+ self.navigation.bind(relay, self.host.browser_hosts(session_id, self.domain))
131
+ except Exception:
132
+ self.host.close(session_id, "browser_unbound")
133
+ raise
134
+
135
+ def _open_session(self, body: dict) -> dict:
112
136
  return self.host.open(
113
137
  session_id=body["sessionId"],
114
138
  version_id=body["versionId"],
@@ -123,14 +147,23 @@ class ControlPlane:
123
147
  pending=body.get("pending") is True,
124
148
  auth=body.get("auth"),
125
149
  ledger_source=body.get("ledgerSource"),
150
+ parameters=body.get("parameters"),
151
+ world_state=parse_world_state(body.get("worldState")),
152
+ memory_peak_mb=parse_memory_peak(body.get("memoryPeakMb")),
126
153
  )
127
154
 
155
+ def _swap_browser(self, session_id: str, browser: Any) -> dict:
156
+ relay = BrowserRelay.parse(browser)
157
+ if relay is not None:
158
+ self.navigation.bind(relay, self.host.browser_hosts(session_id, self.domain))
159
+ return self.host.set_browser(session_id, browser)
160
+
128
161
  def _session(self, method: str, parts: list[str], body: dict) -> Answer:
129
162
  sid = parts[2]
130
163
  self.host.forwarding.refuse(sid)
131
164
  verb = parts[3] if len(parts) > 3 else None
132
165
  if method == "PUT" and verb == "browser" and len(parts) == 4:
133
- return 200, self.host.set_browser(sid, body.get("browser"))
166
+ return 200, self._swap_browser(sid, body.get("browser"))
134
167
  if method == "POST" and verb == "export" and len(parts) == 5:
135
168
  return self._upload(sid, parts[4], body.get("url"))
136
169
  row = session_verb(method, verb)
@@ -31,6 +31,7 @@ from .handler_errors import log_line
31
31
  from .host_owners import PROXY_TIMEOUT_S
32
32
  from .request_ledger import LEDGER_STOP_S
33
33
  from .session_move import same_host
34
+ from .ui_replay import REPLAY_STOP_S
34
35
 
35
36
  if TYPE_CHECKING:
36
37
  from .host_session import Session
@@ -47,7 +48,9 @@ ENLIST_S = 1.0
47
48
  LINGER_S = 5.0
48
49
  IN_FLIGHT_S = PROXY_TIMEOUT_S
49
50
  EXIT_S = 110.0
50
- assert LINGER_S + IN_FLIGHT_S + LEDGER_STOP_S + 5.0 <= EXIT_S, "a stopping host's exit fits the grace past its drain"
51
+ assert (
52
+ LINGER_S + IN_FLIGHT_S + LEDGER_STOP_S + REPLAY_STOP_S + 5.0 <= EXIT_S
53
+ ), "a stopping host's exit fits the grace past its drain"
51
54
 
52
55
  MOVED = "moved"
53
56
  STAYED = "stayed"
@@ -4,6 +4,7 @@ platform what the host is and holds. A step that fails is logged and the next on
4
4
 
5
5
  from __future__ import annotations
6
6
 
7
+ import http.client
7
8
  import json
8
9
  import threading
9
10
  import urllib.error
@@ -18,6 +19,8 @@ if TYPE_CHECKING:
18
19
 
19
20
  LEASE_SWEEP_S = 30.0
20
21
  HEARTBEAT_S = 10.0
22
+ #: The most a heartbeat reads of the platform's answer.
23
+ ANSWER_LIMIT_BYTES = 64 << 10
21
24
 
22
25
 
23
26
  def swept(step: Callable[[], list[str]], done: str) -> list[str]:
@@ -49,7 +52,8 @@ def heartbeat_once(world_host: WorldHost, platform_url: str, secret: str, timeou
49
52
  """Tell the platform what this host is and holds (:meth:`WorldHost.stats`).
50
53
  The platform's registry is what places new sessions: a host it has not
51
54
  heard from lately gets none. Returns whether the platform took it."""
52
- body = json.dumps(world_host.stats()).encode("utf-8")
55
+ stats = world_host.stats()
56
+ body = json.dumps(stats).encode("utf-8")
53
57
  req = urllib.request.Request(
54
58
  f"{platform_url.rstrip('/')}/api/public/world-hosts",
55
59
  data=body,
@@ -58,10 +62,23 @@ def heartbeat_once(world_host: WorldHost, platform_url: str, secret: str, timeou
58
62
  )
59
63
  try:
60
64
  with urllib.request.urlopen(req, timeout=timeout_s) as r:
61
- return 200 <= r.status < 300
62
- except (urllib.error.URLError, OSError, ValueError) as error:
65
+ taken = 200 <= r.status < 300
66
+ peaks_taken = taken and _peaks_taken(r.read(ANSWER_LIMIT_BYTES))
67
+ except (urllib.error.URLError, OSError, ValueError, http.client.HTTPException) as error:
63
68
  log_line(f"heartbeat failed: {error}")
64
69
  return False
70
+ if peaks_taken:
71
+ world_host.reported(stats)
72
+ return taken
73
+
74
+
75
+ def _peaks_taken(answer: bytes) -> bool:
76
+ """Whether the platform says it kept the beat's worker peaks: one that predates them takes the beat
77
+ and ignores them, so they stay pending until a platform that keeps them answers."""
78
+ try:
79
+ return json.loads(answer).get("peaksTaken") is True
80
+ except (ValueError, AttributeError):
81
+ return False
65
82
 
66
83
 
67
84
  class Heartbeat:
@@ -90,8 +107,14 @@ class Heartbeat:
90
107
  self._due.set()
91
108
 
92
109
  def _beat(self) -> None:
93
- heartbeat_once(self.world_host, self.platform_url, self.secret)
110
+ self._once()
94
111
  while not self._stopped.is_set():
95
112
  self._due.wait(self.interval_s)
96
113
  self._due.clear()
114
+ self._once()
115
+
116
+ def _once(self) -> None:
117
+ try:
97
118
  heartbeat_once(self.world_host, self.platform_url, self.secret)
119
+ except Exception as error: # noqa: BLE001 - one failed beat must not stop the beats after it
120
+ log_line(f"heartbeat failed: {type(error).__name__}")
@@ -0,0 +1,69 @@
1
+ """The prometheus exposition of a World Host's :meth:`WorldHost.stats`. ``worlds_host_sessions`` is the
2
+ number the fleet scales on (sessions per host, not cpu)."""
3
+
4
+ from __future__ import annotations
5
+
6
+ from .host_handoff import OUTCOMES
7
+ from .host_workers import WORKERS_POISONED
8
+ from .worker_watch import WORKERS_NEAR_CAP
9
+
10
+
11
+ def _largest_share(workers: list[dict]) -> list[str]:
12
+ """The most of its own cap any worker holds (``VmSize`` over ``RLIMIT_AS``), no sample when none was
13
+ measured. A total: world code on the pod can read ``/metrics``, so no line here names a version."""
14
+ shares = [w["sizeMb"] / w["capMb"] for w in workers if w.get("sizeMb") is not None and w.get("capMb")]
15
+ sample = [f"worlds_host_worker_cap_share_max {max(shares):.3f}"] if shares else []
16
+ return ["# TYPE worlds_host_worker_cap_share_max gauge", *sample]
17
+
18
+
19
+ def exposition(stats: dict) -> str:
20
+ counters: dict[str, int] = stats["counters"]
21
+ lines = [
22
+ "# TYPE worlds_host_sessions gauge",
23
+ f"worlds_host_sessions {stats['sessions']}",
24
+ "# TYPE worlds_host_capacity gauge",
25
+ f"worlds_host_capacity {stats['capacity']}",
26
+ "# TYPE worlds_host_draining gauge",
27
+ f"worlds_host_draining {int(stats['draining'])}",
28
+ "# TYPE worlds_host_workers gauge",
29
+ f"worlds_host_workers {stats['workers']}",
30
+ "# TYPE worlds_host_worker_memory_mb gauge",
31
+ f"worlds_host_worker_memory_mb {stats['workerMemoryMb']}",
32
+ "# TYPE worlds_host_versions_cached gauge",
33
+ f"worlds_host_versions_cached {len(stats['versions'])}",
34
+ "# TYPE worlds_host_uptime_seconds gauge",
35
+ f"worlds_host_uptime_seconds {stats['uptimeSeconds']:.0f}",
36
+ "# TYPE worlds_host_sessions_opened_total counter",
37
+ f"worlds_host_sessions_opened_total {counters.get('opened', 0)}",
38
+ "# TYPE worlds_host_open_failures_total counter",
39
+ f"worlds_host_open_failures_total {counters.get('open_failures', 0)}",
40
+ "# TYPE worlds_host_session_seconds_total counter",
41
+ f"worlds_host_session_seconds_total {stats['sessionSeconds']:.0f}",
42
+ "# TYPE worlds_host_exports_total counter",
43
+ f"worlds_host_exports_total {counters.get('exports', 0)}",
44
+ "# TYPE worlds_host_exports_uploaded_total counter",
45
+ f"worlds_host_exports_uploaded_total {counters.get('exports_uploaded', 0)}",
46
+ "# TYPE worlds_host_sessions_closed_total counter",
47
+ ]
48
+ for reason in ("closed", "expired", "lost", "replaced", "moved"):
49
+ lines.append(f'worlds_host_sessions_closed_total{{reason="{reason}"}} {counters.get(f"closed.{reason}", 0)}')
50
+ lines.append("# TYPE worlds_host_handoffs_total counter")
51
+ for outcome in OUTCOMES:
52
+ lines.append(f'worlds_host_handoffs_total{{outcome="{outcome}"}} {counters.get(f"handoffs.{outcome}", 0)}')
53
+ workers = stats.get("workerMemory") or []
54
+ lines += _largest_share(workers)
55
+ lines += [
56
+ "# TYPE worlds_host_workers_near_cap gauge",
57
+ f"worlds_host_workers_near_cap {sum(1 for w in workers if w.get('nearCap'))}",
58
+ ]
59
+ recoveries = ("workers_replaced", "workers_recycled", WORKERS_POISONED, WORKERS_NEAR_CAP, "sessions_recovered")
60
+ for name in (*recoveries, "sessions_unrecovered"):
61
+ lines += [f"# TYPE worlds_host_{name}_total counter", f"worlds_host_{name}_total {counters.get(name, 0)}"]
62
+ lines.append("# TYPE worlds_host_requests_total counter")
63
+ for surface in ("api", "ui", "browser", "proxied"):
64
+ lines.append(f'worlds_host_requests_total{{surface="{surface}"}} {counters.get(f"requests.{surface}", 0)}')
65
+ lines += [
66
+ "# TYPE worlds_host_platform_bearer_requests_total counter",
67
+ f"worlds_host_platform_bearer_requests_total {counters.get('requests.platform_bearer', 0)}",
68
+ ]
69
+ return "\n".join(lines) + "\n"