@withgateway/sdk 0.3.3 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (158) hide show
  1. package/README.md +4 -3
  2. package/dist/cli.js +3005 -1401
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +2996 -1392
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.js +128 -37
  7. package/dist/files/index.js.map +1 -1
  8. package/dist/files/index.mjs +126 -35
  9. package/dist/files/index.mjs.map +1 -1
  10. package/dist/hooks/index.d.mts +1 -1
  11. package/dist/hooks/index.d.ts +1 -1
  12. package/dist/hooks/index.js +124 -11
  13. package/dist/hooks/index.js.map +1 -1
  14. package/dist/hooks/index.mjs +124 -11
  15. package/dist/hooks/index.mjs.map +1 -1
  16. package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
  17. package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
  18. package/dist/{index-mJhG3vZU.d.mts → index-Di9nNqOY.d.mts} +26 -3
  19. package/dist/{index-DUHmS74p.d.ts → index-LA-PMRJT.d.ts} +26 -3
  20. package/dist/index.d.mts +4 -4
  21. package/dist/index.d.ts +4 -4
  22. package/dist/index.js +1078 -260
  23. package/dist/index.js.map +1 -1
  24. package/dist/index.mjs +1040 -251
  25. package/dist/index.mjs.map +1 -1
  26. package/dist/replay/index.d.mts +1 -1
  27. package/dist/replay/index.d.ts +1 -1
  28. package/dist/replay/index.js.map +1 -1
  29. package/dist/replay/index.mjs.map +1 -1
  30. package/dist/traces/index.js +125 -34
  31. package/dist/traces/index.js.map +1 -1
  32. package/dist/traces/index.mjs +123 -32
  33. package/dist/traces/index.mjs.map +1 -1
  34. package/dist/tracing/index.d.mts +1 -1
  35. package/dist/tracing/index.d.ts +1 -1
  36. package/dist/tracing/index.js +7 -1
  37. package/dist/tracing/index.js.map +1 -1
  38. package/dist/tracing/index.mjs +7 -1
  39. package/dist/tracing/index.mjs.map +1 -1
  40. package/dist/worlds/index.d.mts +819 -438
  41. package/dist/worlds/index.d.ts +819 -438
  42. package/dist/worlds/index.js +912 -236
  43. package/dist/worlds/index.js.map +1 -1
  44. package/dist/worlds/index.mjs +908 -232
  45. package/dist/worlds/index.mjs.map +1 -1
  46. package/package.json +1 -1
  47. package/runtime/gatewaysdk/environment/_bundle.py +4 -6
  48. package/runtime/gatewaysdk/environment/_materialize.py +5 -25
  49. package/runtime/gatewaysdk/environment/_world.py +82 -66
  50. package/runtime/gatewaysdk/environment/backing.py +20 -24
  51. package/runtime/gatewaysdk/environment/core.py +1 -16
  52. package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
  53. package/runtime/gatewaysdk/environment/runtime.py +45 -2
  54. package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
  55. package/runtime/gatewaysdk/environment/schema/api.py +198 -474
  56. package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
  57. package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
  58. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
  59. package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
  60. package/runtime/gatewaysdk/environment/schema/call_capture.py +321 -0
  61. package/runtime/gatewaysdk/environment/schema/call_log.py +276 -18
  62. package/runtime/gatewaysdk/environment/schema/call_redaction.py +207 -19
  63. package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
  64. package/runtime/gatewaysdk/environment/schema/compiler.py +4 -1
  65. package/runtime/gatewaysdk/environment/schema/conform.py +38 -35
  66. package/runtime/gatewaysdk/environment/schema/connector.py +94 -152
  67. package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
  68. package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +2 -1
  69. package/runtime/gatewaysdk/environment/schema/connectors/github/api/handlers.py +217 -58
  70. package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +10 -19
  71. package/runtime/gatewaysdk/environment/schema/connectors/github/scenarios/default/seed/github_webhooks.json +10 -2
  72. package/runtime/gatewaysdk/environment/schema/connectors/github/tests/webhooks.json +47 -0
  73. package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +21 -0
  74. package/runtime/gatewaysdk/environment/schema/connectors/jamf/api/handlers.py +185 -18
  75. package/runtime/gatewaysdk/environment/schema/connectors/jamf/connector.toml +29 -14
  76. package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +43 -12
  77. package/runtime/gatewaysdk/environment/schema/connectors/jamf/scenarios/default/seed/jamf_webhooks.json +2 -2
  78. package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +4 -17
  79. package/runtime/gatewaysdk/environment/schema/connectors/jamf/tests/webhooks.json +107 -0
  80. package/runtime/gatewaysdk/environment/schema/connectors/jamf/vendor-openapi.json +1 -1
  81. package/runtime/gatewaysdk/environment/schema/connectors/linear/api/handlers.py +85 -1
  82. package/runtime/gatewaysdk/environment/schema/connectors/linear/connector.toml +19 -31
  83. package/runtime/gatewaysdk/environment/schema/connectors/linear/provenance.json +18 -8
  84. package/runtime/gatewaysdk/environment/schema/connectors/linear/scenarios/default/seed/webhooks.json +19 -3
  85. package/runtime/gatewaysdk/environment/schema/connectors/linear/tests/test_webhooks.py +71 -0
  86. package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
  87. package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
  88. package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
  89. package/runtime/gatewaysdk/environment/schema/data.py +14 -13
  90. package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
  91. package/runtime/gatewaysdk/environment/schema/database/guard.py +45 -37
  92. package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
  93. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
  94. package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
  95. package/runtime/gatewaysdk/environment/schema/door.py +899 -0
  96. package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
  97. package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
  98. package/runtime/gatewaysdk/environment/schema/handler_errors.py +52 -5
  99. package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
  100. package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
  101. package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
  102. package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
  103. package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
  104. package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
  105. package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
  106. package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
  107. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
  108. package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
  109. package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
  110. package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
  111. package/runtime/gatewaysdk/environment/schema/journal.py +438 -0
  112. package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
  113. package/runtime/gatewaysdk/environment/schema/local.py +32 -116
  114. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +20 -5
  115. package/runtime/gatewaysdk/environment/schema/platform.py +29 -234
  116. package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
  117. package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
  118. package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
  119. package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
  120. package/runtime/gatewaysdk/environment/schema/runtime.py +136 -176
  121. package/runtime/gatewaysdk/environment/schema/scaffold.py +2 -1
  122. package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
  123. package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
  124. package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
  125. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
  126. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +3 -3
  127. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +50 -28
  128. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
  129. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +20 -22
  130. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +25 -25
  131. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
  132. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +71 -7
  133. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
  134. package/runtime/gatewaysdk/environment/schema/snapshot.py +150 -30
  135. package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
  136. package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
  137. package/runtime/gatewaysdk/environment/schema/tools_world.py +103 -203
  138. package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
  139. package/runtime/gatewaysdk/environment/schema/verbs.py +38 -36
  140. package/runtime/gatewaysdk/environment/schema/webhooks.py +687 -0
  141. package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
  142. package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
  143. package/runtime/gatewaysdk/environment/schema/world_host.py +970 -0
  144. package/runtime/gatewaysdk/environment/schema/world_tests.py +83 -11
  145. package/runtime/gatewaysdk/environment/world.py +1 -1
  146. package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
  147. package/skills/agents/connector-template-author.md +5 -1
  148. package/skills/benchmarks-getting-started/SKILL.md +8 -9
  149. package/skills/connector-schema-authoring/CONNECTORS.md +3 -3
  150. package/skills/connector-schema-authoring/SKILL.md +50 -28
  151. package/skills/connector-schema-authoring/connector-worlds.md +282 -18
  152. package/skills/connector-schema-authoring/world-schema.md +7 -6
  153. package/skills/world-data-ingestion/ROWS.md +12 -8
  154. package/skills/world-data-ingestion/SKILL.md +20 -22
  155. package/skills/worlds-getting-started/SKILL.md +25 -25
  156. package/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
  157. package/skills/worlds-getting-started/references/gotchas.md +71 -7
  158. package/skills/worlds-getting-started/references/journal-checks.md +182 -0
@@ -9,13 +9,13 @@ nothing per session in Kubernetes or the load balancer.
9
9
 
10
10
  Two planes on one HTTP server:
11
11
 
12
- * the **control plane** (``/__host/*``, behind ``X-Host-Secret``) is the
13
- platform's half: open, close, reset, seed, state, grade, call. It is
14
- synchronous — a ``seed`` or ``grade`` completes in the response, not over a
12
+ * the **control plane** (``/__host/*`` on the pod's own address, behind ``X-Host-Secret``;
13
+ :mod:`host_control`) is the platform's half: open, close, reset, seed, state, grade,
14
+ call. It is synchronous — a ``seed`` or ``grade`` completes in the response, not over a
15
15
  poll — because the host holds the world in memory.
16
- * the **data plane** (everything else) is the end user's half: the world's own
17
- vendor routes, selected by session and checked against that session's token,
18
- answered in-process by :class:`WorldApp`.
16
+ * the **data plane** (everything else, every path under the worlds domain included) is the
17
+ end user's half: the world's own vendor routes, selected by session and checked against
18
+ that session's token, answered in-process by :class:`WorldApp`.
19
19
 
20
20
  Every session carries a lease. Control calls and data-plane requests renew it;
21
21
  a session the platform stops touching expires on the host and is unlinked, so
@@ -45,1399 +45,71 @@ guessing.
45
45
 
46
46
  from __future__ import annotations
47
47
 
48
- import base64
49
- import hashlib
50
- import hmac
51
- import http.client
52
- import json
53
48
  import os
54
- import re
55
- import shutil
56
49
  import signal
57
50
  import sys
58
- import tarfile
59
51
  import threading
60
- import time
61
- import urllib.error
62
52
  import urllib.parse
63
- import urllib.request
64
- import uuid
65
- from collections import Counter
66
- from collections.abc import Callable, Iterable, Iterator
67
- from contextlib import contextmanager
68
- from dataclasses import dataclass, field
69
53
  from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
70
54
  from pathlib import Path
71
- from typing import Any, Optional, Protocol
72
-
73
- from .._tar import safe_tar_extract
74
- from ..world import WorldTree
75
- from . import snapshot as snapshots
76
- from .api import (
77
- ApiSettings,
78
- Handler,
79
- Router,
80
- Via,
81
- WorldApp,
82
- load_api_settings,
83
- load_handlers,
84
- send_response,
85
- with_query,
86
- )
87
- from .compiler import check_bundle
88
- from .connector import Spec, load_spec
89
- from .credentials import basic_user
90
- from .database import WorldDatabase
91
- from .dependencies import Dependency
92
- from .handler_errors import carried, log_request_failure, request_failure_body
93
- from .host_requests import BrowserRelay, DependencyRef, parse_dependency_refs
55
+ from typing import Any
56
+
57
+ from .api import Via, with_query
58
+ from .call_capture import JSON_TYPE, send_response
59
+ from .credentials import BearerAuth
60
+ from .door import PLATFORM_HEADER, Entry
61
+ from .handler_errors import log_request_failure, request_failure_body
62
+ from .host_control import ControlPlane
63
+ from .host_handoff import DrainHandoff, HandoffDoor, PlatformHandoff
64
+ from .host_loops import HEARTBEAT_S, Heartbeat, start_lease_sweeper
65
+ from .host_owners import CLOSED_SESSION, FRAMING_HEADERS, OwnerLookup, PlatformOwners
66
+ from .host_proxy import OwnerProxy
67
+ from .host_session import Session
94
68
  from .host_surfaces import (
95
69
  HOP_HEADER,
70
+ MAX_HOPS,
71
+ NO_REFERRER,
72
+ NO_SESSION_NAMED,
96
73
  UiSurface,
97
74
  api_target,
98
75
  cookie_token,
99
76
  cross_site,
100
- load_ui_surface,
77
+ hops,
101
78
  matches,
102
79
  mint_live_grant,
103
80
  query_token,
104
81
  session_cookie,
82
+ session_from_host,
105
83
  static_file,
106
84
  tunnel,
107
85
  verify_live_grant,
108
86
  without_query_token,
109
87
  )
110
- from .lanes import Lanes
111
- from .runtime import SnapshotRuntime, TreeRuntime, WorldRuntime
112
- from .served_names import ServedNames
113
- from .session_files import ExportFile, SessionExport, refuse_digest_mismatch
114
- from .tasks import TaskSpec, parse_task_spec
115
- from .tools_world import DependencyWorld, ToolsVersion, is_tools_bundle, load_tools_version
116
- from .validation import ContractError, fail
117
- from .verbs import SESSION_VERBS, session_verb, unknown_verb
88
+ from .host_transfers import parse_origins
89
+ from .host_versions import DEFAULT_MAX_BUNDLE_MB
90
+ from .host_workers import UNKNOWN_SESSION, refusal
91
+ from .request_ledger import LedgerFlusher, LedgerSpill, NoLedger, PlatformLedger
92
+ from .session_move import SessionMoved
93
+ from .validation import ContractError
118
94
  from .workers import (
119
95
  DEFAULT_WORKER_BUDGET_MB,
120
96
  DEFAULT_WORKER_FOOTPRINT_MB,
121
97
  DEFAULT_WORKER_MEMORY_MB,
122
- Worker,
123
- WorkerCapacity,
124
- WorkerGone,
125
98
  WorkerPool,
99
+ seal_process,
126
100
  )
101
+ from .world_host import DEFAULT_LEASE_S, DEFAULT_MAX_SESSIONS, HOSTED_OPERATIONS, WorldHost
127
102
 
128
- BUNDLED_WORLD = "server/world"
129
- WORLD_SPEC = "schema/world.json"
130
- DEFAULT_LEASE_S = 3600.0
131
- LEASE_SWEEP_S = 30.0
132
- OWNER_CACHE_S = 60.0
133
- #: Long enough that a client hammering a closed session costs the platform one call at a time,
134
- #: short enough that a session recovered onto another host is reached at once.
135
- OWNER_MISS_S = 2.0
136
- DEFAULT_MAX_SESSIONS = 1500
137
- #: The platform treats exactly this code (with a 404) as "the host lost the session".
138
- UNKNOWN_SESSION = "unknown_session"
139
- HEARTBEAT_S = 10.0
140
- DEFAULT_DRAIN_S = 45.0
141
- #: Only the framing is dropped: every header the world set must pass through a relay.
142
- FRAMING_HEADERS = frozenset({"content-length", "connection", "transfer-encoding", "server", "date"})
143
- HOP_BY_HOP_HEADERS = frozenset(
144
- {"keep-alive", "proxy-authenticate", "proxy-authorization", "te", "trailer", "upgrade", "proxy-connection"}
145
- )
146
- PROXIED_DROP_HEADERS = FRAMING_HEADERS | HOP_BY_HOP_HEADERS
147
-
148
-
149
- class _NoRedirect(urllib.request.HTTPRedirectHandler):
150
- """An owner's redirect is the client's to follow, not the proxy's: the ui
151
- token exchange answers 303 with the session cookie, and following it here
152
- would drop the cookie and land on the owner's 401."""
103
+ DEFAULT_DRAIN_S = 240.0
153
104
 
154
- def redirect_request(self, req, fp, code, msg, headers, newurl):
155
- return None
156
105
 
157
-
158
- _PROXY_OPENER = urllib.request.build_opener(_NoRedirect)
106
+ _BEARER = BearerAuth()
159
107
  #: Short-lived: the viewer's own grant was already checked, this one only has to survive the handshake.
160
108
  LIVE_RELAY_GRANT_MS = 60_000
161
- TRANSFER_TIMEOUT_S = 300.0
162
- #: Bounded because the cache is one shared disk and a build's memory is the supervisor's.
163
- DEFAULT_MAX_BUNDLE_MB = 2048
164
- MAX_BUNDLE_MEMBERS = 200_000
165
109
  #: A body is held in memory whole, in the supervisor shared by every session: a client must not size that allocation.
166
110
  DEFAULT_MAX_BODY_MB = 32
167
111
 
168
112
 
169
- OBJECT_STORE_ANY = "any"
170
- _ORIGIN_HOST = re.compile(r"^[A-Za-z0-9.-]+(:[0-9]+)?$")
171
-
172
-
173
- def parse_origins(value: str | None) -> frozenset[str] | None:
174
- """``WORLDS_HOST_OBJECT_STORE``: the origins (``scheme://host[:port]``,
175
- comma-separated) the platform's presigned URLs may point at, or the word
176
- ``any`` for a dev host reading ``file://`` bundles (``None``: every URL
177
- admitted). Unset is refused rather than read as ``any``: a deployed host
178
- with no allowlist is a host whose secret can aim it anywhere, and a
179
- template's placeholder (``https://<bucket>.s3...``) is refused by name."""
180
- if value is None or not value.strip():
181
- fail(
182
- "host",
183
- "WORLDS_HOST_OBJECT_STORE is unset; name the object store's origins "
184
- f"(https://bucket.s3.amazonaws.com) or {OBJECT_STORE_ANY!r} for a dev host",
185
- )
186
- raise AssertionError # fail() never returns
187
- if value.strip().lower() == OBJECT_STORE_ANY:
188
- return None
189
- origins = set()
190
- for item in value.split(","):
191
- item = item.strip()
192
- if not item:
193
- continue
194
- parts = urllib.parse.urlsplit(item)
195
- if parts.scheme not in ("http", "https") or not parts.netloc or parts.path not in ("", "/"):
196
- fail(
197
- "host",
198
- f"WORLDS_HOST_OBJECT_STORE entries are origins like https://bucket.s3.amazonaws.com, not {item!r}",
199
- )
200
- if not _ORIGIN_HOST.match(parts.netloc):
201
- fail(
202
- "host",
203
- f"WORLDS_HOST_OBJECT_STORE names a placeholder, not a host: {item!r}; set the real bucket origin",
204
- )
205
- origins.add(f"{parts.scheme}://{parts.netloc}".lower())
206
- return frozenset(origins)
207
-
208
-
209
- def check_object_store_url(url: object, origins: frozenset[str] | None, what: str) -> str:
210
- """The presigned URL the control plane handed us, once it is a string on one
211
- of the object store's origins. The control plane is trusted with the host
212
- secret, not with the host's network: a URL pointing anywhere else (the
213
- metadata service, a cluster Service, the internet) is refused before a
214
- byte moves, so a leaked secret cannot turn a session's state or the host's
215
- reach into someone else's download."""
216
- if not isinstance(url, str) or not url:
217
- fail("host", f"{what} must be a presigned url")
218
- raise AssertionError # fail() never returns
219
- if origins is None:
220
- return url
221
- parts = urllib.parse.urlsplit(url)
222
- origin = f"{parts.scheme}://{parts.netloc}".lower()
223
- if origin not in origins:
224
- fail("host", f"{what} must be on the object store ({', '.join(sorted(origins))}), not {origin}")
225
- return url
226
-
227
-
228
- def file_digest(path: Path) -> tuple[str, int]:
229
- """The hex sha256 and size of a file, streamed."""
230
- digest = hashlib.sha256()
231
- size = 0
232
- with open(path, "rb") as fh:
233
- for chunk in iter(lambda: fh.read(1 << 20), b""):
234
- digest.update(chunk)
235
- size += len(chunk)
236
- return digest.hexdigest(), size
237
-
238
-
239
- def download_verified(url: str, sha256: str, dest: Path) -> Path:
240
- """Fetch ``url`` into ``dest`` and refuse it unless its bytes hash to
241
- ``sha256``; a mismatch leaves nothing behind."""
242
- dest.parent.mkdir(parents=True, exist_ok=True)
243
- staging = dest.with_name(f".{dest.name}.{uuid.uuid4().hex}")
244
- try:
245
- try:
246
- with (
247
- urllib.request.urlopen(urllib.request.Request(url), timeout=TRANSFER_TIMEOUT_S) as r,
248
- open(staging, "wb") as out,
249
- ):
250
- shutil.copyfileobj(r, out)
251
- except Exception as error: # noqa: BLE001 - name the stage in the message
252
- fail("host", f"could not download the session export: {error}")
253
- refuse_digest_mismatch(staging, sha256)
254
- os.replace(staging, dest)
255
- finally:
256
- staging.unlink(missing_ok=True)
257
- return dest
258
-
259
-
260
- def upload_file(url: str, path: Path, sha256: str, size: int, content_type: str) -> None:
261
- """PUT a file to a presigned URL, sending exactly the headers the platform
262
- signed for it: content type, length and the base64 sha256 the object store
263
- verifies on receipt."""
264
- headers = {
265
- "Content-Type": content_type,
266
- "Content-Length": str(size),
267
- "x-amz-checksum-sha256": base64.b64encode(bytes.fromhex(sha256)).decode("ascii"),
268
- }
269
- with open(path, "rb") as fh:
270
- request = urllib.request.Request(url, data=fh, method="PUT", headers=headers)
271
- try:
272
- with urllib.request.urlopen(request, timeout=TRANSFER_TIMEOUT_S) as response:
273
- response.read()
274
- except urllib.error.HTTPError as error:
275
- fail("host", f"the export upload was refused: HTTP {error.code} {error.read()[:200]!r}")
276
- except urllib.error.URLError as error:
277
- fail("host", f"could not upload the export: {error.reason}")
278
-
279
-
280
- def world_tree(bundle: Path) -> Path:
281
- """The compiled schema world inside an extracted bundle (``WorldTree.world``:
282
- ``bench push`` packages a schema world as its api bundle with the tree
283
- under ``server/world``; a bare tree is its own root). The host opens
284
- compiled worlds only, so a tree without ``schema/world.json`` refuses."""
285
- tree = WorldTree.from_dir(bundle)
286
- if tree.has(tree.world_path(WORLD_SPEC)):
287
- return tree.world
288
- fail(
289
- "host",
290
- f"{bundle} holds no schema world the host can open (no {BUNDLED_WORLD}/{WORLD_SPEC} "
291
- f"nor {WORLD_SPEC}); the host serves schema worlds and api bundles only",
292
- )
293
- raise AssertionError # fail() never returns
294
-
295
-
296
- @dataclass
297
- class CachedVersion:
298
- """One cached version, of either shape the host serves: an api bundle (a
299
- connector ``spec`` and ``settings``, sessions cloned from ``snapshot``) or a
300
- schema world stored as itself (``tools``: handlers, tasks and verifiers,
301
- sessions copied from its template — and ``spec`` + ``settings`` too when
302
- its tree carries connector.toml, the HTTP surface the same session serves
303
- over the same store). ``snapshot`` and ``served`` are an api bundle's."""
304
-
305
- version_id: str
306
- bundle: Path
307
- tree: Path
308
- spec: Spec | None
309
- settings: ApiSettings | None
310
- snapshot: Path | None
311
- engine: type[WorldDatabase]
312
- #: None in the supervisor: only a worker imports world-authored code.
313
- handlers: dict[str, Handler] | None = None
314
- ui: UiSurface | None = None
315
- tools: ToolsVersion | None = None
316
- served: snapshots.ServedSnapshot | None = None
317
-
318
- @property
319
- def runtime_class(self) -> type[WorldRuntime]:
320
- """The one place a session's engine is chosen: a schema world runs as
321
- the tree it is; an api bundle on the snapshot READ PATH that step 6
322
- deletes with the bundle shape itself."""
323
- return TreeRuntime if self.tools is not None else SnapshotRuntime
324
-
325
- def router(self) -> Router:
326
- """A fresh router per session: routes are compiled from the shared spec,
327
- but the oauth2 tokens a world mints must stay that session's own."""
328
- assert self.spec is not None and self.settings is not None
329
- return Router(self.spec, self.settings)
330
-
331
- @property
332
- def authored_code(self) -> str | None:
333
- """What world-authored code the version runs, None for none: only a
334
- worker may open a version that runs any."""
335
- return self.runtime_class.authored_code(self)
336
-
337
- @property
338
- def needs_worker(self) -> bool:
339
- return self.authored_code is not None
340
-
341
- def served_names(self, dependencies: dict[str, DependencyWorld]) -> ServedNames:
342
- """The tools a session task may allow, beside the routes it may not."""
343
- return ServedNames.of(self.runtime_class.served_by(self, dependencies), self.spec)
344
-
345
- @property
346
- def declared_dependencies(self) -> dict[str, Dependency]:
347
- return self.runtime_class.declared_dependencies(self)
348
-
349
-
350
- class VersionCache:
351
- """Every world version this host has seen, on local disk. The first session
352
- of a version pays for the download and extract; the rest clone its
353
- snapshot. Nothing here is per session, so a hot version is cheap to serve
354
- many times over."""
355
-
356
- def __init__(self, root: Path, *, run_handlers: bool = False, max_bundle_mb: int = DEFAULT_MAX_BUNDLE_MB):
357
- self.root = Path(root)
358
- self.root.mkdir(parents=True, exist_ok=True)
359
- #: Off in the supervisor: only a version worker imports world-authored code.
360
- self.run_handlers = run_handlers
361
- self.max_bundle_bytes = max_bundle_mb * (1 << 20)
362
- self._lock = threading.Lock()
363
- self._versions: dict[str, CachedVersion] = {}
364
- self._builds = Lanes()
365
-
366
- def ensure(
367
- self,
368
- version_id: str,
369
- bundle_url: str,
370
- snapshot: dict[str, str] | None,
371
- ) -> CachedVersion:
372
- hit = self._cached(version_id)
373
- if hit is not None:
374
- return hit
375
- with self._builds.turn(version_id):
376
- return self._cached(version_id) or self._store(self._build(version_id, bundle_url, snapshot))
377
-
378
- def _cached(self, version_id: str) -> CachedVersion | None:
379
- with self._lock:
380
- return self._versions.get(version_id)
381
-
382
- def _store(self, cached: CachedVersion) -> CachedVersion:
383
- with self._lock:
384
- self._versions[cached.version_id] = cached
385
- return cached
386
-
387
- def has(self, version_id: str) -> bool:
388
- with self._lock:
389
- return version_id in self._versions
390
-
391
- def version_ids(self) -> list[str]:
392
- with self._lock:
393
- return sorted(self._versions)
394
-
395
- def _build(self, version_id: str, bundle_url: str, snapshot: dict[str, str] | None) -> CachedVersion:
396
- bundle = self.root / version_id / "bundle"
397
- if not bundle.is_dir():
398
- self._download(bundle_url, bundle)
399
- if is_tools_bundle(bundle):
400
- tools = load_tools_version(
401
- bundle, self.root / version_id / "template.db", snapshot, run_handlers=self.run_handlers
402
- )
403
- return CachedVersion(
404
- version_id=version_id,
405
- bundle=bundle,
406
- tree=bundle,
407
- spec=tools.spec,
408
- settings=tools.settings,
409
- snapshot=None,
410
- engine=tools.engine,
411
- handlers=tools.api_handlers,
412
- ui=load_ui_surface(bundle) if tools.spec is not None else None,
413
- tools=tools,
414
- )
415
- tree = world_tree(bundle)
416
- contract = check_bundle(tree)
417
- if snapshot:
418
- snapshots.fetch(tree, contract, snapshot["url"], snapshot["sha256"], snapshot["digest"])
419
- served = snapshots.ServedSnapshot.of(tree)
420
- spec = load_spec(tree)
421
- settings = load_api_settings(tree, spec)
422
- handlers = load_handlers(settings) if self.run_handlers and settings.handlers is not None else None
423
- snap = snapshots.default_snapshot(tree, contract)
424
- if not snap.is_file():
425
- fail(
426
- "host",
427
- f"version {version_id} has no compiled snapshot ({snap.relative_to(tree)}); "
428
- "publish a data snapshot or pin one on the version",
429
- )
430
- return CachedVersion(
431
- version_id=version_id,
432
- bundle=bundle,
433
- tree=tree,
434
- spec=spec,
435
- settings=settings,
436
- snapshot=snap,
437
- engine=WorldDatabase.for_contract(contract),
438
- served=served,
439
- handlers=handlers,
440
- ui=load_ui_surface(bundle),
441
- )
442
-
443
- def _download(self, url: str, dest: Path) -> None:
444
- """Fetch and extract into a sibling staging directory, then rename it into
445
- place: ``dest`` exists only once it is complete, so a download that dies
446
- halfway is never mistaken for a cached bundle on the next open."""
447
- dest.parent.mkdir(parents=True, exist_ok=True)
448
- staging = dest.parent / f".{dest.name}.{uuid.uuid4().hex}"
449
- tgz = staging.with_suffix(".tar.gz")
450
- limit = self.max_bundle_bytes
451
- try:
452
- try:
453
- with (
454
- urllib.request.urlopen(urllib.request.Request(url), timeout=TRANSFER_TIMEOUT_S) as r,
455
- open(tgz, "wb") as out,
456
- ):
457
- copied = 0
458
- for chunk in iter(lambda: r.read(1 << 20), b""):
459
- copied += len(chunk)
460
- if copied > limit:
461
- raise ValueError(f"the bundle is larger than {limit} bytes")
462
- out.write(chunk)
463
- except Exception as error: # noqa: BLE001 - name the stage in the message
464
- fail("host", f"could not download the bundle: {error}")
465
- staging.mkdir()
466
- with tarfile.open(tgz, "r:gz") as t:
467
- members = t.getmembers()
468
- if len(members) > MAX_BUNDLE_MEMBERS:
469
- fail("host", f"the bundle holds {len(members)} files, more than the {MAX_BUNDLE_MEMBERS} allowed")
470
- extracted = sum(m.size for m in members)
471
- if extracted > limit:
472
- fail("host", f"the bundle extracts to {extracted} bytes, more than the {limit} allowed")
473
- safe_tar_extract(t, staging)
474
- os.replace(staging, dest)
475
- finally:
476
- tgz.unlink(missing_ok=True)
477
- shutil.rmtree(staging, ignore_errors=True)
478
-
479
-
480
- @dataclass
481
- class Export:
482
- """A session's state copied to a file on this host, hashed, waiting for the
483
- platform to hand over the presigned URL it goes to."""
484
-
485
- export_id: str
486
- path: Path
487
- sha256: str
488
- bytes: int
489
- content_type: str
490
-
491
- def describe(self) -> dict:
492
- return {
493
- "exportId": self.export_id,
494
- "sha256": self.sha256,
495
- "bytes": self.bytes,
496
- "contentType": self.content_type,
497
- }
498
-
499
-
500
- @dataclass
501
- class Session:
502
- session_id: str
503
- version_id: str
504
- task: str | None
505
- spec: Spec | None
506
- token: str | None
507
- bundle: Path
508
- tree: Path
509
- lease_until: float
510
- engine: type[WorldDatabase]
511
- runtime: WorldRuntime | None = None
512
- worker: Worker | None = None
513
- ui: UiSurface | None = None
514
- browser: BrowserRelay | None = None
515
- #: Teardown waits for zero, so a close or expiry never pulls the database from under a handler.
516
- inflight: int = 0
517
- #: Once set, the last request out closes the state.
518
- closing: bool = False
519
- opened_at: float = field(default_factory=time.time)
520
- #: A renewal or an export is the platform looking, not the session working, and must not move it:
521
- #: the platform's idle clock follows this.
522
- touched_at: float = field(default_factory=time.time)
523
- export: Export | None = None
524
- restored: bool = False
525
- task_spec: TaskSpec | None = None
526
- exporting: threading.Lock = field(default_factory=threading.Lock, repr=False, compare=False)
527
-
528
- @property
529
- def app(self) -> WorldApp | None:
530
- """The world's routes over this session's store, when this process
531
- serves it and the tree has any."""
532
- return self.runtime.app if self.runtime is not None else None
533
-
534
- @property
535
- def task_key(self) -> tuple[str | None, dict | None]:
536
- """What an open retry must match: the bundled name or the spec itself."""
537
- return self.task, self.task_spec.raw if self.task_spec is not None else None
538
-
539
- def renew(self, lease_s: float) -> None:
540
- self.lease_until = time.time() + lease_s
541
-
542
- @property
543
- def seconds(self) -> float:
544
- return max(0.0, time.time() - self.opened_at)
545
-
546
-
547
- class WorldHost:
548
- """The session registry and the control operations, in one of two roles:
549
-
550
- * the **supervisor** (the deployed host): serves code-free worlds in
551
- process and places worlds with handlers on its :class:`WorkerPool`;
552
- * a **worker** (``run_handlers=True``, ``pinned_version=...``): serves one
553
- version, handlers included, for its supervisor.
554
- """
555
-
556
- def __init__(
557
- self,
558
- cache_root: Path,
559
- *,
560
- lease_s: float = DEFAULT_LEASE_S,
561
- advertise_url: str | None = None,
562
- workers: WorkerPool | None = None,
563
- versions_root: Path | None = None,
564
- run_handlers: bool = False,
565
- pinned_version: str | None = None,
566
- limits: dict[str, int] | None = None,
567
- max_sessions: int = DEFAULT_MAX_SESSIONS,
568
- max_bundle_mb: int = DEFAULT_MAX_BUNDLE_MB,
569
- object_store: frozenset[str] | None = None,
570
- ):
571
- #: Every presigned URL the control plane hands this host must point here; None admits any.
572
- self.object_store = object_store
573
- self.cache = VersionCache(
574
- versions_root or Path(cache_root) / "versions", run_handlers=run_handlers, max_bundle_mb=max_bundle_mb
575
- )
576
- self.sessions_root = Path(cache_root) / "sessions"
577
- # Sessions live in this process, so whatever a previous one left on disk is unreachable.
578
- shutil.rmtree(self.sessions_root, ignore_errors=True)
579
- self.sessions_root.mkdir(parents=True, exist_ok=True)
580
- self.lease_s = lease_s
581
- self.advertise_url = advertise_url
582
- self.workers = workers
583
- self.run_handlers = run_handlers
584
- self.pinned_version = pinned_version
585
- self.limits = limits
586
- self.max_sessions = max_sessions
587
- self.draining = False
588
- self.started_at = time.time()
589
- self._lock = threading.Lock()
590
- self._sessions: dict[str, Session] = {}
591
- #: Each holds a slot from admission, so concurrent opens cannot all pass the capacity check.
592
- self._opening: set[str] = set()
593
- #: Opens the platform closed before they finished: closed the moment they exist, not at lease end.
594
- self._abandoned: set[str] = set()
595
- self._opens = Lanes()
596
- self._counters: Counter[str] = Counter()
597
- self._session_seconds = 0.0
598
-
599
- def open(
600
- self,
601
- session_id: str,
602
- version_id: str,
603
- bundle_url: str,
604
- task: str | None,
605
- token: str | None,
606
- snapshot: dict[str, str] | None,
607
- browser: dict[str, str] | None = None,
608
- restore: dict[str, str] | None = None,
609
- task_spec: Any = None,
610
- dependencies: list[dict[str, Any]] | None = None,
611
- ) -> dict:
612
- """Open ``session_id`` on ``version_id``, either on the bundled ``task``
613
- or on ``task_spec``, a session task of its own (:mod:`tasks`); exactly
614
- one of the two. ``dependencies`` are the versions the world's
615
- ``[dependencies]`` resolved to, one ``{alias, versionId, bundleUrl,
616
- snapshot?}`` per alias the manifest declares, opened beside it."""
617
- if self.pinned_version is not None and version_id != self.pinned_version:
618
- fail("host", f"this worker serves version {self.pinned_version} only, not {version_id}")
619
- if (task is None) == (task_spec is None):
620
- fail("host", "an open names a bundled task or carries a task spec, one of the two")
621
- if task is not None and not isinstance(task, str):
622
- fail("host", "task is the bundled task's name")
623
- spec = parse_task_spec(task_spec) if task_spec is not None else None
624
- deps = parse_dependency_refs(dependencies)
625
- with self._lock:
626
- existing = self._sessions.get(session_id)
627
- if existing is None:
628
- # A retry of an open already served is answered even while draining: refusing it would
629
- # send the platform to open the same id elsewhere while this copy holds the live writes.
630
- if self.draining:
631
- fail("draining", "this host is draining and takes no new sessions")
632
- self._admit(session_id)
633
- if existing is not None:
634
- # A retried open gets the same session, never a rebuild that drops its writes.
635
- wanted = (version_id, (task, spec.raw if spec is not None else None), token)
636
- if (existing.version_id, existing.task_key, existing.token) != wanted:
637
- fail("host", f"session {session_id} is already open as another version, task or token")
638
- return self._describe(existing)
639
- try:
640
- described = self._open(
641
- session_id, version_id, bundle_url, task, spec, token, snapshot, browser, restore, deps
642
- )
643
- except Exception:
644
- self.tally("open_failures")
645
- with self._lock:
646
- self._opening.discard(session_id)
647
- self._abandoned.discard(session_id)
648
- raise
649
- with self._lock:
650
- self._opening.discard(session_id)
651
- abandoned = session_id in self._abandoned
652
- self._abandoned.discard(session_id)
653
- if abandoned:
654
- self.close(session_id)
655
- fail(UNKNOWN_SESSION, f"session {session_id} was closed while it opened")
656
- return described
657
-
658
- def _admit(self, session_id: str) -> None:
659
- """Reserve one of this host's ``max_sessions`` for ``session_id`` while it
660
- opens. Under ``_lock``."""
661
- if session_id in self._opening:
662
- fail("host", f"session {session_id} is being opened already")
663
- if len(self._sessions) + len(self._opening) >= self.max_sessions:
664
- fail("at_capacity", f"this host holds its {self.max_sessions} sessions; place elsewhere")
665
- self._opening.add(session_id)
666
-
667
- def _open(
668
- self,
669
- session_id: str,
670
- version_id: str,
671
- bundle_url: str,
672
- task: str | None,
673
- spec: TaskSpec | None,
674
- token: str | None,
675
- snapshot: dict[str, str] | None,
676
- browser: dict[str, str] | None,
677
- restore: dict[str, str] | None,
678
- deps: list[DependencyRef],
679
- ) -> dict:
680
- cached = self.cache.ensure(version_id, *self._trusted_version(bundle_url, snapshot))
681
- dependencies = self._dependency_worlds(cached, deps)
682
- if cached.spec is not None:
683
- unsupported = unsupported_auth(cached.spec)
684
- if unsupported:
685
- fail("host", unsupported)
686
- if spec is not None:
687
- cached.served_names(dependencies).check_allowed(spec.tools)
688
- relay = BrowserRelay.parse(browser)
689
- if cached.needs_worker and not self.run_handlers:
690
- return self._open_on_worker(cached, session_id, bundle_url, task, spec, token, relay, restore, deps)
691
- session_dir = self.sessions_root / session_id
692
- with self._opens.turn(version_id):
693
- runtime = WorldRuntime.open(
694
- cached,
695
- session_dir,
696
- task=task,
697
- spec=spec,
698
- token=token,
699
- dependencies=dependencies,
700
- restore=lambda load: self._restore_into(restore, session_dir, load, cached.engine),
701
- )
702
- session = Session(
703
- session_id=session_id,
704
- version_id=version_id,
705
- task=task,
706
- spec=cached.spec,
707
- token=token,
708
- bundle=cached.bundle,
709
- tree=cached.tree,
710
- lease_until=time.time() + self.lease_s,
711
- engine=cached.engine,
712
- runtime=runtime,
713
- ui=cached.ui,
714
- browser=relay,
715
- restored=runtime.restored,
716
- task_spec=spec,
717
- )
718
- self._register(session)
719
- return self._describe(session)
720
-
721
- def _open_on_worker(
722
- self,
723
- cached: CachedVersion,
724
- session_id: str,
725
- bundle_url: str,
726
- task: str | None,
727
- spec: TaskSpec | None,
728
- token: str | None,
729
- relay: BrowserRelay | None,
730
- restore: dict[str, str] | None,
731
- deps: list[DependencyRef],
732
- ) -> dict:
733
- """Place the session on the version's worker. The worker opens it from
734
- the bundle this process already cached (no snapshot to fetch: the file
735
- is on disk), and the supervisor records only where it went. The ui's
736
- static files and the browser tunnel stay the supervisor's; the ui's
737
- api-prefix calls are relayed as they came, so the worker logs them
738
- ``via: ui``. An export to restore from is fetched and verified here
739
- too, then handed to the worker as a local file with its sha256, which
740
- the worker checks again before it opens the file.
741
- A worker every one of whose peers holds sessions cannot be started
742
- past the pool's memory budget: that is ``at_capacity``, and the
743
- platform places the session on another replica."""
744
- if self.workers is None:
745
- fail(
746
- "host",
747
- f"version {cached.version_id} runs world-authored code ({cached.authored_code}); "
748
- "this host runs none in its process and has no version workers",
749
- )
750
- try:
751
- worker = self.workers.get(cached.version_id)
752
- except WorkerCapacity as error:
753
- fail("at_capacity", str(error))
754
- # Claimed before placement, so another open cannot stop this worker as session-less meanwhile.
755
- with self._lock:
756
- worker.sessions.add(session_id)
757
- source = None
758
- try:
759
- source = self._restore_source(restore, self._worker_session_dir(worker, session_id))
760
- with self._opens.turn(cached.version_id):
761
- self._worker_control(
762
- worker,
763
- "POST",
764
- "/__host/sessions",
765
- {
766
- "sessionId": session_id,
767
- "versionId": cached.version_id,
768
- "bundleUrl": bundle_url,
769
- "task": task,
770
- "taskSpec": spec.raw if spec is not None else None,
771
- "token": token,
772
- "restore": source.for_worker() if source is not None else None,
773
- "dependencies": [dep.raw for dep in deps],
774
- },
775
- )
776
- except BaseException:
777
- with self._lock:
778
- worker.sessions.discard(session_id)
779
- if not worker.sessions:
780
- worker.idle_since = time.time()
781
- raise
782
- finally:
783
- if source is not None:
784
- source.path.unlink(missing_ok=True)
785
- session = Session(
786
- session_id=session_id,
787
- version_id=cached.version_id,
788
- task=task,
789
- spec=cached.spec,
790
- token=token,
791
- bundle=cached.bundle,
792
- tree=cached.tree,
793
- lease_until=time.time() + self.lease_s,
794
- engine=cached.engine,
795
- worker=worker,
796
- ui=cached.ui,
797
- browser=relay,
798
- restored=source is not None,
799
- task_spec=spec,
800
- )
801
- self._register(session)
802
- return self._describe(session)
803
-
804
- def _worker_session_dir(self, worker: Worker, session_id: str) -> Path:
805
- """Where a worker keeps a session's files: its own ``WorldHost`` runs
806
- under ``workers/<version>`` of this host's cache root (:mod:`worker`)."""
807
- assert self.workers is not None
808
- return self.workers.root / "workers" / worker.version_id / "sessions" / session_id
809
-
810
- def _restore_into(
811
- self,
812
- restore: dict[str, str] | None,
813
- session_dir: Path,
814
- load: Callable[[SessionExport], None],
815
- engine: type[WorldDatabase],
816
- ) -> bool:
817
- """Load the export ``restore`` names into a freshly opened world, its
818
- bytes checked against the recorded sha256 here, where they are opened,
819
- then drop the fetched file. False when there was nothing to restore."""
820
- source = self._restore_source(restore, session_dir)
821
- if source is None:
822
- return False
823
- try:
824
- load(SessionExport.verified(source, engine))
825
- finally:
826
- source.path.unlink(missing_ok=True)
827
- return True
828
-
829
- def _restore_source(self, restore: dict[str, str] | None, session_dir: Path) -> ExportFile | None:
830
- """The export a session continues from, as a local file and the sha256
831
- the platform recorded for it, or None for a fresh session. The platform
832
- names it by ``{url, sha256}`` and the file is fetched into the session's
833
- directory and verified; a supervisor names it to its worker by ``{path,
834
- sha256}``, and only the supervisor reaches a worker's control socket."""
835
- if not restore:
836
- return None
837
- path = restore.get("path")
838
- if path is not None:
839
- return self._worker_restore_source(Path(path), restore.get("sha256"))
840
- url, sha256 = restore.get("url"), restore.get("sha256")
841
- if not isinstance(url, str) or not isinstance(sha256, str) or len(sha256) != 64:
842
- fail("host", "restore must be {url: <presigned export url>, sha256: <hex digest>}")
843
- raise AssertionError # fail() never returns
844
- check_object_store_url(url, self.object_store, "restore.url")
845
- return ExportFile(download_verified(url, sha256, session_dir / "restore.db"), sha256)
846
-
847
- def _worker_restore_source(self, path: Path, sha256: object) -> ExportFile:
848
- if not self.run_handlers:
849
- fail("host", "restore by path is a worker's; name the export by {url, sha256}")
850
- if not path.is_file():
851
- fail("host", f"restore file {path} is missing")
852
- if not isinstance(sha256, str) or len(sha256) != 64:
853
- fail("host", "restore by path names the export's sha256 beside it")
854
- raise AssertionError # fail() never returns
855
- return ExportFile(path, sha256)
856
-
857
- def _trusted_version(self, bundle_url: str, snapshot: dict[str, str] | None) -> tuple[str, dict[str, str] | None]:
858
- check_object_store_url(bundle_url, self.object_store, "bundleUrl")
859
- if snapshot:
860
- check_object_store_url(snapshot.get("url"), self.object_store, "snapshot.url")
861
- return bundle_url, snapshot
862
-
863
- def _register(self, session: Session) -> None:
864
- with self._lock:
865
- replaced = self._retire(self._sessions.pop(session.session_id, None))
866
- self._sessions[session.session_id] = session
867
- self._counters["opened"] += 1
868
- if replaced is not None:
869
- self._finish(replaced, reason="replaced")
870
-
871
- def _describe(self, session: Session) -> dict:
872
- return {
873
- "sessionId": session.session_id,
874
- "versionId": session.version_id,
875
- "task": session.task,
876
- "taskName": session.task if session.task_spec is None else session.task_spec.name,
877
- "taskSource": "bundled" if session.task_spec is None else "inline",
878
- "tools": (
879
- list(session.task_spec.tools)
880
- if session.task_spec is not None and session.task_spec.tools is not None
881
- else None
882
- ),
883
- "ready": True,
884
- "hostUrl": self.advertise_url,
885
- "leaseSeconds": self.lease_s,
886
- "leaseUntil": session.lease_until,
887
- "workerPid": session.worker.pid if session.worker is not None else None,
888
- "ui": {"entry": session.ui.entry, "apiPath": session.ui.api_path} if session.ui is not None else None,
889
- "browser": session.browser is not None,
890
- "restored": session.restored,
891
- "openedAt": session.opened_at,
892
- "sessionSeconds": session.seconds,
893
- "touchedAt": session.touched_at,
894
- }
895
-
896
- def close(self, session_id: str) -> dict:
897
- with self._lock:
898
- session = self._sessions.pop(session_id, None)
899
- idle = self._retire(session)
900
- if session is None and session_id in self._opening:
901
- self._abandoned.add(session_id)
902
- if idle is not None:
903
- self._finish(idle, reason="closed")
904
- if session is None:
905
- return {"closed": True, "openedAt": None, "sessionSeconds": 0.0}
906
- return {"closed": True, "openedAt": session.opened_at, "sessionSeconds": session.seconds}
907
-
908
- def set_browser(self, session_id: str, browser: Any) -> dict:
909
- """Point the session's browser surface at another pool relay: the one
910
- it had died and the platform leased a replacement. Connections open
911
- on the old relay end with it; every connection after this reaches the
912
- new one. A session opened without a browser never gains one here."""
913
- relay = BrowserRelay.parse(browser)
914
- if relay is None:
915
- fail("host", "browser must be {url: 'ws://host:port', token: <relay token>}")
916
- with self._lock:
917
- session = self._sessions.get(session_id)
918
- if session is None:
919
- fail(UNKNOWN_SESSION, f"no session {session_id} on this host")
920
- if session.browser is None:
921
- fail("host", f"session {session_id} was opened without a browser surface")
922
- session.browser = relay
923
- return {"sessionId": session_id, "browser": True}
924
-
925
- def export(self, session_id: str, known_sha256: str | None = None) -> dict:
926
- """Copy the session's state to a file on this host and hash it, one export of
927
- a session at a time; it waits for :meth:`upload_export`, and a new one discards
928
- the last. A copy hashing to ``known_sha256`` (the platform's last export of the
929
- session) is thrown away and reported ``unchanged``: nothing to upload."""
930
- with self.using(session_id, touch=False) as session, session.exporting:
931
- engine = session.engine
932
- if session.worker is not None:
933
- answer = self._worker_control(session.worker, "POST", f"/__host/sessions/{session_id}/export", {})
934
- path = self._worker_export_path(session, answer)
935
- else:
936
- exports = self.sessions_root / session_id / "exports"
937
- shutil.rmtree(exports, ignore_errors=True)
938
- path = exports / f"{uuid.uuid4().hex}{Path(engine.snapshot_name).suffix}"
939
- assert session.runtime is not None
940
- session.runtime.export(path)
941
- sha256, size = file_digest(path)
942
- if known_sha256 is not None and sha256 == known_sha256:
943
- path.unlink(missing_ok=True)
944
- with self._lock:
945
- session.export = None
946
- self._counters["exports_unchanged"] += 1
947
- return {"exportId": None, "sha256": sha256, "bytes": size, "unchanged": True}
948
- export = Export(export_id=path.stem, path=path, sha256=sha256, bytes=size, content_type=engine.content_type)
949
- with self._lock:
950
- session.export = export
951
- self._counters["exports"] += 1
952
- described = export.describe()
953
- if self.run_handlers:
954
- described["path"] = str(path) # for the supervisor, which uploads on the worker's behalf
955
- return described
956
-
957
- def _worker_export_path(self, session: Session, answer: dict) -> Path:
958
- """The file a worker's export landed in, admitted only from inside that
959
- worker's directory for this session: the worker runs tenant code, so
960
- what it names is checked, and the bytes are hashed here, not trusted."""
961
- assert session.worker is not None
962
- exports = self._worker_session_dir(session.worker, session.session_id) / "exports"
963
- path = Path(str(answer.get("path") or ""))
964
- try:
965
- resolved = path.resolve(strict=True)
966
- resolved.relative_to(exports.resolve())
967
- except (OSError, ValueError):
968
- fail("host", f"the worker's export is not where a worker's exports go: {path}")
969
- raise AssertionError # fail() never returns
970
- return resolved
971
-
972
- def upload_export(self, session_id: str, export_id: str, url: str) -> dict:
973
- """PUT the export :meth:`export` took to the presigned URL the platform
974
- signed for its hash and size; the local copy goes once it is up."""
975
- check_object_store_url(url, self.object_store, "url")
976
- with self.using(session_id, touch=False) as session:
977
- export = session.export
978
- if export is None or export.export_id != export_id:
979
- fail("host", f"no export {export_id} for session {session_id}; take one first")
980
- raise AssertionError # fail() never returns
981
- upload_file(url, export.path, export.sha256, export.bytes, export.content_type)
982
- export.path.unlink(missing_ok=True)
983
- with self._lock:
984
- session.export = None
985
- self._counters["exports_uploaded"] += 1
986
- return {"uploaded": True, **export.describe()}
987
-
988
- def renew(self, session_id: str) -> dict:
989
- with self.using(session_id, touch=False) as session:
990
- return self._describe(session)
991
-
992
- def expire_leases(self, now: float | None = None) -> list[str]:
993
- """Close every session whose lease has run out; returns their ids. The
994
- check and the removal happen under one lock, so a renewal that lands
995
- in between keeps its session, and a session with a request in flight
996
- is left for the next sweep (the request renewed it anyway)."""
997
- now = time.time() if now is None else now
998
- with self._lock:
999
- expired = [s for s in self._sessions.values() if s.lease_until <= now and s.inflight == 0]
1000
- for session in expired:
1001
- del self._sessions[session.session_id]
1002
- self._retire(session)
1003
- for session in expired:
1004
- self._finish(session, reason="expired")
1005
- return [s.session_id for s in expired]
1006
-
1007
- def _retire(self, session: Session | None) -> Session | None:
1008
- """Mark a session that has left the registry; returns it when nobody is
1009
- inside it, so the caller closes the state now. Otherwise the last
1010
- request out does (see :meth:`_release`). Call under the lock."""
1011
- if session is None:
1012
- return None
1013
- session.closing = True
1014
- return session if session.inflight == 0 else None
1015
-
1016
- def _finish(self, session: Session, *, reason: str = "closed") -> None:
1017
- """Free a session that has left the registry and has nobody inside it:
1018
- close its state here, or tell its worker to."""
1019
- with self._lock:
1020
- self._counters[f"closed.{reason}"] += 1
1021
- self._session_seconds += session.seconds
1022
- if session.worker is None:
1023
- assert session.runtime is not None
1024
- session.runtime.close()
1025
- return
1026
- worker = session.worker
1027
- with self._lock:
1028
- worker.sessions.discard(session.session_id)
1029
- if not worker.sessions:
1030
- worker.idle_since = time.time()
1031
- try:
1032
- self._worker_control(worker, "DELETE", f"/__host/sessions/{session.session_id}", None)
1033
- except (WorkerGone, ContractError):
1034
- pass # the worker is gone, or already forgot it: either way the session is over
1035
-
1036
- def _worker_control(self, worker: Worker, method: str, path: str, body: dict | None) -> dict:
1037
- """One control call to a worker, its answer as this host would give it:
1038
- the worker's refusal is re-raised as ours, and a worker that has died
1039
- takes every session it held with it."""
1040
- raw = json.dumps(body).encode("utf-8") if body is not None else b""
1041
- try:
1042
- status, _, payload = worker.request(method, path, {"content-type": "application/json"}, raw, control=True)
1043
- except WorkerGone:
1044
- self.lose_worker(worker)
1045
- raise
1046
- answer = json.loads(payload or b"null")
1047
- if status != 200:
1048
- detail = answer if isinstance(answer, dict) else {}
1049
- fail(str(detail.get("error") or "host"), str(detail.get("message") or detail.get("detail") or answer))
1050
- return answer if isinstance(answer, dict) else {"result": answer}
1051
-
1052
- def lose_worker(self, worker: Worker) -> list[str]:
1053
- """A worker exited: forget it and every session it held. Returns the
1054
- session ids, which are now gone from this host (the platform learns of
1055
- it on the next renew)."""
1056
- if self.workers is not None:
1057
- self.workers.drop(worker.version_id)
1058
- with self._lock:
1059
- lost = [sid for sid, s in self._sessions.items() if s.worker is worker]
1060
- for sid in lost:
1061
- session = self._sessions.pop(sid)
1062
- session.closing = True
1063
- self._counters["closed.lost"] += 1
1064
- self._session_seconds += session.seconds
1065
- worker.sessions.clear()
1066
- return lost
1067
-
1068
- def reap_workers(self) -> list[str]:
1069
- """Stop version workers that have had no session for a while."""
1070
- return self.workers.reap_idle() if self.workers is not None else []
1071
-
1072
- def session_count(self) -> int:
1073
- with self._lock:
1074
- return len(self._sessions)
1075
-
1076
- def worker_count(self) -> int:
1077
- return len(self.workers._workers) if self.workers is not None else 0
1078
-
1079
- def warm(self, version_id: str, bundle_url: str, snapshot: dict[str, str] | None) -> dict:
1080
- """Cache a version before any session asks for it, so the first open
1081
- pays nothing. Idempotent: a version already here is reported, not
1082
- rebuilt."""
1083
- already = self.cache.has(version_id)
1084
- if not already:
1085
- self.cache.ensure(version_id, *self._trusted_version(bundle_url, snapshot))
1086
- return {"versionId": version_id, "cached": True, "alreadyCached": already}
1087
-
1088
- def drain(self) -> None:
1089
- """Stop taking sessions; the ones here are served until they close."""
1090
- self.draining = True
1091
-
1092
- def tally(self, key: str) -> None:
1093
- with self._lock:
1094
- self._counters[key] += 1
1095
-
1096
- def stats(self) -> dict:
1097
- """What this host is and holds, for the heartbeat and ``/__host/stats``:
1098
- the placement inputs (address, load, capacity, draining, which versions
1099
- are already cached) plus the lifetime counters."""
1100
- with self._lock:
1101
- sessions = len(self._sessions)
1102
- versions_in_use = sorted({s.version_id for s in self._sessions.values()})
1103
- counters = dict(self._counters)
1104
- session_seconds = self._session_seconds
1105
- return {
1106
- "hostUrl": self.advertise_url,
1107
- "sessions": sessions,
1108
- "capacity": self.max_sessions,
1109
- "draining": self.draining,
1110
- "workers": self.worker_count(),
1111
- "maxWorkers": self.workers.max_workers if self.workers is not None else 0,
1112
- "versions": self.cache.version_ids(),
1113
- "versionsInUse": versions_in_use,
1114
- "uptimeSeconds": time.time() - self.started_at,
1115
- "counters": counters,
1116
- "sessionSeconds": session_seconds,
1117
- "pid": os.getpid(),
1118
- "limits": self.limits,
1119
- }
1120
-
1121
- def metrics_text(self) -> str:
1122
- """The prometheus exposition of :meth:`stats`. ``worlds_host_sessions``
1123
- is the number the fleet scales on (sessions per host, not cpu)."""
1124
- stats = self.stats()
1125
- counters: dict[str, int] = stats["counters"]
1126
- lines = [
1127
- "# TYPE worlds_host_sessions gauge",
1128
- f"worlds_host_sessions {stats['sessions']}",
1129
- "# TYPE worlds_host_capacity gauge",
1130
- f"worlds_host_capacity {stats['capacity']}",
1131
- "# TYPE worlds_host_draining gauge",
1132
- f"worlds_host_draining {int(stats['draining'])}",
1133
- "# TYPE worlds_host_workers gauge",
1134
- f"worlds_host_workers {stats['workers']}",
1135
- "# TYPE worlds_host_versions_cached gauge",
1136
- f"worlds_host_versions_cached {len(stats['versions'])}",
1137
- "# TYPE worlds_host_uptime_seconds gauge",
1138
- f"worlds_host_uptime_seconds {stats['uptimeSeconds']:.0f}",
1139
- "# TYPE worlds_host_sessions_opened_total counter",
1140
- f"worlds_host_sessions_opened_total {counters.get('opened', 0)}",
1141
- "# TYPE worlds_host_open_failures_total counter",
1142
- f"worlds_host_open_failures_total {counters.get('open_failures', 0)}",
1143
- "# TYPE worlds_host_session_seconds_total counter",
1144
- f"worlds_host_session_seconds_total {stats['sessionSeconds']:.0f}",
1145
- "# TYPE worlds_host_exports_total counter",
1146
- f"worlds_host_exports_total {counters.get('exports', 0)}",
1147
- "# TYPE worlds_host_exports_uploaded_total counter",
1148
- f"worlds_host_exports_uploaded_total {counters.get('exports_uploaded', 0)}",
1149
- "# TYPE worlds_host_sessions_closed_total counter",
1150
- ]
1151
- for reason in ("closed", "expired", "lost", "replaced"):
1152
- lines.append(
1153
- f'worlds_host_sessions_closed_total{{reason="{reason}"}} {counters.get(f"closed.{reason}", 0)}'
1154
- )
1155
- lines.append("# TYPE worlds_host_requests_total counter")
1156
- for surface in ("api", "ui", "browser", "proxied"):
1157
- lines.append(f'worlds_host_requests_total{{surface="{surface}"}} {counters.get(f"requests.{surface}", 0)}')
1158
- return "\n".join(lines) + "\n"
1159
-
1160
- def _dependency_worlds(self, cached: CachedVersion, deps: list[DependencyRef]) -> dict[str, DependencyWorld]:
1161
- """The dependency versions a session opens beside ``cached``, cached the
1162
- way the root is (a worker imports their handlers too). Every alias the
1163
- root's manifest declares must be resolved, and nothing else: a version
1164
- without ``[dependencies]`` opened with any is refused, as is a
1165
- dependency that is not a schema world."""
1166
- declared = cached.declared_dependencies
1167
- if not deps:
1168
- if declared:
1169
- names = ", ".join(sorted(declared))
1170
- fail("host", f"version {cached.version_id} declares dependencies ({names}); the open resolved none")
1171
- return {}
1172
- worlds: dict[str, DependencyWorld] = {}
1173
- for ref in deps:
1174
- dependency = declared.get(ref.alias)
1175
- if dependency is None:
1176
- known = ", ".join(sorted(declared)) or "none"
1177
- fail("host", f"version {cached.version_id} declares no dependency '{ref.alias}' (it declares: {known})")
1178
- raise AssertionError # fail() never returns
1179
- dep = self.cache.ensure(ref.version_id, *self._trusted_version(ref.bundle_url, ref.snapshot))
1180
- if dep.tools is None:
1181
- fail("host", f"dependency '{ref.alias}' ({ref.version_id}) is an api bundle, not a schema world")
1182
- raise AssertionError # fail() never returns
1183
- worlds[ref.alias] = DependencyWorld.of(dependency, dep.tools)
1184
- cached.served_names(worlds) # refuses a root tool that reads as a dependency's
1185
- return worlds
1186
-
1187
- @contextmanager
1188
- def using(self, session_id: str, *, touch: bool = True) -> Iterator[Session]:
1189
- """The session for the duration of one request: its lease renewed (any
1190
- traffic keeps it alive), its ``touched_at`` moved unless ``touch`` is
1191
- off (a renewal or export observing the session, not using it), and its
1192
- teardown held off until the block ends."""
1193
- with self._lock:
1194
- session = self._sessions.get(session_id)
1195
- if session is not None:
1196
- session.renew(self.lease_s)
1197
- if touch:
1198
- session.touched_at = time.time()
1199
- session.inflight += 1
1200
- if session is None:
1201
- fail(UNKNOWN_SESSION, f"no session {session_id} on this host")
1202
- raise AssertionError # fail() never returns
1203
- try:
1204
- yield session
1205
- finally:
1206
- self._release(session)
1207
-
1208
- def _release(self, session: Session) -> None:
1209
- with self._lock:
1210
- session.inflight -= 1
1211
- last_out = session.closing and session.inflight == 0
1212
- if last_out:
1213
- self._finish(session)
1214
-
1215
- def owns(self, session_id: str) -> bool:
1216
- with self._lock:
1217
- return session_id in self._sessions
1218
-
1219
- def _control(self, session: Session, method: str, verb: str, body: dict | None) -> dict:
1220
- return self._worker_control(session.worker, method, f"/__host/sessions/{session.session_id}/{verb}", body)
1221
-
1222
- def _verb(self, session_id: str, method: str, verb: str, body: dict | None, local: Callable[[Any], dict]) -> dict:
1223
- """One session verb: forwarded to the session's worker when it has
1224
- one, else answered by its in-process runtime through ``local``."""
1225
- with self.using(session_id) as session:
1226
- if session.worker is not None:
1227
- return self._control(session, method, verb, body)
1228
- assert session.runtime is not None
1229
- return local(session.runtime)
1230
-
1231
- def tools(self, session_id: str) -> dict:
1232
- return self._verb(session_id, "GET", "tools", None, lambda runtime: {"tools": runtime.served()})
1233
-
1234
- def task(self, session_id: str) -> dict:
1235
- return self._verb(session_id, "GET", "task", None, lambda runtime: runtime.task_card())
1236
-
1237
- def reset(self, session_id: str) -> dict:
1238
- return self._verb(session_id, "POST", "reset", {}, lambda runtime: runtime.reset())
1239
-
1240
- def seed(self, session_id: str, rows: dict, mode: str = "append", redact: str = "off") -> dict:
1241
- args = {"rows": rows, "mode": mode, **({"redact": redact} if redact != "off" else {})}
1242
- return self._verb(session_id, "POST", "seed", {"args": args}, lambda runtime: runtime.seed(rows, mode, redact))
1243
-
1244
- def state(self, session_id: str) -> dict:
1245
- return self._verb(session_id, "GET", "state", None, lambda runtime: runtime.state())
1246
-
1247
- def evidence(self, session_id: str) -> dict:
1248
- """The session's state as a grader reads it (rows keyed by primary key,
1249
- the calls): what the platform gathers from each world of a multi-world
1250
- task and hands another world's ``grade`` as ``context``."""
1251
- return self._verb(session_id, "GET", "evidence", None, lambda runtime: runtime.grader_state())
1252
-
1253
- def grade(self, session_id: str, report: Optional[str] = None, context: Optional[dict] = None) -> dict:
1254
- """Grade the session. ``report`` is the agent's own account of the run — the
1255
- text a rubric judge reads; here it is put beside the state as ``report`` so a
1256
- deterministic grader (assertions, verify.py) can check what the agent CLAIMED
1257
- against what the calls show, without a judge. ``context`` is the other
1258
- worlds' evidence the task's cross-world verifiers read (``{alias: state}``)."""
1259
- body = {key: value for key, value in (("report", report), ("context", context)) if value is not None}
1260
- return self._verb(session_id, "POST", "grade", body, lambda runtime: runtime.grade(report, context))
1261
-
1262
- def call(self, session_id: str, tool: str, args: dict) -> dict:
1263
- """One of the session's declared tools by name, a dependency's as
1264
- ``alias.tool``; a route's name is refused (routes answer over HTTP)."""
1265
- with self.using(session_id) as session:
1266
- if session.worker is not None:
1267
- return self._control(session, "POST", "call", {"tool": tool, "args": args})
1268
- assert session.runtime is not None
1269
- return session.runtime.call(tool, args)
1270
-
1271
-
1272
- #: ``local.py`` binds the same rows to its own operations, so the two cannot drift.
1273
- HOSTED_OPERATIONS: dict[str, Callable[..., dict]] = {
1274
- "GET tools": WorldHost.tools,
1275
- "GET task": WorldHost.task,
1276
- "GET state": WorldHost.state,
1277
- "GET evidence": WorldHost.evidence,
1278
- "POST grade": WorldHost.grade,
1279
- "POST reset": WorldHost.reset,
1280
- "POST seed": WorldHost.seed,
1281
- "POST call": WorldHost.call,
1282
- "POST close": WorldHost.close,
1283
- "POST renew": WorldHost.renew,
1284
- "POST export": WorldHost.export,
1285
- "DELETE <session>": WorldHost.close,
1286
- }
1287
- assert set(HOSTED_OPERATIONS) == {verb.route for verb in SESSION_VERBS}, "the host serves every session verb"
1288
-
1289
-
1290
- def public_host(host_header: str | None, worlds_domain: str) -> bool:
1291
- """True when ``Host`` names something under the worlds domain: a session,
1292
- or any other label the wildcard ALB forwarded. That is the public side.
1293
- Prometheus, the kubelet and the ALB health check address the pod itself
1294
- (``ip:port``), which is not."""
1295
- name = (host_header or "").split(":")[0].rstrip(".").lower()
1296
- domain = worlds_domain.rstrip(".").lower()
1297
- return bool(domain) and (name == domain or name.endswith("." + domain))
1298
-
1299
-
1300
- def session_from_host(host_header: str | None) -> tuple[str, str] | None:
1301
- """``w-<sid>-api.<domain>`` / ``w-<sid>.<domain>`` → (session id, surface).
1302
- Returns None when the host does not name a session."""
1303
- if not host_header:
1304
- return None
1305
- label = host_header.split(":")[0].split(".")[0]
1306
- if not label.startswith("w-"):
1307
- return None
1308
- rest = label[2:]
1309
- if rest.endswith("-api"):
1310
- return rest[: -len("-api")], "api"
1311
- if rest.endswith("-cdp"):
1312
- return rest[: -len("-cdp")], "browser"
1313
- return rest, "ui"
1314
-
1315
-
1316
- #: Explicit so a new ``connector.SCHEMES`` entry is refused at open rather than admitted with no credential.
1317
- SUPPORTED_AUTH_SCHEMES = ("none", "header", "bearer", "basic", "query", "oauth2")
1318
-
1319
-
1320
- def unsupported_auth(spec: Spec) -> str | None:
1321
- if spec.scheme in SUPPORTED_AUTH_SCHEMES:
1322
- return None
1323
- return (
1324
- f"the host cannot present auth.scheme '{spec.scheme}' to a world; it serves {', '.join(SUPPORTED_AUTH_SCHEMES)}"
1325
- )
1326
-
1327
-
1328
- def world_credential(session: Session) -> tuple[dict[str, str], dict[str, str]]:
1329
- """(headers, query) carrying the world's own credential for this session.
1330
- A caller that satisfies the host's session-token gate has already been
1331
- authorized; the host then presents the credential the WORLD itself expects
1332
- (the session token in the vendor's shape, or an oauth2 world's grant), so
1333
- the emulated vendor's own auth check passes."""
1334
- assert session.runtime is not None, "a session on a worker presents its credential there"
1335
- return session.runtime.credential()
1336
-
1337
-
1338
- def bearer_of(headers: dict[str, str]) -> str | None:
1339
- auth = headers.get("authorization", "")
1340
- return auth[7:] if auth.lower().startswith("bearer ") else None
1341
-
1342
-
1343
- def vendor_shaped_token(spec: Spec, headers: dict[str, str], token: str) -> bool:
1344
- """Whether the request carries ``token`` the way the world's own auth scheme
1345
- carries a credential: the declared header for ``header``, the user of a
1346
- Basic authorization for ``basic``. A ``query`` scheme is not honoured here (a
1347
- token in the URL outlives the request in logs); ``bearer`` is the gate's
1348
- own shape and ``oauth2`` has its grant."""
1349
- if spec.scheme == "header":
1350
- presented = headers.get((spec.auth_name or "x-api-key").lower())
1351
- return presented is not None and hmac.compare_digest(presented, token)
1352
- if spec.scheme == "basic":
1353
- user = basic_user(headers.get("authorization", ""))
1354
- return user is not None and hmac.compare_digest(user, token)
1355
- return False
1356
-
1357
-
1358
- def admits(session: Session, path: str, headers: dict[str, str]) -> str | None:
1359
- """How the host's gate admits a data-plane request, or None to refuse.
1360
-
1361
- ``"session"``: the caller presented the session token as a bearer and the
1362
- host will present the world's credential for it. Only the header counts: a
1363
- token in the URL would outlive the request in access logs and referrers.
1364
- ``"world"``: an oauth2 caller doing the vendor's own dance — the grant
1365
- request (the world checks ``client_secret`` against the session token) or
1366
- a bearer this session's world minted — whose headers pass through as sent.
1367
- """
1368
- if session.token is None:
1369
- return "session"
1370
- bearer = bearer_of(headers)
1371
- if bearer is not None and hmac.compare_digest(bearer, session.token):
1372
- return "session"
1373
- if session.spec is not None and vendor_shaped_token(session.spec, headers, session.token):
1374
- # The session token in the vendor's own header shape, never in the URL.
1375
- return "session"
1376
- if session.spec is not None and session.spec.scheme == "oauth2":
1377
- if session.app is None:
1378
- return "world" # the world's tokens live in its worker, which runs this same gate
1379
- token_path = session.app.settings.token_path
1380
- if token_path and session.app._strip(path) == token_path:
1381
- return "world"
1382
- if bearer is not None and session.app.router.authorized(headers, {}, session.token):
1383
- return "world"
1384
- return None
1385
-
1386
-
1387
- class OwnerLookup(Protocol):
1388
- """Which host owns a session this replica does not: its control-plane
1389
- URL, or None for a closed or unknown one. ``fresh`` bypasses whatever the
1390
- lookup remembers, for a caller that has just found the remembered owner
1391
- dead."""
1392
-
1393
- def __call__(self, sid: str, *, fresh: bool = False) -> str | None: ...
1394
-
1395
-
1396
- def platform_owner_lookup(platform_url: str, secret: str | None, self_url: str | None) -> OwnerLookup:
1397
- """Ask the platform which host owns a session the caller reached us for.
1398
- ``GET /api/public/world-sessions/{sid}/host`` is host-secret authenticated
1399
- and answers ``{hostUrl}``; a 404 is a closed or unknown session. Answers are
1400
- held briefly so a busy session does not cost one platform call per request.
1401
- Our own address answers None: a session we do not hold but the platform
1402
- says is ours is gone, not one hop away."""
1403
- cache: dict[str, tuple[float, str | None]] = {}
1404
- lock = threading.Lock()
1405
- base = platform_url.rstrip("/")
1406
-
1407
- def lookup(sid: str, *, fresh: bool = False) -> str | None:
1408
- now = time.time()
1409
- if not fresh:
1410
- with lock:
1411
- hit = cache.get(sid)
1412
- if hit and hit[0] > now:
1413
- return hit[1]
1414
- owner: str | None = None
1415
- req = urllib.request.Request(
1416
- f"{base}/api/public/world-sessions/{urllib.parse.quote(sid, safe='')}/host",
1417
- headers={"X-Host-Secret": secret or ""},
1418
- )
1419
- try:
1420
- with urllib.request.urlopen(req, timeout=5) as r:
1421
- answer = json.loads(r.read() or b"{}")
1422
- url = answer.get("hostUrl") if isinstance(answer, dict) else None
1423
- owner = url if isinstance(url, str) and url else None
1424
- except urllib.error.HTTPError as e:
1425
- if e.code != 404:
1426
- raise
1427
- if owner and self_url and owner.rstrip("/") == self_url.rstrip("/"):
1428
- owner = None
1429
- with lock:
1430
- cache[sid] = (now + (OWNER_CACHE_S if owner else OWNER_MISS_S), owner)
1431
- return owner
1432
-
1433
- return lookup
1434
-
1435
-
1436
- class OwnerUnreachable(Exception):
1437
- """The owner a request was proxied to did not answer: no connection, or
1438
- none in time. Not an answer of any status."""
1439
-
1440
-
1441
113
  class BodyRefused(Exception):
1442
114
  """A request body the host will not read: the answer's status and error."""
1443
115
 
@@ -1456,25 +128,27 @@ def make_host_handler(
1456
128
  max_body_mb: int = DEFAULT_MAX_BODY_MB,
1457
129
  secure_cookies: bool = True,
1458
130
  ):
1459
- """The request handler class for one host. ``secret`` is required: the
1460
- control plane is the only trust boundary, so a host with no secret has no
1461
- control plane at all rather than an open one. ``secure_cookies`` is off
131
+ """The request handler class for one host; its control plane is :class:`ControlPlane`, which
132
+ refuses an empty ``secret`` or ``worlds_domain``. ``secure_cookies`` is off
1462
133
  only for a host whose public surface is reached over plain http (a dev
1463
134
  host); it is the deployment's word, not the request's, since a client
1464
135
  writes the forwarding headers a request arrives with."""
1465
- if not secret:
1466
- raise ValueError("the world host needs a control-plane secret (WORLDS_HOST_SECRET)")
136
+ control = ControlPlane(host, secret, HOSTED_OPERATIONS, worlds_domain)
1467
137
  if max_body_mb <= 0:
1468
138
  raise ValueError(f"the request body cap must be positive (WORLDS_HOST_MAX_BODY_MB={max_body_mb})")
1469
139
  max_body_bytes = max_body_mb * (1 << 20)
1470
140
 
1471
- class Handler(BaseHTTPRequestHandler):
141
+ class Handler(OwnerProxy):
1472
142
  server_version = "gateway-world-host/1"
1473
-
1474
- def log_message(self, format: str, *args: object) -> None:
1475
- pass
143
+ world_host = host
1476
144
 
1477
145
  def _read(self) -> bytes:
146
+ """The request's body, read once: a request sent on after its session moved carries it."""
147
+ if self._body is None:
148
+ self._body = self._read_body()
149
+ return self._body
150
+
151
+ def _read_body(self) -> bytes:
1478
152
  declared = self.headers.get("Content-Length")
1479
153
  if declared is None:
1480
154
  return b""
@@ -1488,45 +162,36 @@ def make_host_handler(
1488
162
  raise BodyRefused(413, f"request body over {max_body_mb} MiB")
1489
163
  return self.rfile.read(length) if length else b""
1490
164
 
1491
- def _reply(self, status: int, body: Any, extra: dict[str, str] | None = None) -> None:
1492
- send_response(self, status, body, extra, "application/json")
165
+ def _reply(self, status: int, body: Any, extra: dict[str, str] | None = None, errors: str = JSON_TYPE) -> None:
166
+ try:
167
+ send_response(self, status, body, extra, errors)
168
+ except ConnectionError:
169
+ self.close_connection = True
1493
170
 
1494
171
  def _headers(self) -> dict[str, str]:
1495
172
  return {k.lower(): v for k, v in self.headers.items()}
1496
173
 
1497
- def do_GET(self):
1498
- self._dispatch("GET")
1499
-
1500
- def do_POST(self):
1501
- self._dispatch("POST")
1502
-
1503
- def do_PUT(self):
1504
- self._dispatch("PUT")
1505
-
1506
- def do_PATCH(self):
1507
- self._dispatch("PATCH")
1508
-
1509
- def do_DELETE(self):
1510
- self._dispatch("DELETE")
1511
-
1512
174
  def _dispatch(self, method: str) -> None:
175
+ self._body: bytes | None = None
1513
176
  url = urllib.parse.urlsplit(self.path)
1514
- # The public worlds domain learns nothing about the host: probes get a bare ok, counters
1515
- # stay on the pod address or behind the control-plane secret.
1516
- if url.path == "/healthz":
177
+ if url.path == "/healthz" and control.internal(self.path, self.headers):
1517
178
  return self._reply(200, {"ok": True})
1518
- if url.path == "/metrics" and not public_host(self.headers.get("Host"), worlds_domain):
179
+ if url.path == "/metrics" and control.serves(self.path, self.headers):
1519
180
  payload = host.metrics_text().encode("utf-8")
1520
181
  return self._relay(
1521
182
  200, [("Content-Type", "text/plain; version=0.0.4; charset=utf-8")], payload, drop=FRAMING_HEADERS
1522
183
  )
1523
184
  try:
1524
- if url.path.startswith("/__host/"):
185
+ if url.path.startswith("/__host/") and control.serves(self.path, self.headers):
1525
186
  return self._control(method, url)
1526
187
  return self._data(method, url)
188
+ except SessionMoved as moved:
189
+ headers = self._headers()
190
+ if hops(headers) >= MAX_HOPS:
191
+ return self._reply(404, CLOSED_SESSION)
192
+ return self._proxy(moved.session_id, moved.owner, owner_lookup, method, headers)
1527
193
  except BodyRefused as refused:
1528
- # The body was not read, so the connection cannot carry another
1529
- # request: answer and hang up.
194
+ # The body was not read, so the connection cannot carry another request: answer and hang up.
1530
195
  self.close_connection = True
1531
196
  return self._reply(refused.status, {"error": refused.error}, {"Connection": "close"})
1532
197
  except Exception as error:
@@ -1534,94 +199,27 @@ def make_host_handler(
1534
199
  return self._reply(500, request_failure_body(error))
1535
200
 
1536
201
  def _control(self, method: str, url) -> None:
1537
- if not hmac.compare_digest(self.headers.get("X-Host-Secret") or "", secret):
202
+ if not control.admits(self.headers.get("X-Host-Secret")):
1538
203
  return self._reply(403, {"error": "forbidden"})
1539
- raw = self._read()
1540
- try:
1541
- body = json.loads(raw) if raw else {}
1542
- except ValueError as error:
1543
- return self._reply(400, {"error": "malformed json", "detail": str(error)})
1544
- if not isinstance(body, dict):
1545
- return self._reply(400, {"error": "malformed json", "detail": "the body must be a json object"})
1546
- parts = url.path.strip("/").split("/")
1547
- try:
1548
- if method == "GET" and parts == ["__host", "stats"]:
1549
- return self._reply(200, host.stats())
1550
- if method == "POST" and parts == ["__host", "drain"]:
1551
- host.drain()
1552
- return self._reply(200, {"draining": True, "sessions": host.session_count()})
1553
- if method == "GET" and parts == ["__host", "versions"]:
1554
- return self._reply(200, {"versions": host.cache.version_ids()})
1555
- if method == "POST" and parts == ["__host", "versions"]:
1556
- return self._reply(
1557
- 200,
1558
- host.warm(body["versionId"], body["bundleUrl"], body.get("snapshot")),
1559
- )
1560
- if method == "POST" and parts == ["__host", "sessions"]:
1561
- return self._reply(
1562
- 200,
1563
- host.open(
1564
- session_id=body["sessionId"],
1565
- version_id=body["versionId"],
1566
- bundle_url=body["bundleUrl"],
1567
- task=body.get("task"),
1568
- token=body.get("token"),
1569
- snapshot=body.get("snapshot"),
1570
- browser=body.get("browser"),
1571
- restore=body.get("restore"),
1572
- task_spec=body.get("taskSpec"),
1573
- dependencies=body.get("dependencies"),
1574
- ),
1575
- )
1576
- if len(parts) >= 3 and parts[:2] == ["__host", "sessions"]:
1577
- sid = parts[2]
1578
- verb = parts[3] if len(parts) > 3 else None
1579
- if method == "PUT" and verb == "browser" and len(parts) == 4:
1580
- return self._reply(200, host.set_browser(sid, body.get("browser")))
1581
- if method == "POST" and verb == "export" and len(parts) == 5:
1582
- url = body.get("url")
1583
- if not isinstance(url, str) or not url.startswith(("https://", "http://")):
1584
- return self._reply(400, {"error": "url must be the presigned upload url"})
1585
- return self._reply(200, host.upload_export(sid, parts[4], url))
1586
- row = session_verb(method, verb)
1587
- if row is None or len(parts) > 4:
1588
- return self._reply(
1589
- 404, unknown_verb(method, verb, hosted=True, prefix=f"/__host/sessions/{sid}/")
1590
- )
1591
- return self._reply(200, carried(HOSTED_OPERATIONS[row.route](host, sid, **row.read(body))))
1592
- return self._reply(404, {"error": "unknown control route"})
1593
- except ContractError as error:
1594
- code = error.detail.get("code", "host")
1595
- # 503 sends the platform to another replica; 404 is the one answer it reads as "lost".
1596
- if code in ("draining", "at_capacity"):
1597
- status = 503
1598
- elif code == UNKNOWN_SESSION:
1599
- status = 404
1600
- else:
1601
- status = 400
1602
- return self._reply(
1603
- status, {"error": code, "detail": str(error), "message": str(error.detail.get("message", ""))}
1604
- )
1605
- except KeyError as error:
1606
- return self._reply(400, {"error": "missing field", "field": str(error)})
1607
- except WorkerGone as error:
1608
- return self._reply(502, {"error": "worker exited", "detail": str(error)})
204
+ return self._reply(*control.answer(method, url.path, self._read()))
1609
205
 
1610
206
  def _data(self, method: str, url) -> None:
1611
207
  headers = self._headers()
1612
208
  named = session_from_host(headers.get("host"))
1613
209
  if named is None:
1614
- return self._reply(404, {"error": "no session named by Host"})
1615
- sid, surface = named
210
+ return self._reply(404, NO_SESSION_NAMED)
211
+ sid, surface = named.session_id, named.surface
1616
212
  if not host.owns(sid):
1617
- # Proxy at most one hop: a request that already hopped is not sent on again.
1618
- hopped = HOP_HEADER in headers
1619
- owner = owner_lookup(sid) if owner_lookup and not hopped else None
1620
- if owner and owner_lookup:
213
+ looked_up = owner_lookup if owner_lookup and not hops(headers) else None
214
+ owner = host.forwarding.owner(sid) or (looked_up(sid) if looked_up else None)
215
+ if owner and hops(headers) < MAX_HOPS:
1621
216
  host.tally("requests.proxied")
1622
217
  return self._proxy(sid, owner, owner_lookup, method, headers)
1623
- return self._reply(404, {"error": "unknown or closed session"})
218
+ return self._reply(404, CLOSED_SESSION)
219
+ if not host.reached_by(named):
220
+ return self._reply(404, NO_SESSION_NAMED)
1624
221
  host.tally(f"requests.{surface}")
222
+ opened = named.capability is not None
1625
223
  try:
1626
224
  with host.using(sid) as session:
1627
225
  if session.spec is None:
@@ -1631,84 +229,119 @@ def make_host_handler(
1631
229
  if surface == "browser":
1632
230
  self._serve_browser(session, url, headers)
1633
231
  elif surface == "ui":
1634
- self._serve_ui(session, method, url, headers)
232
+ self._serve_ui(session, method, url, headers, opened)
1635
233
  else:
1636
- self._serve(session, method, url, headers)
1637
- except ContractError:
1638
- # Closed between the ownership check and the lookup.
1639
- self._reply(404, {"error": "unknown or closed session"})
1640
-
1641
- def _serve(self, session: Session, method: str, url, headers: dict[str, str]) -> None:
1642
- admitted = admits(session, url.path, headers)
1643
- if admitted is None:
1644
- return self._reply(401, {"error": "invalid session token"})
234
+ self._serve(session, method, url, headers, opened)
235
+ except ContractError as error:
236
+ owner = host.forwarding.owner(sid) if error.detail.get("code") == UNKNOWN_SESSION else None
237
+ if owner is not None:
238
+ raise SessionMoved(sid, owner) from error
239
+ self._refused(error)
240
+
241
+ def _refused(self, error: ContractError) -> None:
242
+ if error.detail.get("code") == UNKNOWN_SESSION:
243
+ return self._reply(404, CLOSED_SESSION)
244
+ self._reply(*refusal(error))
245
+
246
+ def _serve(self, session: Session, method: str, url, headers: dict[str, str], opened: bool) -> None:
247
+ preflight = session.preflight(method, headers)
248
+ if preflight is not None:
249
+ return self._reply(*preflight)
250
+ assert session.door is not None, "a session with an http surface has a door"
251
+ door = session.door.opened() if opened else session.door
252
+ entry = door.session_entry(method, url.path, url.query, headers)
253
+ if entry is None:
254
+ status, body, errors = door.session_refusal
255
+ return self._reply(status, body, None, errors)
256
+ if door.by_platform_bearer(entry, headers):
257
+ host.tally("requests.platform_bearer")
258
+ headers = door.with_platform_header(headers)
1645
259
  if session.worker is not None:
1646
260
  return self._on_worker(session, method, headers)
1647
- self._serve_api(session, method, self.path, headers, admitted, Via.API)
261
+ self._serve_api(session, method, self.path, headers, entry, Via.API, opened)
1648
262
 
1649
263
  def _on_worker(self, session: Session, method: str, headers: dict[str, str]) -> None:
1650
264
  """Relay the request as it came, Host and path unchanged, to the
1651
- session's version worker: it runs the same gate and picks the same
1652
- door (api or ui) on its own copy of the session."""
1653
- assert session.worker is not None
1654
- try:
1655
- status, answer_headers, payload = session.worker.request(method, self.path, headers, self._read())
1656
- except WorkerGone as error:
1657
- host.lose_worker(session.worker)
1658
- return self._reply(502, {"error": "worker exited", "detail": str(error)})
265
+ session's version worker, once it is admitted on the version: the
266
+ worker runs the same gate and picks the same door (api or ui) on its
267
+ own copy of the session."""
268
+ assert host.placement is not None
269
+ raw = self._read()
270
+ with host.admitted(session, HOP_HEADER in headers):
271
+ status, answer_headers, payload = host.placement.relay(session, method, self.path, headers, raw)
1659
272
  return self._relay(status, answer_headers, payload, drop=FRAMING_HEADERS)
1660
273
 
1661
274
  def _serve_api(
1662
- self, session: Session, method: str, target: str, headers: dict[str, str], admitted: str, via: Via
275
+ self,
276
+ session: Session,
277
+ method: str,
278
+ target: str,
279
+ headers: dict[str, str],
280
+ entry: Entry,
281
+ via: Via,
282
+ opened: bool = False,
1663
283
  ) -> None:
1664
284
  """Answer ``target`` (a path plus query) from the session's world in
1665
285
  this process, once the gate has admitted it; the call is logged as
1666
- having come in by ``via``, the door this host took it in by."""
1667
- assert session.app is not None
286
+ having come in by ``via``, the door this host took it in by. A caller
287
+ the platform admitted is presented to the world with the session's
288
+ credential the vendor's way, and the platform's own header never
289
+ reaches it, so the world's own check and its handlers see what the
290
+ vendor's client would send. A caller on an open session's capability
291
+ host is presented nothing: world code never hands the session token to
292
+ someone who holds only the capability. A link the world signs points at
293
+ the session's own api host, where its signature alone admits it."""
294
+ assert session.app is not None and session.door is not None
1668
295
  raw = self._read()
1669
- world_headers = headers
1670
- if admitted == "session":
1671
- # The host's token gate is the access control; present the world
1672
- # its OWN expected credential so the emulated vendor auth passes.
1673
- credential_headers, credential_query = world_credential(session)
1674
- world_headers = {**headers, **credential_headers}
296
+ world_headers = {name: value for name, value in headers.items() if name != PLATFORM_HEADER}
297
+ if entry is Entry.PLATFORM and not opened:
298
+ credential_headers, credential_query = session.door.present()
299
+ world_headers = {**world_headers, **credential_headers}
1675
300
  target = with_query(target, credential_query)
1676
- default_host = f"w-{session.session_id}-api.{worlds_domain}"
1677
- self._reply(*session.app.handle(method, target, world_headers, raw, default_host, via=via))
301
+ default_host = session.api_host(worlds_domain, opened)
302
+ link_host = session.api_host(worlds_domain, opened=False)
303
+ with host.admitted(session, HOP_HEADER in headers):
304
+ answer = session.app.handle(
305
+ method, target, world_headers, raw, default_host, via=via, opened=opened, link_host=link_host
306
+ )
307
+ self._reply(*answer, session.app.settings.error_content_type)
1678
308
 
1679
- def _serve_ui(self, session: Session, method: str, url, headers: dict[str, str]) -> None:
309
+ def _serve_ui(self, session: Session, method: str, url, headers: dict[str, str], opened: bool) -> None:
1680
310
  ui = session.ui
1681
311
  if ui is None:
1682
312
  return self._reply(404, {"error": "this world has no static ui"})
1683
- token = session.token
313
+ token = None if opened else session.token
1684
314
  if token is not None and matches(query_token(url.query), token):
1685
315
  return self._trade_token_for_cookie(url, token)
1686
- if token is not None and not (matches(cookie_token(headers), token) or matches(bearer_of(headers), token)):
316
+ if token is not None and not (
317
+ matches(cookie_token(headers), token) or matches(_BEARER.read(headers, {}), token)
318
+ ):
1687
319
  return self._reply(401, {"error": "invalid session token"})
1688
320
  route = api_target(ui, url.path)
1689
321
  if route is None:
1690
- return self._serve_static(ui, method, url)
322
+ return self._serve_static(ui, method, url, opened)
1691
323
  if cross_site(headers, lambda origin: urllib.parse.urlsplit(origin).netloc == headers.get("host")):
1692
324
  return self._reply(403, {"error": "the ui's api prefix answers its own page, not another site"})
1693
325
  if session.worker is not None:
1694
326
  return self._on_worker(session, method, headers)
1695
327
  target = urllib.parse.urlunsplit(("", "", route, url.query, ""))
1696
328
  api_headers = {k: v for k, v in headers.items() if k not in ("cookie", "authorization")}
1697
- api_headers["host"] = f"w-{session.session_id}-api.{worlds_domain}"
1698
- if token is not None:
1699
- api_headers["authorization"] = f"Bearer {token}"
1700
- return self._serve_api(session, method, target, api_headers, "session", Via.UI)
329
+ api_headers["host"] = session.api_host(worlds_domain, opened)
330
+ return self._serve_api(session, method, target, api_headers, Entry.PLATFORM, Via.UI, opened)
1701
331
 
1702
332
  def _trade_token_for_cookie(self, url, token: str) -> None:
1703
333
  landing = without_query_token(url)
1704
334
  cookie = session_cookie(token, secure=secure_cookies, max_age_s=host.lease_s)
1705
335
  self._reply(303, {"location": landing}, {"Location": landing, "Set-Cookie": cookie})
1706
336
 
1707
- def _serve_static(self, ui: UiSurface, method: str, url) -> None:
337
+ def _serve_static(self, ui: UiSurface, method: str, url, opened: bool) -> None:
338
+ """A file of the ui; on a capability host with no ``Referer``, so a page that
339
+ loads another site's resource never hands it the capability."""
1708
340
  if method != "GET":
1709
341
  return self._reply(405, {"error": "the static ui answers GET only"})
1710
342
  status, static_headers, payload = static_file(ui, url.path)
1711
- self._relay(status, static_headers.items(), payload, drop=FRAMING_HEADERS)
343
+ sealed = {**static_headers, **(NO_REFERRER if opened else {})}
344
+ self._relay(status, sealed.items(), payload, drop=FRAMING_HEADERS)
1712
345
 
1713
346
  def _serve_browser(self, session: Session, url, headers: dict[str, str]) -> None:
1714
347
  """``/cdp/<session token>`` and ``/live/<grant>`` on the session's
@@ -1726,8 +359,7 @@ def make_host_handler(
1726
359
  if kind == "cdp":
1727
360
  if not matches(credential, token):
1728
361
  return self._reply(401, {"error": "invalid session token"})
1729
- # The relay decodes the path segment, so a token with '/' or
1730
- # '=' (openssl rand -base64) must travel percent-encoded.
362
+ # The relay decodes the path segment: a token with '/' or '=' must travel percent-encoded.
1731
363
  upstream_path = f"/cdp/{urllib.parse.quote(relay.token, safe='')}"
1732
364
  else:
1733
365
  scope = verify_live_grant(credential, token)
@@ -1742,146 +374,17 @@ def make_host_handler(
1742
374
  except OSError as error:
1743
375
  self._reply(502, {"error": "browser unreachable", "detail": str(error)})
1744
376
 
1745
- def _proxy(self, sid: str, owner: str, lookup: OwnerLookup, method: str, headers: dict[str, str]) -> None:
1746
- """Send the request on to ``owner``. An owner that does not answer
1747
- at all may be a host that died with the session since recovered
1748
- elsewhere, so the platform is asked once more, uncached, and the
1749
- request goes to the owner it names now; only then is it a 502.
1750
- The retry stays one hop (the hop header goes with it)."""
1751
- websocket = headers.get("upgrade", "").lower() == "websocket"
1752
- raw = b"" if websocket else self._read()
1753
- try:
1754
- return self._forward(owner, method, headers, raw, websocket)
1755
- except OwnerUnreachable as stale:
1756
- current = lookup(sid, fresh=True)
1757
- if not current or current.rstrip("/") == owner.rstrip("/"):
1758
- return self._reply(502, {"error": "owner unreachable", "detail": str(stale)})
1759
- host.tally("requests.rerouted")
1760
- try:
1761
- self._forward(current, method, headers, raw, websocket)
1762
- except OwnerUnreachable as error:
1763
- self._reply(502, {"error": "owner unreachable", "detail": str(error)})
1764
-
1765
- def _forward(self, owner: str, method: str, headers: dict[str, str], raw: bytes, websocket: bool) -> None:
1766
- """One attempt at ``owner``; raises :class:`OwnerUnreachable` when it
1767
- did not answer, before anything has been written to the client."""
1768
- if websocket:
1769
- # A browser session on another host: the owner runs the
1770
- # credential check; this hop only carries bytes. ``tunnel``
1771
- # fails before the 101 or not at all.
1772
- owner_ws = "ws://" + owner.split("://", 1)[-1].rstrip("/")
1773
- self.close_connection = True
1774
- try:
1775
- tunnel(self.connection, owner_ws, self.path, {**headers, HOP_HEADER: "1"}, host=headers.get("host"))
1776
- except OSError as error:
1777
- raise OwnerUnreachable(str(error)) from error
1778
- return
1779
- target = f"{owner.rstrip('/')}{self.path}"
1780
- forward = {k: v for k, v in headers.items() if k not in ("host", "content-length")}
1781
- forward["host"] = headers.get("host", "")
1782
- forward[HOP_HEADER] = "1"
1783
- req = urllib.request.Request(target, data=raw or None, method=method, headers=forward)
1784
- try:
1785
- with _PROXY_OPENER.open(req, timeout=60) as r:
1786
- self._relay(r.status, r.headers.items(), r.read(), drop=PROXIED_DROP_HEADERS)
1787
- except urllib.error.HTTPError as e:
1788
- self._relay(e.code, e.headers.items(), e.read(), drop=PROXIED_DROP_HEADERS)
1789
- except (OSError, http.client.HTTPException) as error:
1790
- # URLError is an OSError: no connection, or none in time.
1791
- raise OwnerUnreachable(str(error)) from error
1792
-
1793
- def _relay(
1794
- self,
1795
- status: int,
1796
- headers: Iterable[tuple[str, str]],
1797
- payload: bytes,
1798
- *,
1799
- drop: frozenset[str],
1800
- ) -> None:
1801
- """Write another process's answer through as it came: same status,
1802
- same body bytes, every header the world set except ``drop`` (the
1803
- framing, which is this connection's, and from an owner the
1804
- hop-by-hop headers too). The world's contract, its ``page_headers``
1805
- included, reads the same from every replica."""
1806
- self.send_response(status)
1807
- content_type = None
1808
- for name, value in headers:
1809
- lower = name.lower()
1810
- if lower == "content-type":
1811
- content_type = value
1812
- elif lower not in drop:
1813
- self.send_header(name, value)
1814
- self.send_header("Content-Type", content_type or "application/json")
1815
- self.send_header("Content-Length", str(len(payload)))
1816
- self.end_headers()
1817
- self.wfile.write(payload)
1818
-
1819
377
  return Handler
1820
378
 
1821
379
 
1822
- def start_lease_sweeper(world_host: WorldHost, interval_s: float = LEASE_SWEEP_S) -> threading.Event:
1823
- """Expire leases every ``interval_s`` until the returned event is set."""
1824
- stop = threading.Event()
1825
-
1826
- def sweep() -> None:
1827
- while not stop.wait(interval_s):
1828
- for sid in world_host.expire_leases():
1829
- print(
1830
- f"world host: lease expired, closed session {sid}",
1831
- file=sys.stderr,
1832
- flush=True,
1833
- )
1834
- for version_id in world_host.reap_workers():
1835
- print(
1836
- f"world host: stopped idle worker for version {version_id}",
1837
- file=sys.stderr,
1838
- flush=True,
1839
- )
1840
-
1841
- threading.Thread(target=sweep, name="world-host-leases", daemon=True).start()
1842
- return stop
1843
-
1844
-
1845
- def heartbeat_once(world_host: WorldHost, platform_url: str, secret: str, timeout_s: float = 5.0) -> bool:
1846
- """Tell the platform what this host is and holds (:meth:`WorldHost.stats`).
1847
- The platform's registry is what places new sessions: a host it has not
1848
- heard from lately gets none. Returns whether the platform took it."""
1849
- body = json.dumps(world_host.stats()).encode("utf-8")
1850
- req = urllib.request.Request(
1851
- f"{platform_url.rstrip('/')}/api/public/world-hosts",
1852
- data=body,
1853
- method="POST",
1854
- headers={"content-type": "application/json", "x-host-secret": secret},
1855
- )
1856
- try:
1857
- with urllib.request.urlopen(req, timeout=timeout_s) as r:
1858
- return 200 <= r.status < 300
1859
- except (urllib.error.URLError, OSError, ValueError) as error:
1860
- print(f"world host: heartbeat failed: {error}", file=sys.stderr, flush=True)
1861
- return False
1862
-
1863
-
1864
- def start_heartbeat(
1865
- world_host: WorldHost, platform_url: str, secret: str, interval_s: float = HEARTBEAT_S
1866
- ) -> threading.Event:
1867
- """Heartbeat every ``interval_s`` until the returned event is set; the
1868
- first beat goes out at once so a fresh replica is placeable immediately,
1869
- and a last one goes out on stop so the registry sees it draining."""
1870
- stop = threading.Event()
1871
-
1872
- def beat() -> None:
1873
- heartbeat_once(world_host, platform_url, secret)
1874
- while not stop.wait(interval_s):
1875
- heartbeat_once(world_host, platform_url, secret)
1876
- heartbeat_once(world_host, platform_url, secret)
1877
-
1878
- threading.Thread(target=beat, name="world-host-heartbeat", daemon=True).start()
1879
- return stop
1880
-
1881
-
1882
380
  class HostServer(ThreadingHTTPServer):
1883
381
  """The listening socket plus what shutdown has to stop: the lease sweeper,
1884
- the heartbeat and the version workers."""
382
+ the heartbeat and the version workers. Its backlog holds a burst of
383
+ connections; their requests wait their turn in :class:`Admission`. It
384
+ counts the requests it is answering, so a drain stops only once the
385
+ requests sent on to the hosts its sessions moved to are answered."""
386
+
387
+ request_queue_size = 1024
1885
388
 
1886
389
  def __init__(
1887
390
  self,
@@ -1892,79 +395,104 @@ class HostServer(ThreadingHTTPServer):
1892
395
  heartbeat: tuple[str, str] | None = None,
1893
396
  heartbeat_s: float = HEARTBEAT_S,
1894
397
  drain_s: float = DEFAULT_DRAIN_S,
398
+ ledger: LedgerFlusher | None = None,
399
+ handoff: HandoffDoor | None = None,
1895
400
  ):
1896
401
  super().__init__(address, handler)
1897
402
  self.world_host = world_host
1898
403
  self.lease_sweeper = start_lease_sweeper(world_host)
1899
- self.heartbeat = start_heartbeat(world_host, *heartbeat, interval_s=heartbeat_s) if heartbeat else None
1900
- self.drain_s = drain_s
404
+ self.heartbeat = Heartbeat(world_host, *heartbeat, interval_s=heartbeat_s).start() if heartbeat else None
405
+ world_host.ledger = ledger.start() if ledger else NoLedger()
406
+ self.handoff = DrainHandoff(world_host, handoff, drain_s)
407
+ self._answering = 0
408
+ self._draining = False
409
+ self._lock = threading.Lock()
410
+
411
+ def process_request_thread(self, request, client_address) -> None:
412
+ with self._lock:
413
+ self._answering += 1
414
+ try:
415
+ super().process_request_thread(request, client_address)
416
+ finally:
417
+ with self._lock:
418
+ self._answering -= 1
419
+
420
+ def idle(self) -> bool:
421
+ with self._lock:
422
+ return self._answering == 0
1901
423
 
1902
424
  def drain(self) -> None:
1903
- """SIGTERM: refuse new sessions, announce it, serve what is here until
1904
- it closes or the grace runs out, then stop. Runs off the signal thread
1905
- so ``serve_forever`` keeps answering meanwhile."""
425
+ """SIGTERM: refuse new sessions, tell the platform at once and keep beating
426
+ (a draining host is alive), hand every session off (:class:`DrainHandoff`),
427
+ then stop. Runs off the signal thread so ``serve_forever`` keeps answering
428
+ meanwhile; a second call does nothing."""
429
+ with self._lock:
430
+ if self._draining:
431
+ return
432
+ self._draining = True
1906
433
  self.world_host.drain()
1907
434
  if self.heartbeat is not None:
1908
- self.heartbeat.set() # its last beat carries ``draining: true``
435
+ self.heartbeat.wake()
436
+ threading.Thread(target=self._hand_off_then_stop, name="world-host-drain", daemon=True).start()
1909
437
 
1910
- def wait_then_stop() -> None:
1911
- deadline = time.time() + self.drain_s
1912
- while self.world_host.session_count() > 0 and time.time() < deadline:
1913
- time.sleep(0.5)
438
+ def _hand_off_then_stop(self) -> None:
439
+ try:
440
+ self.handoff.run(self.idle)
441
+ finally:
1914
442
  self.shutdown()
1915
443
 
1916
- threading.Thread(target=wait_then_stop, name="world-host-drain", daemon=True).start()
1917
-
1918
444
 
1919
445
  def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0") -> HostServer:
1920
446
  """The host as deployed, configured from the environment:
1921
447
 
1922
- ``WORLDS_HOST_SECRET`` gates the control plane; ``WORLDS_INGRESS_DOMAIN`` is
1923
- the domain the session hostnames live under; ``WORLDS_HOST_ADVERTISE_URL`` is
1924
- this replica's own address (``http://<pod ip>:<port>``), returned on open so
1925
- the platform pins the session to it; ``WORLDS_PLATFORM_URL`` lets a
1926
- non-owner replica ask the platform who owns a session and proxy one hop;
1927
- ``WORLDS_HOST_LEASE_S`` is how long an untouched session lives;
1928
- ``WORLDS_HOST_MAX_SESSIONS`` is how many it takes before answering 503;
1929
- ``WORLDS_HOST_HEARTBEAT_S`` is how often it reports to the platform (with
1930
- ``WORLDS_PLATFORM_URL`` and an advertised address, else no heartbeat);
1931
- ``WORLDS_HOST_DRAIN_S`` is how long a SIGTERM waits for sessions to close;
1932
- ``WORLDS_WORKER_MEMORY_MB`` caps each version worker's address space,
1933
- ``WORLDS_WORKER_BUDGET_MB`` is the memory all of them together are held to
1934
- and ``WORLDS_WORKER_FOOTPRINT_MB`` what each is counted at against it
1935
- (unset, its whole address-space cap; their count is the quotient, and one
1936
- more version answers 503 ``at_capacity``),
1937
- ``WORLDS_HOST_OBJECT_STORE`` is where every presigned URL it is handed
1938
- may point (required; ``any`` for a dev host),
1939
- ``WORLDS_HOST_MAX_BUNDLE_MB`` is the largest version bundle the cache
1940
- accepts, ``WORLDS_HOST_MAX_BODY_MB`` the largest request body it reads,
1941
- ``WORLDS_HOST_PLAIN_HTTP=1`` says the public surface is reached over plain
1942
- http (a dev host: the ui cookie is not marked ``Secure``) and
1943
- ``WORLDS_HOST_WORKERS=0`` runs without workers (handler worlds refused)."""
448
+ ``WORLDS_HOST_SECRET`` gates the control plane; ``WORLDS_INGRESS_DOMAIN`` is the domain the session hostnames
449
+ live under (required); ``WORLDS_HOST_ADVERTISE_URL`` is this replica's own address (``http://<pod ip>:<port>``),
450
+ returned on open so the platform pins the session to it; ``WORLDS_PLATFORM_URL`` lets a non-owner replica ask
451
+ the platform who owns a session and proxy one hop, and is where the request ledger goes; ``WORLDS_HOST_LEASE_S``
452
+ is how long an untouched session lives; ``WORLDS_HOST_MAX_SESSIONS`` is how many it takes before answering 503;
453
+ ``WORLDS_HOST_HEARTBEAT_S`` is how often it reports to the platform (with ``WORLDS_PLATFORM_URL`` and an
454
+ advertised address, else no heartbeat), and ``WORLDS_HOST_BUILD`` which pod template it runs, so a session
455
+ handed off goes to the newest one; ``WORLDS_HOST_DRAIN_S`` is how long a SIGTERM's drain may take to hand its
456
+ sessions to other hosts through the platform (with an advertised address) and serve what could not move;
457
+ ``WORLDS_WORKER_MEMORY_MB`` caps each version worker's address space, ``WORLDS_WORKER_BUDGET_MB`` is the memory
458
+ all of them together are held to and ``WORLDS_WORKER_FOOTPRINT_MB`` what each is counted at against it (unset,
459
+ its whole address-space cap; their count is the quotient, and one more version answers 503 ``at_capacity``),
460
+ ``WORLDS_HOST_OBJECT_STORE`` is where every presigned URL it is handed may point (required; ``any`` for a dev
461
+ host), ``WORLDS_HOST_MAX_BUNDLE_MB`` is the largest version bundle the cache accepts,
462
+ ``WORLDS_HOST_MAX_BODY_MB`` the largest request body it reads, ``WORLDS_HOST_LEDGER_SPILL_DIR`` (under the cache
463
+ root) and ``WORLDS_HOST_LEDGER_SPILL_MB`` are where and how much of the request ledger waits on disk for the
464
+ platform, ``WORLDS_HOST_PLAIN_HTTP=1`` says the public surface is reached over plain http (a dev host: the ui
465
+ cookie is not marked ``Secure``) and ``WORLDS_HOST_WORKERS=0`` runs without workers (handler worlds refused)."""
466
+ if not seal_process() and sys.platform.startswith("linux"):
467
+ raise SystemExit("world host: cannot make this process non-dumpable; its workers could read its secret")
1944
468
  secret = os.environ.get("WORLDS_HOST_SECRET") or ""
1945
469
  if not secret:
1946
470
  raise SystemExit("world host: WORLDS_HOST_SECRET is unset; refusing to start without a control-plane secret")
471
+ spill_dir, spill_mb = os.environ.get("WORLDS_HOST_LEDGER_SPILL_DIR"), os.environ.get("WORLDS_HOST_LEDGER_SPILL_MB")
472
+ platform_url = os.environ.get("WORLDS_PLATFORM_URL") or None
1947
473
  try:
1948
474
  object_store = parse_origins(os.environ.get("WORLDS_HOST_OBJECT_STORE"))
1949
- except ContractError as error:
475
+ worlds_domain = ControlPlane.domain_of(os.environ.get("WORLDS_INGRESS_DOMAIN"))
476
+ spill = LedgerSpill.at(Path(cache_root), spill_dir, spill_mb, secret)
477
+ spill = spill.open() if platform_url else spill
478
+ except (ContractError, ValueError, OSError) as error:
1950
479
  raise SystemExit(f"world host: {error}; refusing to start") from error
480
+ # Nothing this process starts inherits them; /proc/<pid>/environ keeps the exec-time copy, closed by the seal.
481
+ os.environ.pop("WORLDS_HOST_SECRET", None)
482
+ os.environ.pop("WORLDS_HOST_OBJECT_STORE", None)
1951
483
  advertise = os.environ.get("WORLDS_HOST_ADVERTISE_URL") or None
1952
- platform_url = os.environ.get("WORLDS_PLATFORM_URL") or None
1953
484
  lease_s = float(os.environ.get("WORLDS_HOST_LEASE_S") or DEFAULT_LEASE_S)
1954
- worlds_domain = os.environ.get("WORLDS_INGRESS_DOMAIN", "worlds.local")
1955
485
  workers = None
486
+ footprint = os.environ.get("WORLDS_WORKER_FOOTPRINT_MB")
1956
487
  if os.environ.get("WORLDS_HOST_WORKERS", "1") != "0":
1957
488
  workers = WorkerPool(
1958
489
  Path(cache_root),
1959
490
  worlds_domain=worlds_domain,
1960
491
  memory_mb=int(os.environ.get("WORLDS_WORKER_MEMORY_MB") or DEFAULT_WORKER_MEMORY_MB),
1961
492
  budget_mb=int(os.environ.get("WORLDS_WORKER_BUDGET_MB") or DEFAULT_WORKER_BUDGET_MB),
1962
- footprint_mb=(
1963
- int(os.environ["WORLDS_WORKER_FOOTPRINT_MB"])
1964
- if os.environ.get("WORLDS_WORKER_FOOTPRINT_MB")
1965
- else DEFAULT_WORKER_FOOTPRINT_MB
1966
- ),
493
+ footprint_mb=int(footprint) if footprint else DEFAULT_WORKER_FOOTPRINT_MB,
1967
494
  )
495
+ owners = PlatformOwners(platform_url, secret, advertise) if platform_url else None
1968
496
  world_host = WorldHost(
1969
497
  Path(cache_root),
1970
498
  lease_s=lease_s,
@@ -1973,22 +501,27 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
1973
501
  max_sessions=int(os.environ.get("WORLDS_HOST_MAX_SESSIONS") or DEFAULT_MAX_SESSIONS),
1974
502
  max_bundle_mb=int(os.environ.get("WORLDS_HOST_MAX_BUNDLE_MB") or DEFAULT_MAX_BUNDLE_MB),
1975
503
  object_store=object_store,
504
+ build=os.environ.get("WORLDS_HOST_BUILD") or None,
505
+ owners=owners,
1976
506
  )
1977
507
  handler = make_host_handler(
1978
508
  world_host,
1979
509
  secret=secret,
1980
510
  worlds_domain=worlds_domain,
1981
- owner_lookup=platform_owner_lookup(platform_url, secret, advertise) if platform_url else None,
511
+ owner_lookup=owners,
1982
512
  max_body_mb=int(os.environ.get("WORLDS_HOST_MAX_BODY_MB") or DEFAULT_MAX_BODY_MB),
1983
513
  secure_cookies=os.environ.get("WORLDS_HOST_PLAIN_HTTP", "0") != "1",
1984
514
  )
515
+ sink = PlatformLedger(platform_url, secret, host_url=advertise) if platform_url else None
1985
516
  return HostServer(
1986
517
  (host_addr, port),
1987
518
  handler,
1988
519
  world_host,
1989
520
  heartbeat=(platform_url, secret) if platform_url and advertise else None,
521
+ ledger=LedgerFlusher(world_host, sink, spill) if sink else None,
1990
522
  heartbeat_s=float(os.environ.get("WORLDS_HOST_HEARTBEAT_S") or HEARTBEAT_S),
1991
523
  drain_s=float(os.environ.get("WORLDS_HOST_DRAIN_S") or DEFAULT_DRAIN_S),
524
+ handoff=PlatformHandoff(platform_url, secret) if platform_url and advertise else None,
1992
525
  )
1993
526
 
1994
527
 
@@ -2001,7 +534,7 @@ def main(argv: list[str] | None = None) -> int:
2001
534
  args = parser.parse_args(argv)
2002
535
  server = serve(args.cache_root, port=args.port)
2003
536
  print(
2004
- f"world host listening on :{args.port}, cache {args.cache_root}",
537
+ f"world host listening on :{server.server_address[1]}, cache {args.cache_root}",
2005
538
  file=sys.stderr,
2006
539
  flush=True,
2007
540
  )
@@ -2009,9 +542,10 @@ def main(argv: list[str] | None = None) -> int:
2009
542
  try:
2010
543
  server.serve_forever()
2011
544
  finally:
545
+ server.world_host.ledger.stop(settle=server.world_host.finishing.settle)
2012
546
  server.lease_sweeper.set()
2013
547
  if server.heartbeat is not None:
2014
- server.heartbeat.set()
548
+ server.heartbeat.stop()
2015
549
  if server.world_host.workers is not None:
2016
550
  server.world_host.workers.stop_all()
2017
551
  server.server_close()