@withgateway/sdk 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. package/README.md +8 -6
  2. package/dist/cli.js +4372 -2265
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +4362 -2256
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +259 -58
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +249 -50
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/hooks/index.d.mts +1 -1
  13. package/dist/hooks/index.d.ts +1 -1
  14. package/dist/hooks/index.js +124 -11
  15. package/dist/hooks/index.js.map +1 -1
  16. package/dist/hooks/index.mjs +124 -11
  17. package/dist/hooks/index.mjs.map +1 -1
  18. package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
  19. package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
  20. package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
  21. package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
  22. package/dist/index.d.mts +5 -5
  23. package/dist/index.d.ts +5 -5
  24. package/dist/index.js +2240 -789
  25. package/dist/index.js.map +1 -1
  26. package/dist/index.mjs +2150 -739
  27. package/dist/index.mjs.map +1 -1
  28. package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
  29. package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
  30. package/dist/traces/index.d.mts +1 -1
  31. package/dist/traces/index.d.ts +1 -1
  32. package/dist/traces/index.js +253 -52
  33. package/dist/traces/index.js.map +1 -1
  34. package/dist/traces/index.mjs +243 -44
  35. package/dist/traces/index.mjs.map +1 -1
  36. package/dist/tracing/index.d.mts +2 -2
  37. package/dist/tracing/index.d.ts +2 -2
  38. package/dist/tracing/index.js +75 -13
  39. package/dist/tracing/index.js.map +1 -1
  40. package/dist/tracing/index.mjs +75 -13
  41. package/dist/tracing/index.mjs.map +1 -1
  42. package/dist/worlds/index.d.mts +893 -482
  43. package/dist/worlds/index.d.ts +893 -482
  44. package/dist/worlds/index.js +1968 -669
  45. package/dist/worlds/index.js.map +1 -1
  46. package/dist/worlds/index.mjs +1960 -662
  47. package/dist/worlds/index.mjs.map +1 -1
  48. package/package.json +1 -1
  49. package/runtime/gatewaysdk/environment/_bundle.py +2 -27
  50. package/runtime/gatewaysdk/environment/_materialize.py +48 -88
  51. package/runtime/gatewaysdk/environment/_world.py +155 -237
  52. package/runtime/gatewaysdk/environment/backing.py +133 -0
  53. package/runtime/gatewaysdk/environment/core.py +2 -17
  54. package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
  55. package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
  56. package/runtime/gatewaysdk/environment/runtime.py +8 -13
  57. package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
  58. package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
  59. package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
  60. package/runtime/gatewaysdk/environment/schema/api.py +347 -892
  61. package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
  62. package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
  63. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
  64. package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
  65. package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
  66. package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
  67. package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
  68. package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
  69. package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
  70. package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
  71. package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
  72. package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
  73. package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
  74. package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
  75. package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
  76. package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
  77. package/runtime/gatewaysdk/environment/schema/data.py +60 -216
  78. package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
  79. package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
  80. package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
  81. package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
  82. package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
  83. package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
  84. package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
  85. package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
  86. package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
  87. package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
  88. package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
  89. package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
  90. package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
  91. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
  92. package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
  93. package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
  94. package/runtime/gatewaysdk/environment/schema/door.py +899 -0
  95. package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
  96. package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
  97. package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
  98. package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
  99. package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
  100. package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
  101. package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
  102. package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
  103. package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
  104. package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
  105. package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
  106. package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
  107. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
  108. package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
  109. package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
  110. package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
  111. package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
  112. package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
  113. package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
  114. package/runtime/gatewaysdk/environment/schema/local.py +58 -170
  115. package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
  116. package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
  117. package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
  118. package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
  119. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
  120. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
  121. package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
  122. package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
  123. package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
  124. package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
  125. package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
  126. package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
  127. package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
  128. package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
  129. package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
  130. package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
  131. package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
  132. package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
  133. package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
  134. package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
  135. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
  136. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
  137. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
  138. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
  139. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
  140. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
  141. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
  142. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
  143. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
  144. package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
  145. package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
  146. package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
  147. package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
  148. package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
  149. package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
  150. package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
  151. package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
  152. package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
  153. package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
  154. package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
  155. package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
  156. package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
  157. package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
  158. package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
  159. package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
  160. package/runtime/gatewaysdk/environment/score.py +57 -0
  161. package/runtime/gatewaysdk/environment/secret_values.py +36 -0
  162. package/runtime/gatewaysdk/environment/world.py +43 -4
  163. package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
  164. package/skills/agents/connector-template-author.md +5 -1
  165. package/skills/benchmarks-getting-started/SKILL.md +10 -11
  166. package/skills/connector-schema-authoring/SKILL.md +49 -16
  167. package/skills/connector-schema-authoring/connector-worlds.md +142 -11
  168. package/skills/connector-schema-authoring/world-schema.md +103 -51
  169. package/skills/world-data-ingestion/ROWS.md +39 -9
  170. package/skills/world-data-ingestion/SKILL.md +27 -27
  171. package/skills/worlds-getting-started/SKILL.md +33 -33
  172. package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
  173. package/skills/worlds-getting-started/references/gotchas.md +49 -1
  174. package/skills/worlds-getting-started/references/journal-checks.md +182 -0
  175. package/skills/worlds-getting-started/references/openapi-import.md +50 -21
  176. package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
@@ -9,19 +9,19 @@ nothing per session in Kubernetes or the load balancer.
9
9
 
10
10
  Two planes on one HTTP server:
11
11
 
12
- * the **control plane** (``/__host/*``, behind ``X-Host-Secret``) is the
13
- platform's half: open, close, reset, seed, state, grade, call. It is
14
- synchronous — a ``seed`` or ``grade`` completes in the response, not over a
12
+ * the **control plane** (``/__host/*`` on the pod's own address, behind ``X-Host-Secret``;
13
+ :mod:`host_control`) is the platform's half: open, close, reset, seed, state, grade,
14
+ call. It is synchronous — a ``seed`` or ``grade`` completes in the response, not over a
15
15
  poll — because the host holds the world in memory.
16
- * the **data plane** (everything else) is the end user's half: the world's own
17
- vendor routes, selected by session and checked against that session's token,
18
- answered in-process by :class:`WorldApp`.
16
+ * the **data plane** (everything else, every path under the worlds domain included) is the
17
+ end user's half: the world's own vendor routes, selected by session and checked against
18
+ that session's token, answered in-process by :class:`WorldApp`.
19
19
 
20
20
  Every session carries a lease. Control calls and data-plane requests renew it;
21
21
  a session the platform stops touching expires on the host and is unlinked, so
22
22
  a missed reconcile tick on the platform is recovered, not leaked.
23
23
 
24
- A session's state is one sqlite file, so it is also its own durable form: the
24
+ A session's state is one database file, so it is also its own durable form: the
25
25
  platform asks the host to **export** a session (a consistent copy, hashed) and
26
26
  to **upload** that copy to a presigned URL the platform holds the key for, and
27
27
  later **opens** the same session id on any host with ``restore`` naming the
@@ -36,8 +36,8 @@ environment, running the same :class:`WorldHost` with ``run_handlers`` on. The
36
36
  supervisor keeps the registry, the leases and the token gate; the worker keeps
37
37
  the world. The **ui** surface is the version's prebuilt static app, served from
38
38
  the cache by the supervisor with its ``api_path`` rewritten onto the session's
39
- api and logged ``via: ui`` (the api origin's calls ``via: api``, an operation
40
- called by name ``via: tool``); the **browser** surface is a websocket tunnel to the pool Chromium the
39
+ api and logged ``via: ui`` (the api origin's calls ``via: api``, a declared
40
+ tool called by name ``via: tool``); the **browser** surface is a websocket tunnel to the pool Chromium the
41
41
  platform leased for the session (:mod:`host_surfaces`). Custom-server worlds
42
42
  stay on pods; this module refuses a version it cannot open rather than
43
43
  guessing.
@@ -45,1484 +45,71 @@ guessing.
45
45
 
46
46
  from __future__ import annotations
47
47
 
48
- import base64
49
- import hashlib
50
- import hmac
51
- import http.client
52
- import json
53
48
  import os
54
- import re
55
- import shutil
56
49
  import signal
57
50
  import sys
58
- import tarfile
59
51
  import threading
60
- import time
61
- import urllib.error
62
52
  import urllib.parse
63
- import urllib.request
64
- import uuid
65
- from collections import Counter
66
- from collections.abc import Callable, Iterable, Iterator
67
- from contextlib import contextmanager
68
- from dataclasses import dataclass, field
69
53
  from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
70
54
  from pathlib import Path
71
- from typing import Any, Optional, Protocol
72
-
73
- from .._tar import safe_tar_extract
74
- from ..world import WorldTree
75
- from . import snapshot as snapshots
76
- from .api import (
77
- ApiSettings,
78
- Handler,
79
- Router,
80
- Via,
81
- WorldApp,
82
- load_api_settings,
83
- load_handlers,
84
- send_response,
85
- with_query,
86
- )
87
- from .compiler import check_bundle
88
- from .connector import Spec, load_spec
89
- from .credentials import basic_user
90
- from .dependencies import Dependency
91
- from .host_requests import BrowserRelay, DependencyRef, parse_dependency_refs
55
+ from typing import Any
56
+
57
+ from .api import Via, with_query
58
+ from .call_capture import JSON_TYPE, send_response
59
+ from .credentials import BearerAuth
60
+ from .door import PLATFORM_HEADER, Entry
61
+ from .handler_errors import log_request_failure, request_failure_body
62
+ from .host_control import ControlPlane
63
+ from .host_handoff import DrainHandoff, HandoffDoor, PlatformHandoff
64
+ from .host_loops import HEARTBEAT_S, Heartbeat, start_lease_sweeper
65
+ from .host_owners import CLOSED_SESSION, FRAMING_HEADERS, OwnerLookup, PlatformOwners
66
+ from .host_proxy import OwnerProxy
67
+ from .host_session import Session
92
68
  from .host_surfaces import (
93
69
  HOP_HEADER,
70
+ MAX_HOPS,
71
+ NO_REFERRER,
72
+ NO_SESSION_NAMED,
94
73
  UiSurface,
95
74
  api_target,
96
75
  cookie_token,
97
76
  cross_site,
98
- load_ui_surface,
77
+ hops,
99
78
  matches,
100
79
  mint_live_grant,
101
80
  query_token,
102
81
  session_cookie,
82
+ session_from_host,
103
83
  static_file,
104
84
  tunnel,
105
85
  verify_live_grant,
106
86
  without_query_token,
107
87
  )
108
- from .runtime import SnapshotRuntime, TreeRuntime, WorldRuntime
109
- from .tasks import TaskSpec, parse_task_spec, unknown_task_tools
110
- from .tools_world import DependencyWorld, ToolsVersion, is_tools_bundle, load_tools_version
111
- from .validation import ContractError, fail
112
- from .verbs import SESSION_VERBS, session_verb, unknown_verb
88
+ from .host_transfers import parse_origins
89
+ from .host_versions import DEFAULT_MAX_BUNDLE_MB
90
+ from .host_workers import UNKNOWN_SESSION, refusal
91
+ from .request_ledger import LedgerFlusher, LedgerSpill, NoLedger, PlatformLedger
92
+ from .session_move import SessionMoved
93
+ from .validation import ContractError
113
94
  from .workers import (
114
95
  DEFAULT_WORKER_BUDGET_MB,
115
96
  DEFAULT_WORKER_FOOTPRINT_MB,
116
97
  DEFAULT_WORKER_MEMORY_MB,
117
- Worker,
118
- WorkerCapacity,
119
- WorkerGone,
120
98
  WorkerPool,
99
+ seal_process,
121
100
  )
101
+ from .world_host import DEFAULT_LEASE_S, DEFAULT_MAX_SESSIONS, HOSTED_OPERATIONS, WorldHost
122
102
 
123
- #: The world tree inside an api bundle, or the bundle root for a bare tree.
124
- BUNDLED_WORLD = "server/world"
125
- WORLD_SPEC = "schema/world.json"
126
- #: How long a session lives with no control call or request renewing it.
127
- DEFAULT_LEASE_S = 3600.0
128
- #: How often the host sweeps expired leases.
129
- LEASE_SWEEP_S = 30.0
130
- #: How long an owner lookup answer is trusted before asking the platform again.
131
- #: An owner that does not answer a proxied request is asked about again at once.
132
- OWNER_CACHE_S = 60.0
133
- #: How long "no owner" is trusted: long enough that a client hammering a closed
134
- #: session costs the platform one call at a time, short enough that a session
135
- #: recovered onto another host is reached at once.
136
- OWNER_MISS_S = 2.0
137
- #: How many sessions one host takes before the platform places elsewhere.
138
- DEFAULT_MAX_SESSIONS = 1500
139
- #: The error code (and a 404) a control call gets for a session this host does
140
- #: not hold. The platform treats exactly this answer as "the host lost it".
141
- UNKNOWN_SESSION = "unknown_session"
142
- #: How often a host tells the platform it is alive and what it holds.
143
- HEARTBEAT_S = 10.0
144
- #: How long a draining host waits for its sessions to close before it exits.
145
- DEFAULT_DRAIN_S = 45.0
146
- #: Response headers a relayed answer does NOT keep: the framing is this
147
- #: connection's, everything else the world set (its ``page_headers``, a
148
- #: handler's own headers) passes through, from a worker and from an owner alike.
149
- FRAMING_HEADERS = frozenset({"content-length", "connection", "transfer-encoding", "server", "date"})
150
- #: Hop-by-hop headers (RFC 9110 §7.6.1) an owner's answer does not carry past
151
- #: the proxying replica.
152
- HOP_BY_HOP_HEADERS = frozenset(
153
- {"keep-alive", "proxy-authenticate", "proxy-authorization", "te", "trailer", "upgrade", "proxy-connection"}
154
- )
155
- PROXIED_DROP_HEADERS = FRAMING_HEADERS | HOP_BY_HOP_HEADERS
156
-
157
-
158
- class _NoRedirect(urllib.request.HTTPRedirectHandler):
159
- """An owner's redirect is the client's to follow, not the proxy's: the ui
160
- token exchange answers 303 with the session cookie, and following it here
161
- would drop the cookie and land on the owner's 401."""
103
+ DEFAULT_DRAIN_S = 240.0
162
104
 
163
- def redirect_request(self, req, fp, code, msg, headers, newurl):
164
- return None
165
105
 
166
-
167
- #: The opener the one-hop proxy uses: relays the owner's answer as is, a
168
- #: redirect included, rather than following it.
169
- _PROXY_OPENER = urllib.request.build_opener(_NoRedirect)
170
- #: How long the grant the host mints towards the relay lives: the viewer's own
171
- #: grant was already checked, this one only has to survive the handshake.
106
+ _BEARER = BearerAuth()
107
+ #: Short-lived: the viewer's own grant was already checked, this one only has to survive the handshake.
172
108
  LIVE_RELAY_GRANT_MS = 60_000
173
- #: What a session export is: the session's sqlite file, as the store wrote it.
174
- EXPORT_CONTENT_TYPE = "application/vnd.sqlite3"
175
- #: How long one export upload or restore download may take.
176
- TRANSFER_TIMEOUT_S = 300.0
177
- #: The most a version bundle may weigh, compressed and extracted, before the
178
- #: host refuses it: the cache is one shared disk and a build's memory is the
179
- #: supervisor's.
180
- DEFAULT_MAX_BUNDLE_MB = 2048
181
- MAX_BUNDLE_MEMBERS = 200_000
182
- #: The largest request body the host reads, on either plane. A body is held
183
- #: in memory whole (the seed rows, a tool call's args, a vendor route's
184
- #: payload), and the memory is the supervisor's, shared by every session on
185
- #: the replica; a client cannot be let size that allocation.
109
+ #: A body is held in memory whole, in the supervisor shared by every session: a client must not size that allocation.
186
110
  DEFAULT_MAX_BODY_MB = 32
187
111
 
188
112
 
189
- OBJECT_STORE_ANY = "any"
190
- _ORIGIN_HOST = re.compile(r"^[A-Za-z0-9.-]+(:[0-9]+)?$")
191
-
192
-
193
- def parse_origins(value: str | None) -> frozenset[str] | None:
194
- """``WORLDS_HOST_OBJECT_STORE``: the origins (``scheme://host[:port]``,
195
- comma-separated) the platform's presigned URLs may point at, or the word
196
- ``any`` for a dev host reading ``file://`` bundles (``None``: every URL
197
- admitted). Unset is refused rather than read as ``any``: a deployed host
198
- with no allowlist is a host whose secret can aim it anywhere, and a
199
- template's placeholder (``https://<bucket>.s3...``) is refused by name."""
200
- if value is None or not value.strip():
201
- fail(
202
- "host",
203
- "WORLDS_HOST_OBJECT_STORE is unset; name the object store's origins "
204
- f"(https://bucket.s3.amazonaws.com) or {OBJECT_STORE_ANY!r} for a dev host",
205
- )
206
- raise AssertionError # fail() never returns
207
- if value.strip().lower() == OBJECT_STORE_ANY:
208
- return None
209
- origins = set()
210
- for item in value.split(","):
211
- item = item.strip()
212
- if not item:
213
- continue
214
- parts = urllib.parse.urlsplit(item)
215
- if parts.scheme not in ("http", "https") or not parts.netloc or parts.path not in ("", "/"):
216
- fail(
217
- "host",
218
- f"WORLDS_HOST_OBJECT_STORE entries are origins like https://bucket.s3.amazonaws.com, not {item!r}",
219
- )
220
- if not _ORIGIN_HOST.match(parts.netloc):
221
- fail(
222
- "host",
223
- f"WORLDS_HOST_OBJECT_STORE names a placeholder, not a host: {item!r}; set the real bucket origin",
224
- )
225
- origins.add(f"{parts.scheme}://{parts.netloc}".lower())
226
- return frozenset(origins)
227
-
228
-
229
- def check_object_store_url(url: object, origins: frozenset[str] | None, what: str) -> str:
230
- """The presigned URL the control plane handed us, once it is a string on one
231
- of the object store's origins. The control plane is trusted with the host
232
- secret, not with the host's network: a URL pointing anywhere else (the
233
- metadata service, a cluster Service, the internet) is refused before a
234
- byte moves, so a leaked secret cannot turn a session's state or the host's
235
- reach into someone else's download."""
236
- if not isinstance(url, str) or not url:
237
- fail("host", f"{what} must be a presigned url")
238
- raise AssertionError # fail() never returns
239
- if origins is None:
240
- return url
241
- parts = urllib.parse.urlsplit(url)
242
- origin = f"{parts.scheme}://{parts.netloc}".lower()
243
- if origin not in origins:
244
- fail("host", f"{what} must be on the object store ({', '.join(sorted(origins))}), not {origin}")
245
- return url
246
-
247
-
248
- def file_digest(path: Path) -> tuple[str, int]:
249
- """The hex sha256 and size of a file, streamed."""
250
- digest = hashlib.sha256()
251
- size = 0
252
- with open(path, "rb") as fh:
253
- for chunk in iter(lambda: fh.read(1 << 20), b""):
254
- digest.update(chunk)
255
- size += len(chunk)
256
- return digest.hexdigest(), size
257
-
258
-
259
- def download_verified(url: str, sha256: str, dest: Path) -> Path:
260
- """Fetch ``url`` into ``dest`` and refuse it unless its bytes hash to
261
- ``sha256``; a mismatch leaves nothing behind."""
262
- dest.parent.mkdir(parents=True, exist_ok=True)
263
- staging = dest.with_name(f".{dest.name}.{uuid.uuid4().hex}")
264
- try:
265
- try:
266
- with (
267
- urllib.request.urlopen(urllib.request.Request(url), timeout=TRANSFER_TIMEOUT_S) as r,
268
- open(staging, "wb") as out,
269
- ):
270
- shutil.copyfileobj(r, out)
271
- except Exception as error: # noqa: BLE001 - name the stage in the message
272
- fail("host", f"could not download the session export: {error}")
273
- actual, _ = file_digest(staging)
274
- if actual != sha256.lower():
275
- fail("host", f"session export checksum mismatch: expected {sha256}, got {actual}")
276
- os.replace(staging, dest)
277
- finally:
278
- staging.unlink(missing_ok=True)
279
- return dest
280
-
281
-
282
- def upload_file(url: str, path: Path, sha256: str, size: int, content_type: str = EXPORT_CONTENT_TYPE) -> None:
283
- """PUT a file to a presigned URL, sending exactly the headers the platform
284
- signed for it: content type, length and the base64 sha256 the object store
285
- verifies on receipt."""
286
- headers = {
287
- "Content-Type": content_type,
288
- "Content-Length": str(size),
289
- "x-amz-checksum-sha256": base64.b64encode(bytes.fromhex(sha256)).decode("ascii"),
290
- }
291
- with open(path, "rb") as fh:
292
- request = urllib.request.Request(url, data=fh, method="PUT", headers=headers)
293
- try:
294
- with urllib.request.urlopen(request, timeout=TRANSFER_TIMEOUT_S) as response:
295
- response.read()
296
- except urllib.error.HTTPError as error:
297
- fail("host", f"the export upload was refused: HTTP {error.code} {error.read()[:200]!r}")
298
- except urllib.error.URLError as error:
299
- fail("host", f"could not upload the export: {error.reason}")
300
-
301
-
302
- def world_tree(bundle: Path) -> Path:
303
- """The compiled schema world inside an extracted bundle (``WorldTree.world``:
304
- ``bench push`` packages a schema world as its api bundle with the tree
305
- under ``server/world``; a bare tree is its own root). The host opens
306
- compiled worlds only, so a tree without ``schema/world.json`` refuses."""
307
- tree = WorldTree.from_dir(bundle)
308
- if tree.has(tree.world_path(WORLD_SPEC)):
309
- return tree.world
310
- fail(
311
- "host",
312
- f"{bundle} holds no schema world the host can open (no {BUNDLED_WORLD}/{WORLD_SPEC} "
313
- f"nor {WORLD_SPEC}); the host serves schema worlds and api bundles only",
314
- )
315
- raise AssertionError # fail() never returns
316
-
317
-
318
- @dataclass
319
- class CachedVersion:
320
- """One cached version, of either shape the host serves: an api bundle (a
321
- connector ``spec`` and ``settings``, sessions cloned from ``snapshot``) or a
322
- schema world stored as itself (``tools``: handlers, tasks and verifiers,
323
- sessions copied from its template — and ``spec`` + ``settings`` too when
324
- its tree carries connector.toml, the HTTP surface the same session serves
325
- over the same store). ``snapshot`` is set for an api bundle only."""
326
-
327
- version_id: str
328
- bundle: Path
329
- tree: Path
330
- spec: Spec | None
331
- settings: ApiSettings | None
332
- snapshot: Path | None
333
- #: The world's own code, imported once per version — only in a process
334
- #: that runs handlers (a worker). None in the supervisor.
335
- handlers: dict[str, Handler] | None = None
336
- #: The version's prebuilt static ui, when its manifest declares one.
337
- ui: UiSurface | None = None
338
- #: A mock-tools version; None for an api world.
339
- tools: ToolsVersion | None = None
340
-
341
- @property
342
- def runtime_class(self) -> type[WorldRuntime]:
343
- """The one place a session's engine is chosen: a schema world runs as
344
- the tree it is; an api bundle on the snapshot READ PATH that step 6
345
- deletes with the bundle shape itself."""
346
- return TreeRuntime if self.tools is not None else SnapshotRuntime
347
-
348
- def router(self) -> Router:
349
- """A fresh router per session: routes are compiled from the shared spec,
350
- but the oauth2 tokens a world mints must stay that session's own."""
351
- assert self.spec is not None and self.settings is not None
352
- return Router(self.spec, self.settings)
353
-
354
- @property
355
- def authored_code(self) -> str | None:
356
- """What world-authored code the version runs, None for none: only a
357
- worker may open a version that runs any."""
358
- return self.runtime_class.authored_code(self)
359
-
360
- @property
361
- def needs_worker(self) -> bool:
362
- return self.authored_code is not None
363
-
364
- def served_tools(self, dependencies: dict[str, DependencyWorld]) -> list[str]:
365
- """Every name a session task may allow."""
366
- return self.runtime_class.served_by(self, dependencies)
367
-
368
- @property
369
- def declared_dependencies(self) -> dict[str, Dependency]:
370
- return self.runtime_class.declared_dependencies(self)
371
-
372
-
373
- class VersionCache:
374
- """Every world version this host has seen, on local disk. The first session
375
- of a version pays for the download and extract; the rest clone its
376
- snapshot. Nothing here is per session, so a hot version is cheap to serve
377
- many times over."""
378
-
379
- def __init__(self, root: Path, *, run_handlers: bool = False, max_bundle_mb: int = DEFAULT_MAX_BUNDLE_MB):
380
- self.root = Path(root)
381
- self.root.mkdir(parents=True, exist_ok=True)
382
- #: Whether this process imports a version's ``handlers.py``. Off in the
383
- #: supervisor, on in a version worker.
384
- self.run_handlers = run_handlers
385
- self.max_bundle_bytes = max_bundle_mb * (1 << 20)
386
- self._lock = threading.Lock()
387
- self._versions: dict[str, CachedVersion] = {}
388
- #: One build at a time per version: the first open of a version does the
389
- #: download, concurrent opens of the same version wait for it.
390
- self._building: dict[str, threading.Lock] = {}
391
-
392
- def ensure(
393
- self,
394
- version_id: str,
395
- bundle_url: str,
396
- snapshot: dict[str, str] | None,
397
- ) -> CachedVersion:
398
- with self._lock:
399
- hit = self._versions.get(version_id)
400
- if hit is not None:
401
- return hit
402
- building = self._building.setdefault(version_id, threading.Lock())
403
- with building:
404
- with self._lock:
405
- hit = self._versions.get(version_id)
406
- if hit is not None:
407
- return hit
408
- try:
409
- cached = self._build(version_id, bundle_url, snapshot)
410
- with self._lock:
411
- self._versions[version_id] = cached
412
- return cached
413
- finally:
414
- # Success or failure, the build slot is over: a failed build
415
- # leaves nothing behind, and the next open of the version starts
416
- # a fresh one.
417
- with self._lock:
418
- self._building.pop(version_id, None)
419
-
420
- def has(self, version_id: str) -> bool:
421
- with self._lock:
422
- return version_id in self._versions
423
-
424
- def version_ids(self) -> list[str]:
425
- with self._lock:
426
- return sorted(self._versions)
427
-
428
- def _build(self, version_id: str, bundle_url: str, snapshot: dict[str, str] | None) -> CachedVersion:
429
- bundle = self.root / version_id / "bundle"
430
- if not bundle.is_dir():
431
- self._download(bundle_url, bundle)
432
- if is_tools_bundle(bundle):
433
- tools = load_tools_version(
434
- bundle, self.root / version_id / "template.db", snapshot, run_handlers=self.run_handlers
435
- )
436
- # A schema world stored as itself: its tools, and — when the tree
437
- # carries connector.toml — its HTTP surface from the same session,
438
- # plus the static ui that connector.toml's [ui] declares over it.
439
- return CachedVersion(
440
- version_id=version_id,
441
- bundle=bundle,
442
- tree=bundle,
443
- spec=tools.spec,
444
- settings=tools.settings,
445
- snapshot=None,
446
- handlers=tools.api_handlers,
447
- ui=load_ui_surface(bundle) if tools.spec is not None else None,
448
- tools=tools,
449
- )
450
- tree = world_tree(bundle)
451
- contract = check_bundle(tree)
452
- if snapshot:
453
- snapshots.fetch(tree, contract, snapshot["url"], snapshot["sha256"], snapshot["digest"])
454
- spec = load_spec(tree)
455
- settings = load_api_settings(tree, spec)
456
- handlers = load_handlers(settings) if self.run_handlers and settings.handlers is not None else None
457
- snap = snapshots.default_snapshot(tree)
458
- if not snap.is_file():
459
- fail(
460
- "host",
461
- f"version {version_id} has no compiled snapshot (db/state.sqlite); "
462
- "publish a data snapshot or pin one on the version",
463
- )
464
- return CachedVersion(
465
- version_id=version_id,
466
- bundle=bundle,
467
- tree=tree,
468
- spec=spec,
469
- settings=settings,
470
- snapshot=snap,
471
- handlers=handlers,
472
- ui=load_ui_surface(bundle),
473
- )
474
-
475
- def _download(self, url: str, dest: Path) -> None:
476
- """Fetch and extract into a sibling staging directory, then rename it into
477
- place: ``dest`` exists only once it is complete, so a download that dies
478
- halfway is never mistaken for a cached bundle on the next open."""
479
- dest.parent.mkdir(parents=True, exist_ok=True)
480
- staging = dest.parent / f".{dest.name}.{uuid.uuid4().hex}"
481
- tgz = staging.with_suffix(".tar.gz")
482
- limit = self.max_bundle_bytes
483
- try:
484
- try:
485
- with (
486
- urllib.request.urlopen(urllib.request.Request(url), timeout=TRANSFER_TIMEOUT_S) as r,
487
- open(tgz, "wb") as out,
488
- ):
489
- copied = 0
490
- for chunk in iter(lambda: r.read(1 << 20), b""):
491
- copied += len(chunk)
492
- if copied > limit:
493
- raise ValueError(f"the bundle is larger than {limit} bytes")
494
- out.write(chunk)
495
- except Exception as error: # noqa: BLE001 - name the stage in the message
496
- fail("host", f"could not download the bundle from {url}: {error}")
497
- staging.mkdir()
498
- with tarfile.open(tgz, "r:gz") as t:
499
- members = t.getmembers()
500
- if len(members) > MAX_BUNDLE_MEMBERS:
501
- fail("host", f"the bundle holds {len(members)} files, more than the {MAX_BUNDLE_MEMBERS} allowed")
502
- extracted = sum(m.size for m in members)
503
- if extracted > limit:
504
- fail("host", f"the bundle extracts to {extracted} bytes, more than the {limit} allowed")
505
- safe_tar_extract(t, staging)
506
- os.replace(staging, dest)
507
- finally:
508
- tgz.unlink(missing_ok=True)
509
- shutil.rmtree(staging, ignore_errors=True)
510
-
511
-
512
- @dataclass
513
- class Export:
514
- """A session's state copied to a file on this host, hashed, waiting for the
515
- platform to hand over the presigned URL it goes to."""
516
-
517
- export_id: str
518
- path: Path
519
- sha256: str
520
- bytes: int
521
-
522
- def describe(self) -> dict:
523
- return {
524
- "exportId": self.export_id,
525
- "sha256": self.sha256,
526
- "bytes": self.bytes,
527
- "contentType": EXPORT_CONTENT_TYPE,
528
- }
529
-
530
-
531
- @dataclass
532
- class Session:
533
- session_id: str
534
- version_id: str
535
- #: The bundled task (a ``scenarios/<task>`` of the version), or None for a
536
- #: session opened on a task of its own.
537
- task: str | None
538
- #: The connector spec of an api bundle, or of a schema world whose tree
539
- #: carries connector.toml; None for a tools-only tree, which has no http
540
- #: surface (its data plane is 404, its tools are ``call``).
541
- spec: Spec | None
542
- token: str | None
543
- bundle: Path
544
- tree: Path
545
- #: Unix time the lease runs out unless renewed.
546
- lease_until: float
547
- #: The world, when this process serves it (:class:`WorldRuntime`). None
548
- #: for a session placed on a worker, whose world lives in that process.
549
- runtime: WorldRuntime | None = None
550
- #: The version worker serving this session, or None for in-process.
551
- worker: Worker | None = None
552
- #: The version's static ui, served by the supervisor for this session.
553
- ui: UiSurface | None = None
554
- #: The pool browser leased for this session (:class:`BrowserRelay`), or
555
- #: None when the session has no browser surface. Swapped in place when
556
- #: the platform replaces a browser that died (:meth:`WorldHost.set_browser`).
557
- browser: BrowserRelay | None = None
558
- #: Requests inside the world right now. A session is only torn down once
559
- #: this reaches zero, so a close or an expiry never pulls the database out
560
- #: from under a handler mid-request.
561
- inflight: int = 0
562
- #: Set once the session has left the registry; the last request out closes
563
- #: the state.
564
- closing: bool = False
565
- #: Unix time the session was opened; its session-seconds are counted on close.
566
- opened_at: float = field(default_factory=time.time)
567
- #: Unix time of the last request that used the world: a data-plane call or
568
- #: a control operation (seed, reset, grade, call, state). A renewal or an
569
- #: export is the platform looking, not the session working, and does not
570
- #: move it. Reported on renew so the platform's idle clock follows traffic
571
- #: that never passes through it.
572
- touched_at: float = field(default_factory=time.time)
573
- #: The last export taken and not yet uploaded, if any.
574
- export: Export | None = None
575
- #: Set when the session was opened from an export rather than the snapshot.
576
- restored: bool = False
577
- #: The task supplied at open, when the session has one instead of a
578
- #: bundled task: its seed is applied after the clone, its grader answers
579
- #: ``grade``, and it lives under ``<session>/task/`` (:mod:`tasks`).
580
- task_spec: TaskSpec | None = None
581
-
582
- @property
583
- def app(self) -> WorldApp | None:
584
- """The world's routes over this session's store, when this process
585
- serves it and the tree has any."""
586
- return self.runtime.app if self.runtime is not None else None
587
-
588
- @property
589
- def task_key(self) -> tuple[str | None, dict | None]:
590
- """What an open retry must match: the bundled name or the spec itself."""
591
- return self.task, self.task_spec.raw if self.task_spec is not None else None
592
-
593
- def renew(self, lease_s: float) -> None:
594
- self.lease_until = time.time() + lease_s
595
-
596
- @property
597
- def seconds(self) -> float:
598
- return max(0.0, time.time() - self.opened_at)
599
-
600
-
601
- class WorldHost:
602
- """The session registry and the control operations, in one of two roles:
603
-
604
- * the **supervisor** (the deployed host): serves code-free worlds in
605
- process and places worlds with handlers on its :class:`WorkerPool`;
606
- * a **worker** (``run_handlers=True``, ``pinned_version=...``): serves one
607
- version, handlers included, for its supervisor.
608
- """
609
-
610
- def __init__(
611
- self,
612
- cache_root: Path,
613
- *,
614
- lease_s: float = DEFAULT_LEASE_S,
615
- advertise_url: str | None = None,
616
- workers: WorkerPool | None = None,
617
- versions_root: Path | None = None,
618
- run_handlers: bool = False,
619
- pinned_version: str | None = None,
620
- limits: dict[str, int] | None = None,
621
- max_sessions: int = DEFAULT_MAX_SESSIONS,
622
- max_bundle_mb: int = DEFAULT_MAX_BUNDLE_MB,
623
- object_store: frozenset[str] | None = None,
624
- ):
625
- #: Where every presigned URL the control plane hands this host must
626
- #: point (bundle, snapshot, export upload, restore); None admits any.
627
- self.object_store = object_store
628
- self.cache = VersionCache(
629
- versions_root or Path(cache_root) / "versions", run_handlers=run_handlers, max_bundle_mb=max_bundle_mb
630
- )
631
- self.sessions_root = Path(cache_root) / "sessions"
632
- # Sessions live in this process; whatever a previous one left on disk is
633
- # unreachable, so a start begins with an empty sessions tree.
634
- shutil.rmtree(self.sessions_root, ignore_errors=True)
635
- self.sessions_root.mkdir(parents=True, exist_ok=True)
636
- self.lease_s = lease_s
637
- #: The address the platform pins a session to, so control calls reach
638
- #: this replica and not the Service's round-robin. None = one replica.
639
- self.advertise_url = advertise_url
640
- self.workers = workers
641
- self.run_handlers = run_handlers
642
- self.pinned_version = pinned_version
643
- #: The rlimits this process runs under (a worker's), for ``/healthz``.
644
- self.limits = limits
645
- #: Sessions this host admits before ``open`` answers 503 and the
646
- #: platform places elsewhere; the heartbeat advertises it as capacity.
647
- self.max_sessions = max_sessions
648
- #: Set on SIGTERM: no new sessions, existing ones served until they
649
- #: close or the drain grace runs out.
650
- self.draining = False
651
- self.started_at = time.time()
652
- self._lock = threading.Lock()
653
- self._sessions: dict[str, Session] = {}
654
- #: Ids of sessions being opened: each holds one of ``max_sessions`` from
655
- #: the moment its open is admitted, so concurrent opens cannot all pass
656
- #: the capacity check and land past the limit.
657
- self._opening: set[str] = set()
658
- #: Opens the platform closed before they finished: it gave up waiting
659
- #: and told this host to forget the session, so the session is closed
660
- #: the moment it exists rather than held until its lease runs out.
661
- self._abandoned: set[str] = set()
662
- #: Lifetime counts for ``/metrics``: opens, closes by reason, requests
663
- #: by surface, session-seconds of closed sessions.
664
- self._counters: Counter[str] = Counter()
665
- self._session_seconds = 0.0
666
-
667
- def open(
668
- self,
669
- session_id: str,
670
- version_id: str,
671
- bundle_url: str,
672
- task: str | None,
673
- token: str | None,
674
- snapshot: dict[str, str] | None,
675
- browser: dict[str, str] | None = None,
676
- restore: dict[str, str] | None = None,
677
- task_spec: Any = None,
678
- dependencies: list[dict[str, Any]] | None = None,
679
- ) -> dict:
680
- """Open ``session_id`` on ``version_id``, either on the bundled ``task``
681
- or on ``task_spec``, a session task of its own (:mod:`tasks`); exactly
682
- one of the two. ``dependencies`` are the versions the world's
683
- ``[dependencies]`` resolved to, one ``{alias, versionId, bundleUrl,
684
- snapshot?}`` per alias the manifest declares, opened beside it."""
685
- if self.pinned_version is not None and version_id != self.pinned_version:
686
- fail("host", f"this worker serves version {self.pinned_version} only, not {version_id}")
687
- if (task is None) == (task_spec is None):
688
- fail("host", "an open names a bundled task or carries a task spec, one of the two")
689
- if task is not None and not isinstance(task, str):
690
- fail("host", "task is the bundled task's name")
691
- spec = parse_task_spec(task_spec) if task_spec is not None else None
692
- deps = parse_dependency_refs(dependencies)
693
- with self._lock:
694
- existing = self._sessions.get(session_id)
695
- if existing is None:
696
- # A retry of an open this host already served is answered even
697
- # while draining: refusing it would send the platform to open
698
- # the same id elsewhere while this copy holds the live writes.
699
- if self.draining:
700
- fail("draining", "this host is draining and takes no new sessions")
701
- self._admit(session_id)
702
- if existing is not None:
703
- # The platform retried an open that already succeeded: the same
704
- # session is the answer, never a rebuild that drops its writes.
705
- wanted = (version_id, (task, spec.raw if spec is not None else None), token)
706
- if (existing.version_id, existing.task_key, existing.token) != wanted:
707
- fail("host", f"session {session_id} is already open as another version, task or token")
708
- return self._describe(existing)
709
- try:
710
- described = self._open(
711
- session_id, version_id, bundle_url, task, spec, token, snapshot, browser, restore, deps
712
- )
713
- except Exception:
714
- self.tally("open_failures")
715
- with self._lock:
716
- self._opening.discard(session_id)
717
- self._abandoned.discard(session_id)
718
- raise
719
- with self._lock:
720
- self._opening.discard(session_id)
721
- abandoned = session_id in self._abandoned
722
- self._abandoned.discard(session_id)
723
- if abandoned:
724
- self.close(session_id)
725
- fail(UNKNOWN_SESSION, f"session {session_id} was closed while it opened")
726
- return described
727
-
728
- def _admit(self, session_id: str) -> None:
729
- """Reserve one of this host's ``max_sessions`` for ``session_id`` while it
730
- opens. Under ``_lock``."""
731
- if session_id in self._opening:
732
- fail("host", f"session {session_id} is being opened already")
733
- if len(self._sessions) + len(self._opening) >= self.max_sessions:
734
- fail("at_capacity", f"this host holds its {self.max_sessions} sessions; place elsewhere")
735
- self._opening.add(session_id)
736
-
737
- def _open(
738
- self,
739
- session_id: str,
740
- version_id: str,
741
- bundle_url: str,
742
- task: str | None,
743
- spec: TaskSpec | None,
744
- token: str | None,
745
- snapshot: dict[str, str] | None,
746
- browser: dict[str, str] | None,
747
- restore: dict[str, str] | None,
748
- deps: list[DependencyRef],
749
- ) -> dict:
750
- cached = self.cache.ensure(version_id, *self._trusted_version(bundle_url, snapshot))
751
- dependencies = self._dependency_worlds(cached, deps)
752
- # A session task runs on either shape the host serves: an api bundle
753
- # clones its snapshot and grades by the spec; a schema world copies its
754
- # template and grades by the spec too, else with its bundled verifiers
755
- # (:meth:`_grade`). Its allow-list may name a tool or, on a tree
756
- # with connector.toml, an operation.
757
- if cached.spec is not None:
758
- unsupported = unsupported_auth(cached.spec)
759
- if unsupported:
760
- fail("host", unsupported)
761
- if spec is not None:
762
- self._check_task_tools(spec, cached.served_tools(dependencies))
763
- relay = BrowserRelay.parse(browser)
764
- if cached.needs_worker and not self.run_handlers:
765
- if self.workers is None:
766
- fail(
767
- "host",
768
- f"version {version_id} runs world-authored code ({cached.authored_code}); "
769
- "this host runs none in its process and has no version workers",
770
- )
771
- return self._open_on_worker(
772
- cached, session_id, bundle_url, task, spec, token, relay, restore, deps
773
- )
774
- session_dir = self.sessions_root / session_id
775
- runtime = WorldRuntime.open(
776
- cached,
777
- session_dir,
778
- task=task,
779
- spec=spec,
780
- token=token,
781
- dependencies=dependencies,
782
- restore=lambda load: self._restore_into(restore, session_dir, load),
783
- )
784
- session = Session(
785
- session_id=session_id,
786
- version_id=version_id,
787
- task=task,
788
- spec=cached.spec,
789
- token=token,
790
- bundle=cached.bundle,
791
- tree=cached.tree,
792
- lease_until=time.time() + self.lease_s,
793
- runtime=runtime,
794
- ui=cached.ui,
795
- browser=relay,
796
- restored=runtime.restored,
797
- task_spec=spec,
798
- )
799
- self._register(session)
800
- return self._describe(session)
801
-
802
- def _open_on_worker(
803
- self,
804
- cached: CachedVersion,
805
- session_id: str,
806
- bundle_url: str,
807
- task: str | None,
808
- spec: TaskSpec | None,
809
- token: str | None,
810
- relay: BrowserRelay | None,
811
- restore: dict[str, str] | None,
812
- deps: list[DependencyRef],
813
- ) -> dict:
814
- """Place the session on the version's worker. The worker opens it from
815
- the bundle this process already cached (no snapshot to fetch: the file
816
- is on disk), and the supervisor records only where it went. The ui's
817
- static files and the browser tunnel stay the supervisor's; the ui's
818
- api-prefix calls are relayed as they came, so the worker logs them
819
- ``via: ui``. An export to restore from is fetched
820
- and verified here too, then handed to the worker as a local file.
821
- A worker every one of whose peers holds sessions cannot be started
822
- past the pool's memory budget: that is ``at_capacity``, and the
823
- platform places the session on another replica."""
824
- assert self.workers is not None
825
- try:
826
- worker = self.workers.get(cached.version_id)
827
- except WorkerCapacity as error:
828
- fail("at_capacity", str(error))
829
- # The session is the worker's from here, so another version's open
830
- # cannot stop this worker as session-less while it is being placed.
831
- with self._lock:
832
- worker.sessions.add(session_id)
833
- source = None
834
- try:
835
- source = self._restore_source(restore, self._worker_session_dir(worker, session_id))
836
- self._worker_control(
837
- worker,
838
- "POST",
839
- "/__host/sessions",
840
- {
841
- "sessionId": session_id,
842
- "versionId": cached.version_id,
843
- "bundleUrl": bundle_url,
844
- "task": task,
845
- "taskSpec": spec.raw if spec is not None else None,
846
- "token": token,
847
- "restore": {"path": str(source)} if source is not None else None,
848
- "dependencies": [dep.raw for dep in deps],
849
- },
850
- )
851
- except BaseException:
852
- with self._lock:
853
- worker.sessions.discard(session_id)
854
- if not worker.sessions:
855
- worker.idle_since = time.time()
856
- raise
857
- finally:
858
- if source is not None:
859
- source.unlink(missing_ok=True)
860
- session = Session(
861
- session_id=session_id,
862
- version_id=cached.version_id,
863
- task=task,
864
- spec=cached.spec,
865
- token=token,
866
- bundle=cached.bundle,
867
- tree=cached.tree,
868
- lease_until=time.time() + self.lease_s,
869
- worker=worker,
870
- ui=cached.ui,
871
- browser=relay,
872
- restored=source is not None,
873
- task_spec=spec,
874
- )
875
- self._register(session)
876
- return self._describe(session)
877
-
878
- def _worker_session_dir(self, worker: Worker, session_id: str) -> Path:
879
- """Where a worker keeps a session's files: its own ``WorldHost`` runs
880
- under ``workers/<version>`` of this host's cache root (:mod:`worker`)."""
881
- assert self.workers is not None
882
- return self.workers.root / "workers" / worker.version_id / "sessions" / session_id
883
-
884
- def _restore_into(self, restore: dict[str, str] | None, session_dir: Path, load: Callable[[Path], None]) -> bool:
885
- """Load the export ``restore`` names into a freshly opened world, then
886
- drop the fetched file. False when there was nothing to restore."""
887
- source = self._restore_source(restore, session_dir)
888
- if source is None:
889
- return False
890
- try:
891
- load(source)
892
- finally:
893
- source.unlink(missing_ok=True)
894
- return True
895
-
896
- def _restore_source(self, restore: dict[str, str] | None, session_dir: Path) -> Path | None:
897
- """The export a session continues from, as a local file, or None for a
898
- fresh session. The platform names it by ``{url, sha256}`` and the file is
899
- fetched into the session's directory and verified; a supervisor names it
900
- to its worker by ``{path}``, already verified, and only the supervisor
901
- reaches a worker's control socket."""
902
- if not restore:
903
- return None
904
- path = restore.get("path")
905
- if path is not None:
906
- if not self.run_handlers:
907
- fail("host", "restore by path is a worker's; name the export by {url, sha256}")
908
- source = Path(path)
909
- if not source.is_file():
910
- fail("host", f"restore file {path} is missing")
911
- return source
912
- url, sha256 = restore.get("url"), restore.get("sha256")
913
- if not isinstance(url, str) or not isinstance(sha256, str) or len(sha256) != 64:
914
- fail("host", "restore must be {url: <presigned export url>, sha256: <hex digest>}")
915
- raise AssertionError # fail() never returns
916
- check_object_store_url(url, self.object_store, "restore.url")
917
- return download_verified(url, sha256, session_dir / "restore.sqlite")
918
-
919
- def _trusted_version(self, bundle_url: str, snapshot: dict[str, str] | None) -> tuple[str, dict[str, str] | None]:
920
- check_object_store_url(bundle_url, self.object_store, "bundleUrl")
921
- if snapshot:
922
- check_object_store_url(snapshot.get("url"), self.object_store, "snapshot.url")
923
- return bundle_url, snapshot
924
-
925
- def _register(self, session: Session) -> None:
926
- with self._lock:
927
- replaced = self._retire(self._sessions.pop(session.session_id, None))
928
- self._sessions[session.session_id] = session
929
- self._counters["opened"] += 1
930
- if replaced is not None:
931
- self._finish(replaced, reason="replaced")
932
-
933
- def _describe(self, session: Session) -> dict:
934
- return {
935
- "sessionId": session.session_id,
936
- "versionId": session.version_id,
937
- "task": session.task,
938
- "taskName": session.task if session.task_spec is None else session.task_spec.name,
939
- "taskSource": "bundled" if session.task_spec is None else "inline",
940
- "tools": (
941
- list(session.task_spec.tools)
942
- if session.task_spec is not None and session.task_spec.tools is not None
943
- else None
944
- ),
945
- "ready": True,
946
- "hostUrl": self.advertise_url,
947
- "leaseSeconds": self.lease_s,
948
- "leaseUntil": session.lease_until,
949
- "workerPid": session.worker.pid if session.worker is not None else None,
950
- "ui": {"entry": session.ui.entry, "apiPath": session.ui.api_path} if session.ui is not None else None,
951
- "browser": session.browser is not None,
952
- "restored": session.restored,
953
- "openedAt": session.opened_at,
954
- "sessionSeconds": session.seconds,
955
- "touchedAt": session.touched_at,
956
- }
957
-
958
- def close(self, session_id: str) -> dict:
959
- with self._lock:
960
- session = self._sessions.pop(session_id, None)
961
- idle = self._retire(session)
962
- if session is None and session_id in self._opening:
963
- self._abandoned.add(session_id)
964
- if idle is not None:
965
- self._finish(idle, reason="closed")
966
- if session is None:
967
- return {"closed": True, "openedAt": None, "sessionSeconds": 0.0}
968
- return {"closed": True, "openedAt": session.opened_at, "sessionSeconds": session.seconds}
969
-
970
- def set_browser(self, session_id: str, browser: Any) -> dict:
971
- """Point the session's browser surface at another pool relay: the one
972
- it had died and the platform leased a replacement. Connections open
973
- on the old relay end with it; every connection after this reaches the
974
- new one. A session opened without a browser never gains one here."""
975
- relay = BrowserRelay.parse(browser)
976
- if relay is None:
977
- fail("host", "browser must be {url: 'ws://host:port', token: <relay token>}")
978
- with self._lock:
979
- session = self._sessions.get(session_id)
980
- if session is None:
981
- fail(UNKNOWN_SESSION, f"no session {session_id} on this host")
982
- if session.browser is None:
983
- fail("host", f"session {session_id} was opened without a browser surface")
984
- session.browser = relay
985
- return {"sessionId": session_id, "browser": True}
986
-
987
- def export(self, session_id: str, known_sha256: str | None = None) -> dict:
988
- """Copy the session's state to a file on this host and hash it. The copy
989
- is consistent (sqlite's online backup under the store's lock) and waits
990
- for :meth:`upload_export`; taking a new one discards the last. A copy
991
- that hashes to ``known_sha256`` (the platform's last export of the
992
- session) is thrown away and reported ``unchanged``: nothing to upload."""
993
- with self.using(session_id, touch=False) as session:
994
- if session.worker is not None:
995
- answer = self._worker_control(session.worker, "POST", f"/__host/sessions/{session_id}/export", {})
996
- path = self._worker_export_path(session, answer)
997
- else:
998
- exports = self.sessions_root / session_id / "exports"
999
- shutil.rmtree(exports, ignore_errors=True)
1000
- path = exports / f"{uuid.uuid4().hex}.sqlite"
1001
- assert session.runtime is not None
1002
- session.runtime.export(path)
1003
- sha256, size = file_digest(path)
1004
- if known_sha256 is not None and sha256 == known_sha256:
1005
- path.unlink(missing_ok=True)
1006
- with self._lock:
1007
- session.export = None
1008
- self._counters["exports_unchanged"] += 1
1009
- return {"exportId": None, "sha256": sha256, "bytes": size, "unchanged": True}
1010
- export = Export(export_id=path.stem, path=path, sha256=sha256, bytes=size)
1011
- with self._lock:
1012
- session.export = export
1013
- self._counters["exports"] += 1
1014
- described = export.describe()
1015
- if self.run_handlers:
1016
- described["path"] = str(path) # for the supervisor, which uploads on the worker's behalf
1017
- return described
1018
-
1019
- def _worker_export_path(self, session: Session, answer: dict) -> Path:
1020
- """The file a worker's export landed in, admitted only from inside that
1021
- worker's directory for this session: the worker runs tenant code, so
1022
- what it names is checked, and the bytes are hashed here, not trusted."""
1023
- assert session.worker is not None
1024
- exports = self._worker_session_dir(session.worker, session.session_id) / "exports"
1025
- path = Path(str(answer.get("path") or ""))
1026
- try:
1027
- resolved = path.resolve(strict=True)
1028
- resolved.relative_to(exports.resolve())
1029
- except (OSError, ValueError):
1030
- fail("host", f"the worker's export is not where a worker's exports go: {path}")
1031
- raise AssertionError # fail() never returns
1032
- return resolved
1033
-
1034
- def upload_export(self, session_id: str, export_id: str, url: str) -> dict:
1035
- """PUT the export :meth:`export` took to the presigned URL the platform
1036
- signed for its hash and size; the local copy goes once it is up."""
1037
- check_object_store_url(url, self.object_store, "url")
1038
- with self.using(session_id, touch=False) as session:
1039
- export = session.export
1040
- if export is None or export.export_id != export_id:
1041
- fail("host", f"no export {export_id} for session {session_id}; take one first")
1042
- raise AssertionError # fail() never returns
1043
- upload_file(url, export.path, export.sha256, export.bytes)
1044
- export.path.unlink(missing_ok=True)
1045
- with self._lock:
1046
- session.export = None
1047
- self._counters["exports_uploaded"] += 1
1048
- return {"uploaded": True, **export.describe()}
1049
-
1050
- def renew(self, session_id: str) -> dict:
1051
- with self.using(session_id, touch=False) as session:
1052
- return self._describe(session)
1053
-
1054
- def expire_leases(self, now: float | None = None) -> list[str]:
1055
- """Close every session whose lease has run out; returns their ids. The
1056
- check and the removal happen under one lock, so a renewal that lands
1057
- in between keeps its session, and a session with a request in flight
1058
- is left for the next sweep (the request renewed it anyway)."""
1059
- now = time.time() if now is None else now
1060
- with self._lock:
1061
- expired = [s for s in self._sessions.values() if s.lease_until <= now and s.inflight == 0]
1062
- for session in expired:
1063
- del self._sessions[session.session_id]
1064
- self._retire(session)
1065
- for session in expired:
1066
- self._finish(session, reason="expired")
1067
- return [s.session_id for s in expired]
1068
-
1069
- def _retire(self, session: Session | None) -> Session | None:
1070
- """Mark a session that has left the registry; returns it when nobody is
1071
- inside it, so the caller closes the state now. Otherwise the last
1072
- request out does (see :meth:`_release`). Call under the lock."""
1073
- if session is None:
1074
- return None
1075
- session.closing = True
1076
- return session if session.inflight == 0 else None
1077
-
1078
- def _finish(self, session: Session, *, reason: str = "closed") -> None:
1079
- """Free a session that has left the registry and has nobody inside it:
1080
- close its state here, or tell its worker to."""
1081
- with self._lock:
1082
- self._counters[f"closed.{reason}"] += 1
1083
- self._session_seconds += session.seconds
1084
- if session.worker is None:
1085
- assert session.runtime is not None
1086
- session.runtime.close()
1087
- return
1088
- worker = session.worker
1089
- with self._lock:
1090
- worker.sessions.discard(session.session_id)
1091
- if not worker.sessions:
1092
- worker.idle_since = time.time()
1093
- try:
1094
- self._worker_control(worker, "DELETE", f"/__host/sessions/{session.session_id}", None)
1095
- except (WorkerGone, ContractError):
1096
- pass # the worker is gone, or already forgot it: either way the session is over
1097
-
1098
- def _worker_control(self, worker: Worker, method: str, path: str, body: dict | None) -> dict:
1099
- """One control call to a worker, its answer as this host would give it:
1100
- the worker's refusal is re-raised as ours, and a worker that has died
1101
- takes every session it held with it."""
1102
- raw = json.dumps(body).encode("utf-8") if body is not None else b""
1103
- try:
1104
- status, _, payload = worker.request(method, path, {"content-type": "application/json"}, raw, control=True)
1105
- except WorkerGone:
1106
- self.lose_worker(worker)
1107
- raise
1108
- answer = json.loads(payload or b"null")
1109
- if status != 200:
1110
- detail = answer if isinstance(answer, dict) else {}
1111
- fail(str(detail.get("error") or "host"), str(detail.get("message") or detail.get("detail") or answer))
1112
- return answer if isinstance(answer, dict) else {"result": answer}
1113
-
1114
- def lose_worker(self, worker: Worker) -> list[str]:
1115
- """A worker exited: forget it and every session it held. Returns the
1116
- session ids, which are now gone from this host (the platform learns of
1117
- it on the next renew)."""
1118
- if self.workers is not None:
1119
- self.workers.drop(worker.version_id)
1120
- with self._lock:
1121
- lost = [sid for sid, s in self._sessions.items() if s.worker is worker]
1122
- for sid in lost:
1123
- session = self._sessions.pop(sid)
1124
- session.closing = True
1125
- self._counters["closed.lost"] += 1
1126
- self._session_seconds += session.seconds
1127
- worker.sessions.clear()
1128
- return lost
1129
-
1130
- def reap_workers(self) -> list[str]:
1131
- """Stop version workers that have had no session for a while."""
1132
- return self.workers.reap_idle() if self.workers is not None else []
1133
-
1134
- def session_count(self) -> int:
1135
- with self._lock:
1136
- return len(self._sessions)
1137
-
1138
- def worker_count(self) -> int:
1139
- return len(self.workers._workers) if self.workers is not None else 0
1140
-
1141
- def warm(self, version_id: str, bundle_url: str, snapshot: dict[str, str] | None) -> dict:
1142
- """Cache a version before any session asks for it, so the first open
1143
- pays nothing. Idempotent: a version already here is reported, not
1144
- rebuilt."""
1145
- already = self.cache.has(version_id)
1146
- if not already:
1147
- self.cache.ensure(version_id, *self._trusted_version(bundle_url, snapshot))
1148
- return {"versionId": version_id, "cached": True, "alreadyCached": already}
1149
-
1150
- def drain(self) -> None:
1151
- """Stop taking sessions; the ones here are served until they close."""
1152
- self.draining = True
1153
-
1154
- def tally(self, key: str) -> None:
1155
- with self._lock:
1156
- self._counters[key] += 1
1157
-
1158
- def stats(self) -> dict:
1159
- """What this host is and holds, for the heartbeat and ``/__host/stats``:
1160
- the placement inputs (address, load, capacity, draining, which versions
1161
- are already cached) plus the lifetime counters."""
1162
- with self._lock:
1163
- sessions = len(self._sessions)
1164
- versions_in_use = sorted({s.version_id for s in self._sessions.values()})
1165
- counters = dict(self._counters)
1166
- session_seconds = self._session_seconds
1167
- return {
1168
- "hostUrl": self.advertise_url,
1169
- "sessions": sessions,
1170
- "capacity": self.max_sessions,
1171
- "draining": self.draining,
1172
- "workers": self.worker_count(),
1173
- "maxWorkers": self.workers.max_workers if self.workers is not None else 0,
1174
- "versions": self.cache.version_ids(),
1175
- "versionsInUse": versions_in_use,
1176
- "uptimeSeconds": time.time() - self.started_at,
1177
- "counters": counters,
1178
- "sessionSeconds": session_seconds,
1179
- "pid": os.getpid(),
1180
- "limits": self.limits,
1181
- }
1182
-
1183
- def metrics_text(self) -> str:
1184
- """The prometheus exposition of :meth:`stats`. ``worlds_host_sessions``
1185
- is the number the fleet scales on (sessions per host, not cpu)."""
1186
- stats = self.stats()
1187
- counters: dict[str, int] = stats["counters"]
1188
- lines = [
1189
- "# TYPE worlds_host_sessions gauge",
1190
- f"worlds_host_sessions {stats['sessions']}",
1191
- "# TYPE worlds_host_capacity gauge",
1192
- f"worlds_host_capacity {stats['capacity']}",
1193
- "# TYPE worlds_host_draining gauge",
1194
- f"worlds_host_draining {int(stats['draining'])}",
1195
- "# TYPE worlds_host_workers gauge",
1196
- f"worlds_host_workers {stats['workers']}",
1197
- "# TYPE worlds_host_versions_cached gauge",
1198
- f"worlds_host_versions_cached {len(stats['versions'])}",
1199
- "# TYPE worlds_host_uptime_seconds gauge",
1200
- f"worlds_host_uptime_seconds {stats['uptimeSeconds']:.0f}",
1201
- "# TYPE worlds_host_sessions_opened_total counter",
1202
- f"worlds_host_sessions_opened_total {counters.get('opened', 0)}",
1203
- "# TYPE worlds_host_open_failures_total counter",
1204
- f"worlds_host_open_failures_total {counters.get('open_failures', 0)}",
1205
- "# TYPE worlds_host_session_seconds_total counter",
1206
- f"worlds_host_session_seconds_total {stats['sessionSeconds']:.0f}",
1207
- "# TYPE worlds_host_exports_total counter",
1208
- f"worlds_host_exports_total {counters.get('exports', 0)}",
1209
- "# TYPE worlds_host_exports_uploaded_total counter",
1210
- f"worlds_host_exports_uploaded_total {counters.get('exports_uploaded', 0)}",
1211
- "# TYPE worlds_host_sessions_closed_total counter",
1212
- ]
1213
- for reason in ("closed", "expired", "lost", "replaced"):
1214
- lines.append(
1215
- f'worlds_host_sessions_closed_total{{reason="{reason}"}} {counters.get(f"closed.{reason}", 0)}'
1216
- )
1217
- lines.append("# TYPE worlds_host_requests_total counter")
1218
- for surface in ("api", "ui", "browser", "proxied"):
1219
- lines.append(f'worlds_host_requests_total{{surface="{surface}"}} {counters.get(f"requests.{surface}", 0)}')
1220
- return "\n".join(lines) + "\n"
1221
-
1222
- def _dependency_worlds(self, cached: CachedVersion, deps: list[DependencyRef]) -> dict[str, DependencyWorld]:
1223
- """The dependency versions a session opens beside ``cached``, cached the
1224
- way the root is (a worker imports their handlers too). Every alias the
1225
- root's manifest declares must be resolved, and nothing else: a version
1226
- without ``[dependencies]`` opened with any is refused, as is a
1227
- dependency that is not a schema world."""
1228
- declared = cached.declared_dependencies
1229
- if not deps:
1230
- if declared:
1231
- names = ", ".join(sorted(declared))
1232
- fail("host", f"version {cached.version_id} declares dependencies ({names}); the open resolved none")
1233
- return {}
1234
- worlds: dict[str, DependencyWorld] = {}
1235
- for ref in deps:
1236
- dependency = declared.get(ref.alias)
1237
- if dependency is None:
1238
- known = ", ".join(sorted(declared)) or "none"
1239
- fail("host", f"version {cached.version_id} declares no dependency '{ref.alias}' (it declares: {known})")
1240
- raise AssertionError # fail() never returns
1241
- dep = self.cache.ensure(ref.version_id, *self._trusted_version(ref.bundle_url, ref.snapshot))
1242
- if dep.tools is None:
1243
- fail("host", f"dependency '{ref.alias}' ({ref.version_id}) is an api bundle, not a schema world")
1244
- raise AssertionError # fail() never returns
1245
- worlds[ref.alias] = DependencyWorld.of(dependency, dep.tools)
1246
- cached.served_tools(worlds) # refuses a root tool that reads as a dependency's
1247
- return worlds
1248
-
1249
- @staticmethod
1250
- def _check_task_tools(spec: TaskSpec, served_tools: Iterable[str]) -> None:
1251
- """A task's allow-list names tools the world serves, every one of them:
1252
- a misspelt name would narrow the list without anyone noticing. The
1253
- served tools are what :meth:`CachedVersion.served_tools` lists."""
1254
- served_tools = list(served_tools)
1255
- unknown = unknown_task_tools(spec, served_tools)
1256
- if unknown:
1257
- served = ", ".join(sorted(served_tools)) or "none"
1258
- fail(
1259
- "task",
1260
- f"the task allows {', '.join(unknown)}, which this world does not serve; its tools are {served}",
1261
- "/task/tools",
1262
- )
1263
-
1264
- @contextmanager
1265
- def using(self, session_id: str, *, touch: bool = True) -> Iterator[Session]:
1266
- """The session for the duration of one request: its lease renewed (any
1267
- traffic keeps it alive), its ``touched_at`` moved unless ``touch`` is
1268
- off (a renewal or export observing the session, not using it), and its
1269
- teardown held off until the block ends."""
1270
- with self._lock:
1271
- session = self._sessions.get(session_id)
1272
- if session is not None:
1273
- session.renew(self.lease_s)
1274
- if touch:
1275
- session.touched_at = time.time()
1276
- session.inflight += 1
1277
- if session is None:
1278
- fail(UNKNOWN_SESSION, f"no session {session_id} on this host")
1279
- raise AssertionError # fail() never returns
1280
- try:
1281
- yield session
1282
- finally:
1283
- self._release(session)
1284
-
1285
- def _release(self, session: Session) -> None:
1286
- with self._lock:
1287
- session.inflight -= 1
1288
- last_out = session.closing and session.inflight == 0
1289
- if last_out:
1290
- self._finish(session)
1291
-
1292
- def owns(self, session_id: str) -> bool:
1293
- with self._lock:
1294
- return session_id in self._sessions
1295
-
1296
- def _control(self, session: Session, method: str, verb: str, body: dict | None) -> dict:
1297
- return self._worker_control(session.worker, method, f"/__host/sessions/{session.session_id}/{verb}", body)
1298
-
1299
- def _verb(self, session_id: str, method: str, verb: str, body: dict | None, local: Callable[[Any], dict]) -> dict:
1300
- """One session verb: forwarded to the session's worker when it has
1301
- one, else answered by its in-process runtime through ``local``."""
1302
- with self.using(session_id) as session:
1303
- if session.worker is not None:
1304
- return self._control(session, method, verb, body)
1305
- assert session.runtime is not None
1306
- return local(session.runtime)
1307
-
1308
- def tools(self, session_id: str) -> dict:
1309
- return self._verb(session_id, "GET", "tools", None, lambda runtime: {"tools": runtime.served()})
1310
-
1311
- def task(self, session_id: str) -> dict:
1312
- return self._verb(session_id, "GET", "task", None, lambda runtime: runtime.task_card())
1313
-
1314
- def reset(self, session_id: str) -> dict:
1315
- return self._verb(session_id, "POST", "reset", {}, lambda runtime: runtime.reset())
1316
-
1317
- def seed(self, session_id: str, rows: dict, mode: str = "append", redact: str = "off") -> dict:
1318
- args = {"rows": rows, "mode": mode, **({"redact": redact} if redact != "off" else {})}
1319
- return self._verb(session_id, "POST", "seed", {"args": args}, lambda runtime: runtime.seed(rows, mode, redact))
1320
-
1321
- def state(self, session_id: str) -> dict:
1322
- return self._verb(session_id, "GET", "state", None, lambda runtime: runtime.state())
1323
-
1324
- def evidence(self, session_id: str) -> dict:
1325
- """The session's state as a grader reads it (rows keyed by primary key,
1326
- the calls): what the platform gathers from each world of a multi-world
1327
- task and hands another world's ``grade`` as ``context``."""
1328
- return self._verb(session_id, "GET", "evidence", None, lambda runtime: runtime.grader_state())
1329
-
1330
- def grade(self, session_id: str, report: Optional[str] = None, context: Optional[dict] = None) -> dict:
1331
- """Grade the session. ``report`` is the agent's own account of the run — the
1332
- text a rubric judge reads; here it is put beside the state as ``report`` so a
1333
- deterministic grader (assertions, verify.py) can check what the agent CLAIMED
1334
- against what the calls show, without a judge. ``context`` is the other
1335
- worlds' evidence the task's cross-world verifiers read (``{alias: state}``)."""
1336
- body = {key: value for key, value in (("report", report), ("context", context)) if value is not None}
1337
- return self._verb(session_id, "POST", "grade", body, lambda runtime: runtime.grade(report, context))
1338
-
1339
- def call(self, session_id: str, tool: str, args: dict) -> dict:
1340
- """Invoke one of the world's tools or vendor routes by name, in process.
1341
- A route's arg shape mirrors the runner's ``execute_route``: ``path``/
1342
- ``query``/``body``, or leftover top-level keys as the body."""
1343
- with self.using(session_id) as session:
1344
- if session.worker is not None:
1345
- return self._control(session, "POST", "call", {"tool": tool, "args": args})
1346
- assert session.runtime is not None
1347
- return session.runtime.call(tool, args)
1348
-
1349
-
1350
- #: The session protocol (:data:`verbs.SESSION_VERBS`) bound to the host's
1351
- #: operations, by route. Every row of the table is served: the local session
1352
- #: (``local.py``) binds the same rows to its own, so the two cannot drift.
1353
- HOSTED_OPERATIONS: dict[str, Callable[..., dict]] = {
1354
- "GET tools": WorldHost.tools,
1355
- "GET task": WorldHost.task,
1356
- "GET state": WorldHost.state,
1357
- "GET evidence": WorldHost.evidence,
1358
- "POST grade": WorldHost.grade,
1359
- "POST reset": WorldHost.reset,
1360
- "POST seed": WorldHost.seed,
1361
- "POST call": WorldHost.call,
1362
- "POST close": WorldHost.close,
1363
- "POST renew": WorldHost.renew,
1364
- "POST export": WorldHost.export,
1365
- "DELETE <session>": WorldHost.close,
1366
- }
1367
- assert set(HOSTED_OPERATIONS) == {verb.route for verb in SESSION_VERBS}, "the host serves every session verb"
1368
-
1369
-
1370
- def public_host(host_header: str | None, worlds_domain: str) -> bool:
1371
- """True when ``Host`` names something under the worlds domain: a session,
1372
- or any other label the wildcard ALB forwarded. That is the public side.
1373
- Prometheus, the kubelet and the ALB health check address the pod itself
1374
- (``ip:port``), which is not."""
1375
- name = (host_header or "").split(":")[0].rstrip(".").lower()
1376
- domain = worlds_domain.rstrip(".").lower()
1377
- return bool(domain) and (name == domain or name.endswith("." + domain))
1378
-
1379
-
1380
- def session_from_host(host_header: str | None) -> tuple[str, str] | None:
1381
- """``w-<sid>-api.<domain>`` / ``w-<sid>.<domain>`` → (session id, surface).
1382
- Returns None when the host does not name a session."""
1383
- if not host_header:
1384
- return None
1385
- label = host_header.split(":")[0].split(".")[0]
1386
- if not label.startswith("w-"):
1387
- return None
1388
- rest = label[2:]
1389
- if rest.endswith("-api"):
1390
- return rest[: -len("-api")], "api"
1391
- if rest.endswith("-cdp"):
1392
- return rest[: -len("-cdp")], "browser"
1393
- return rest, "ui"
1394
-
1395
-
1396
- #: Auth schemes the host can present a world's own credential for. Everything
1397
- #: in ``connector.SCHEMES`` is here; the list is explicit so a new scheme is
1398
- #: refused at open rather than admitted with no credential.
1399
- SUPPORTED_AUTH_SCHEMES = ("none", "header", "bearer", "basic", "query", "oauth2")
1400
-
1401
-
1402
- def unsupported_auth(spec: Spec) -> str | None:
1403
- if spec.scheme in SUPPORTED_AUTH_SCHEMES:
1404
- return None
1405
- return (
1406
- f"the host cannot present auth.scheme '{spec.scheme}' to a world; it serves {', '.join(SUPPORTED_AUTH_SCHEMES)}"
1407
- )
1408
-
1409
-
1410
- def world_credential(session: Session) -> tuple[dict[str, str], dict[str, str]]:
1411
- """(headers, query) carrying the world's own credential for this session.
1412
- A caller that satisfies the host's session-token gate has already been
1413
- authorized; the host then presents the credential the WORLD itself expects
1414
- (the session token in the vendor's shape, or an oauth2 world's grant), so
1415
- the emulated vendor's own auth check passes."""
1416
- assert session.runtime is not None, "a session on a worker presents its credential there"
1417
- return session.runtime.credential()
1418
-
1419
-
1420
- def bearer_of(headers: dict[str, str]) -> str | None:
1421
- auth = headers.get("authorization", "")
1422
- return auth[7:] if auth.lower().startswith("bearer ") else None
1423
-
1424
-
1425
- def vendor_shaped_token(spec: Spec, headers: dict[str, str], token: str) -> bool:
1426
- """Whether the request carries ``token`` the way the world's own auth scheme
1427
- carries a credential: the declared header for ``header``, the user of a
1428
- Basic authorization for ``basic``. A ``query`` scheme is not honoured here (a
1429
- token in the URL outlives the request in logs); ``bearer`` is the gate's
1430
- own shape and ``oauth2`` has its grant."""
1431
- if spec.scheme == "header":
1432
- presented = headers.get((spec.auth_name or "x-api-key").lower())
1433
- return presented is not None and hmac.compare_digest(presented, token)
1434
- if spec.scheme == "basic":
1435
- user = basic_user(headers.get("authorization", ""))
1436
- return user is not None and hmac.compare_digest(user, token)
1437
- return False
1438
-
1439
-
1440
- def admits(session: Session, path: str, headers: dict[str, str]) -> str | None:
1441
- """How the host's gate admits a data-plane request, or None to refuse.
1442
-
1443
- ``"session"``: the caller presented the session token as a bearer and the
1444
- host will present the world's credential for it. Only the header counts: a
1445
- token in the URL would outlive the request in access logs and referrers.
1446
- ``"world"``: an oauth2 caller doing the vendor's own dance — the grant
1447
- request (the world checks ``client_secret`` against the session token) or
1448
- a bearer this session's world minted — whose headers pass through as sent.
1449
- """
1450
- if session.token is None:
1451
- return "session"
1452
- bearer = bearer_of(headers)
1453
- if bearer is not None and hmac.compare_digest(bearer, session.token):
1454
- return "session"
1455
- if session.spec is not None and vendor_shaped_token(session.spec, headers, session.token):
1456
- # The vendor's own wire shape carrying the session token: a client
1457
- # written against the real vendor (`x-api-key: …`, a Basic user) works
1458
- # unchanged with the session token as its key. The same secret as the
1459
- # bearer, in the header the world declares — never in the URL.
1460
- return "session"
1461
- if session.spec is not None and session.spec.scheme == "oauth2":
1462
- if session.app is None:
1463
- return "world" # the world's tokens live in its worker, which runs this same gate
1464
- token_path = session.app.settings.token_path
1465
- if token_path and session.app._strip(path) == token_path:
1466
- return "world"
1467
- if bearer is not None and session.app.router.authorized(headers, {}, session.token):
1468
- return "world"
1469
- return None
1470
-
1471
-
1472
- class OwnerLookup(Protocol):
1473
- """Which host owns a session this replica does not: its control-plane
1474
- URL, or None for a closed or unknown one. ``fresh`` bypasses whatever the
1475
- lookup remembers, for a caller that has just found the remembered owner
1476
- dead."""
1477
-
1478
- def __call__(self, sid: str, *, fresh: bool = False) -> str | None: ...
1479
-
1480
-
1481
- def platform_owner_lookup(platform_url: str, secret: str | None, self_url: str | None) -> OwnerLookup:
1482
- """Ask the platform which host owns a session the caller reached us for.
1483
- ``GET /api/public/world-sessions/{sid}/host`` is host-secret authenticated
1484
- and answers ``{hostUrl}``; a 404 is a closed or unknown session. Answers are
1485
- held briefly so a busy session does not cost one platform call per request.
1486
- Our own address answers None: a session we do not hold but the platform
1487
- says is ours is gone, not one hop away."""
1488
- cache: dict[str, tuple[float, str | None]] = {}
1489
- lock = threading.Lock()
1490
- base = platform_url.rstrip("/")
1491
-
1492
- def lookup(sid: str, *, fresh: bool = False) -> str | None:
1493
- now = time.time()
1494
- if not fresh:
1495
- with lock:
1496
- hit = cache.get(sid)
1497
- if hit and hit[0] > now:
1498
- return hit[1]
1499
- owner: str | None = None
1500
- req = urllib.request.Request(
1501
- f"{base}/api/public/world-sessions/{urllib.parse.quote(sid, safe='')}/host",
1502
- headers={"X-Host-Secret": secret or ""},
1503
- )
1504
- try:
1505
- with urllib.request.urlopen(req, timeout=5) as r:
1506
- answer = json.loads(r.read() or b"{}")
1507
- url = answer.get("hostUrl") if isinstance(answer, dict) else None
1508
- owner = url if isinstance(url, str) and url else None
1509
- except urllib.error.HTTPError as e:
1510
- if e.code != 404:
1511
- raise
1512
- if owner and self_url and owner.rstrip("/") == self_url.rstrip("/"):
1513
- owner = None
1514
- with lock:
1515
- cache[sid] = (now + (OWNER_CACHE_S if owner else OWNER_MISS_S), owner)
1516
- return owner
1517
-
1518
- return lookup
1519
-
1520
-
1521
- class OwnerUnreachable(Exception):
1522
- """The owner a request was proxied to did not answer: no connection, or
1523
- none in time. Not an answer of any status."""
1524
-
1525
-
1526
113
  class BodyRefused(Exception):
1527
114
  """A request body the host will not read: the answer's status and error."""
1528
115
 
@@ -1541,25 +128,27 @@ def make_host_handler(
1541
128
  max_body_mb: int = DEFAULT_MAX_BODY_MB,
1542
129
  secure_cookies: bool = True,
1543
130
  ):
1544
- """The request handler class for one host. ``secret`` is required: the
1545
- control plane is the only trust boundary, so a host with no secret has no
1546
- control plane at all rather than an open one. ``secure_cookies`` is off
131
+ """The request handler class for one host; its control plane is :class:`ControlPlane`, which
132
+ refuses an empty ``secret`` or ``worlds_domain``. ``secure_cookies`` is off
1547
133
  only for a host whose public surface is reached over plain http (a dev
1548
134
  host); it is the deployment's word, not the request's, since a client
1549
135
  writes the forwarding headers a request arrives with."""
1550
- if not secret:
1551
- raise ValueError("the world host needs a control-plane secret (WORLDS_HOST_SECRET)")
136
+ control = ControlPlane(host, secret, HOSTED_OPERATIONS, worlds_domain)
1552
137
  if max_body_mb <= 0:
1553
138
  raise ValueError(f"the request body cap must be positive (WORLDS_HOST_MAX_BODY_MB={max_body_mb})")
1554
139
  max_body_bytes = max_body_mb * (1 << 20)
1555
140
 
1556
- class Handler(BaseHTTPRequestHandler):
141
+ class Handler(OwnerProxy):
1557
142
  server_version = "gateway-world-host/1"
1558
-
1559
- def log_message(self, format: str, *args: object) -> None:
1560
- pass
143
+ world_host = host
1561
144
 
1562
145
  def _read(self) -> bytes:
146
+ """The request's body, read once: a request sent on after its session moved carries it."""
147
+ if self._body is None:
148
+ self._body = self._read_body()
149
+ return self._body
150
+
151
+ def _read_body(self) -> bytes:
1563
152
  declared = self.headers.get("Content-Length")
1564
153
  if declared is None:
1565
154
  return b""
@@ -1573,144 +162,64 @@ def make_host_handler(
1573
162
  raise BodyRefused(413, f"request body over {max_body_mb} MiB")
1574
163
  return self.rfile.read(length) if length else b""
1575
164
 
1576
- def _reply(self, status: int, body: Any, extra: dict[str, str] | None = None) -> None:
1577
- send_response(self, status, body, extra, "application/json")
165
+ def _reply(self, status: int, body: Any, extra: dict[str, str] | None = None, errors: str = JSON_TYPE) -> None:
166
+ try:
167
+ send_response(self, status, body, extra, errors)
168
+ except ConnectionError:
169
+ self.close_connection = True
1578
170
 
1579
171
  def _headers(self) -> dict[str, str]:
1580
172
  return {k.lower(): v for k, v in self.headers.items()}
1581
173
 
1582
- def do_GET(self):
1583
- self._dispatch("GET")
1584
-
1585
- def do_POST(self):
1586
- self._dispatch("POST")
1587
-
1588
- def do_PUT(self):
1589
- self._dispatch("PUT")
1590
-
1591
- def do_PATCH(self):
1592
- self._dispatch("PATCH")
1593
-
1594
- def do_DELETE(self):
1595
- self._dispatch("DELETE")
1596
-
1597
174
  def _dispatch(self, method: str) -> None:
175
+ self._body: bytes | None = None
1598
176
  url = urllib.parse.urlsplit(self.path)
1599
- # The public side is whatever arrives under the worlds domain. It
1600
- # gets sessions and nothing about the host: the probes' answer is
1601
- # a bare ok, the counters and the rest live on the pod address
1602
- # (prometheus) or behind the control-plane secret (/__host/stats).
1603
- if url.path == "/healthz":
177
+ if url.path == "/healthz" and control.internal(self.path, self.headers):
1604
178
  return self._reply(200, {"ok": True})
1605
- if url.path == "/metrics" and not public_host(self.headers.get("Host"), worlds_domain):
179
+ if url.path == "/metrics" and control.serves(self.path, self.headers):
1606
180
  payload = host.metrics_text().encode("utf-8")
1607
181
  return self._relay(
1608
182
  200, [("Content-Type", "text/plain; version=0.0.4; charset=utf-8")], payload, drop=FRAMING_HEADERS
1609
183
  )
1610
184
  try:
1611
- if url.path.startswith("/__host/"):
185
+ if url.path.startswith("/__host/") and control.serves(self.path, self.headers):
1612
186
  return self._control(method, url)
1613
187
  return self._data(method, url)
188
+ except SessionMoved as moved:
189
+ headers = self._headers()
190
+ if hops(headers) >= MAX_HOPS:
191
+ return self._reply(404, CLOSED_SESSION)
192
+ return self._proxy(moved.session_id, moved.owner, owner_lookup, method, headers)
1614
193
  except BodyRefused as refused:
1615
- # The body was not read, so the connection cannot carry another
1616
- # request: answer and hang up.
194
+ # The body was not read, so the connection cannot carry another request: answer and hang up.
1617
195
  self.close_connection = True
1618
196
  return self._reply(refused.status, {"error": refused.error}, {"Connection": "close"})
197
+ except Exception as error:
198
+ log_request_failure(error, method, url.path)
199
+ return self._reply(500, request_failure_body(error))
1619
200
 
1620
201
  def _control(self, method: str, url) -> None:
1621
- if not hmac.compare_digest(self.headers.get("X-Host-Secret") or "", secret):
202
+ if not control.admits(self.headers.get("X-Host-Secret")):
1622
203
  return self._reply(403, {"error": "forbidden"})
1623
- raw = self._read()
1624
- try:
1625
- body = json.loads(raw) if raw else {}
1626
- except ValueError as error:
1627
- return self._reply(400, {"error": "malformed json", "detail": str(error)})
1628
- if not isinstance(body, dict):
1629
- return self._reply(400, {"error": "malformed json", "detail": "the body must be a json object"})
1630
- parts = url.path.strip("/").split("/") # __host, sessions, [sid], [verb]
1631
- try:
1632
- if method == "GET" and parts == ["__host", "stats"]:
1633
- return self._reply(200, host.stats())
1634
- if method == "POST" and parts == ["__host", "drain"]:
1635
- host.drain()
1636
- return self._reply(200, {"draining": True, "sessions": host.session_count()})
1637
- if method == "GET" and parts == ["__host", "versions"]:
1638
- return self._reply(200, {"versions": host.cache.version_ids()})
1639
- if method == "POST" and parts == ["__host", "versions"]:
1640
- return self._reply(
1641
- 200,
1642
- host.warm(body["versionId"], body["bundleUrl"], body.get("snapshot")),
1643
- )
1644
- if method == "POST" and parts == ["__host", "sessions"]:
1645
- return self._reply(
1646
- 200,
1647
- host.open(
1648
- session_id=body["sessionId"],
1649
- version_id=body["versionId"],
1650
- bundle_url=body["bundleUrl"],
1651
- task=body.get("task"),
1652
- token=body.get("token"),
1653
- snapshot=body.get("snapshot"),
1654
- browser=body.get("browser"),
1655
- restore=body.get("restore"),
1656
- task_spec=body.get("taskSpec"),
1657
- dependencies=body.get("dependencies"),
1658
- ),
1659
- )
1660
- if len(parts) >= 3 and parts[:2] == ["__host", "sessions"]:
1661
- sid = parts[2]
1662
- verb = parts[3] if len(parts) > 3 else None
1663
- if method == "PUT" and verb == "browser" and len(parts) == 4:
1664
- return self._reply(200, host.set_browser(sid, body.get("browser")))
1665
- if method == "POST" and verb == "export" and len(parts) == 5:
1666
- url = body.get("url")
1667
- if not isinstance(url, str) or not url.startswith(("https://", "http://")):
1668
- return self._reply(400, {"error": "url must be the presigned upload url"})
1669
- return self._reply(200, host.upload_export(sid, parts[4], url))
1670
- row = session_verb(method, verb)
1671
- if row is None or len(parts) > 4:
1672
- return self._reply(
1673
- 404, unknown_verb(method, verb, hosted=True, prefix=f"/__host/sessions/{sid}/")
1674
- )
1675
- return self._reply(200, HOSTED_OPERATIONS[row.route](host, sid, **row.read(body)))
1676
- return self._reply(404, {"error": "unknown control route"})
1677
- except ContractError as error:
1678
- code = error.detail.get("code", "host")
1679
- # A host that will not take the session is not a bad request:
1680
- # the platform places it on another replica. A session this
1681
- # host does not hold is the one answer the platform may read
1682
- # as "lost"; every other refusal is about the request.
1683
- if code in ("draining", "at_capacity"):
1684
- status = 503
1685
- elif code == UNKNOWN_SESSION:
1686
- status = 404
1687
- else:
1688
- status = 400
1689
- return self._reply(
1690
- status, {"error": code, "detail": str(error), "message": str(error.detail.get("message", ""))}
1691
- )
1692
- except KeyError as error:
1693
- return self._reply(400, {"error": "missing field", "field": str(error)})
1694
- except WorkerGone as error:
1695
- return self._reply(502, {"error": "worker exited", "detail": str(error)})
204
+ return self._reply(*control.answer(method, url.path, self._read()))
1696
205
 
1697
206
  def _data(self, method: str, url) -> None:
1698
207
  headers = self._headers()
1699
208
  named = session_from_host(headers.get("host"))
1700
209
  if named is None:
1701
- return self._reply(404, {"error": "no session named by Host"})
1702
- sid, surface = named
210
+ return self._reply(404, NO_SESSION_NAMED)
211
+ sid, surface = named.session_id, named.surface
1703
212
  if not host.owns(sid):
1704
- # Another host owns it (or it is closed). Proxy one hop when the
1705
- # platform can point us at the owner; a request that already
1706
- # hopped once is not sent on again.
1707
- hopped = HOP_HEADER in headers
1708
- owner = owner_lookup(sid) if owner_lookup and not hopped else None
1709
- if owner and owner_lookup:
213
+ looked_up = owner_lookup if owner_lookup and not hops(headers) else None
214
+ owner = host.forwarding.owner(sid) or (looked_up(sid) if looked_up else None)
215
+ if owner and hops(headers) < MAX_HOPS:
1710
216
  host.tally("requests.proxied")
1711
217
  return self._proxy(sid, owner, owner_lookup, method, headers)
1712
- return self._reply(404, {"error": "unknown or closed session"})
218
+ return self._reply(404, CLOSED_SESSION)
219
+ if not host.reached_by(named):
220
+ return self._reply(404, NO_SESSION_NAMED)
1713
221
  host.tally(f"requests.{surface}")
222
+ opened = named.capability is not None
1714
223
  try:
1715
224
  with host.using(sid) as session:
1716
225
  if session.spec is None:
@@ -1720,85 +229,119 @@ def make_host_handler(
1720
229
  if surface == "browser":
1721
230
  self._serve_browser(session, url, headers)
1722
231
  elif surface == "ui":
1723
- self._serve_ui(session, method, url, headers)
232
+ self._serve_ui(session, method, url, headers, opened)
1724
233
  else:
1725
- self._serve(session, method, url, headers)
1726
- except ContractError:
1727
- # Closed between the ownership check and the lookup.
1728
- self._reply(404, {"error": "unknown or closed session"})
1729
-
1730
- def _serve(self, session: Session, method: str, url, headers: dict[str, str]) -> None:
1731
- admitted = admits(session, url.path, headers)
1732
- if admitted is None:
1733
- return self._reply(401, {"error": "invalid session token"})
234
+ self._serve(session, method, url, headers, opened)
235
+ except ContractError as error:
236
+ owner = host.forwarding.owner(sid) if error.detail.get("code") == UNKNOWN_SESSION else None
237
+ if owner is not None:
238
+ raise SessionMoved(sid, owner) from error
239
+ self._refused(error)
240
+
241
+ def _refused(self, error: ContractError) -> None:
242
+ if error.detail.get("code") == UNKNOWN_SESSION:
243
+ return self._reply(404, CLOSED_SESSION)
244
+ self._reply(*refusal(error))
245
+
246
+ def _serve(self, session: Session, method: str, url, headers: dict[str, str], opened: bool) -> None:
247
+ preflight = session.preflight(method, headers)
248
+ if preflight is not None:
249
+ return self._reply(*preflight)
250
+ assert session.door is not None, "a session with an http surface has a door"
251
+ door = session.door.opened() if opened else session.door
252
+ entry = door.session_entry(method, url.path, url.query, headers)
253
+ if entry is None:
254
+ status, body, errors = door.session_refusal
255
+ return self._reply(status, body, None, errors)
256
+ if door.by_platform_bearer(entry, headers):
257
+ host.tally("requests.platform_bearer")
258
+ headers = door.with_platform_header(headers)
1734
259
  if session.worker is not None:
1735
260
  return self._on_worker(session, method, headers)
1736
- self._serve_api(session, method, self.path, headers, admitted, Via.API)
261
+ self._serve_api(session, method, self.path, headers, entry, Via.API, opened)
1737
262
 
1738
263
  def _on_worker(self, session: Session, method: str, headers: dict[str, str]) -> None:
1739
264
  """Relay the request as it came, Host and path unchanged, to the
1740
- session's version worker: it runs the same gate and picks the same
1741
- door (api or ui) on its own copy of the session."""
1742
- assert session.worker is not None
1743
- try:
1744
- status, answer_headers, payload = session.worker.request(method, self.path, headers, self._read())
1745
- except WorkerGone as error:
1746
- host.lose_worker(session.worker)
1747
- return self._reply(502, {"error": "worker exited", "detail": str(error)})
265
+ session's version worker, once it is admitted on the version: the
266
+ worker runs the same gate and picks the same door (api or ui) on its
267
+ own copy of the session."""
268
+ assert host.placement is not None
269
+ raw = self._read()
270
+ with host.admitted(session, HOP_HEADER in headers):
271
+ status, answer_headers, payload = host.placement.relay(session, method, self.path, headers, raw)
1748
272
  return self._relay(status, answer_headers, payload, drop=FRAMING_HEADERS)
1749
273
 
1750
274
  def _serve_api(
1751
- self, session: Session, method: str, target: str, headers: dict[str, str], admitted: str, via: Via
275
+ self,
276
+ session: Session,
277
+ method: str,
278
+ target: str,
279
+ headers: dict[str, str],
280
+ entry: Entry,
281
+ via: Via,
282
+ opened: bool = False,
1752
283
  ) -> None:
1753
284
  """Answer ``target`` (a path plus query) from the session's world in
1754
285
  this process, once the gate has admitted it; the call is logged as
1755
- having come in by ``via``, the door this host took it in by."""
1756
- assert session.app is not None
286
+ having come in by ``via``, the door this host took it in by. A caller
287
+ the platform admitted is presented to the world with the session's
288
+ credential the vendor's way, and the platform's own header never
289
+ reaches it, so the world's own check and its handlers see what the
290
+ vendor's client would send. A caller on an open session's capability
291
+ host is presented nothing: world code never hands the session token to
292
+ someone who holds only the capability. A link the world signs points at
293
+ the session's own api host, where its signature alone admits it."""
294
+ assert session.app is not None and session.door is not None
1757
295
  raw = self._read()
1758
- world_headers = headers
1759
- if admitted == "session":
1760
- # The host's token gate is the access control; present the world
1761
- # its OWN expected credential so the emulated vendor auth passes.
1762
- credential_headers, credential_query = world_credential(session)
1763
- world_headers = {**headers, **credential_headers}
296
+ world_headers = {name: value for name, value in headers.items() if name != PLATFORM_HEADER}
297
+ if entry is Entry.PLATFORM and not opened:
298
+ credential_headers, credential_query = session.door.present()
299
+ world_headers = {**world_headers, **credential_headers}
1764
300
  target = with_query(target, credential_query)
1765
- default_host = f"w-{session.session_id}-api.{worlds_domain}"
1766
- status, body, extra = session.app.handle(method, target, world_headers, raw, default_host, via=via)
1767
- self._reply(status, body, extra)
301
+ default_host = session.api_host(worlds_domain, opened)
302
+ link_host = session.api_host(worlds_domain, opened=False)
303
+ with host.admitted(session, HOP_HEADER in headers):
304
+ answer = session.app.handle(
305
+ method, target, world_headers, raw, default_host, via=via, opened=opened, link_host=link_host
306
+ )
307
+ self._reply(*answer, session.app.settings.error_content_type)
1768
308
 
1769
- def _serve_ui(self, session: Session, method: str, url, headers: dict[str, str]) -> None:
309
+ def _serve_ui(self, session: Session, method: str, url, headers: dict[str, str], opened: bool) -> None:
1770
310
  ui = session.ui
1771
311
  if ui is None:
1772
312
  return self._reply(404, {"error": "this world has no static ui"})
1773
- token = session.token
313
+ token = None if opened else session.token
1774
314
  if token is not None and matches(query_token(url.query), token):
1775
315
  return self._trade_token_for_cookie(url, token)
1776
- if token is not None and not (matches(cookie_token(headers), token) or matches(bearer_of(headers), token)):
316
+ if token is not None and not (
317
+ matches(cookie_token(headers), token) or matches(_BEARER.read(headers, {}), token)
318
+ ):
1777
319
  return self._reply(401, {"error": "invalid session token"})
1778
320
  route = api_target(ui, url.path)
1779
321
  if route is None:
1780
- return self._serve_static(ui, method, url)
322
+ return self._serve_static(ui, method, url, opened)
1781
323
  if cross_site(headers, lambda origin: urllib.parse.urlsplit(origin).netloc == headers.get("host")):
1782
324
  return self._reply(403, {"error": "the ui's api prefix answers its own page, not another site"})
1783
325
  if session.worker is not None:
1784
326
  return self._on_worker(session, method, headers)
1785
327
  target = urllib.parse.urlunsplit(("", "", route, url.query, ""))
1786
328
  api_headers = {k: v for k, v in headers.items() if k not in ("cookie", "authorization")}
1787
- api_headers["host"] = f"w-{session.session_id}-api.{worlds_domain}"
1788
- if token is not None:
1789
- api_headers["authorization"] = f"Bearer {token}"
1790
- return self._serve_api(session, method, target, api_headers, "session", Via.UI)
329
+ api_headers["host"] = session.api_host(worlds_domain, opened)
330
+ return self._serve_api(session, method, target, api_headers, Entry.PLATFORM, Via.UI, opened)
1791
331
 
1792
332
  def _trade_token_for_cookie(self, url, token: str) -> None:
1793
333
  landing = without_query_token(url)
1794
334
  cookie = session_cookie(token, secure=secure_cookies, max_age_s=host.lease_s)
1795
335
  self._reply(303, {"location": landing}, {"Location": landing, "Set-Cookie": cookie})
1796
336
 
1797
- def _serve_static(self, ui: UiSurface, method: str, url) -> None:
337
+ def _serve_static(self, ui: UiSurface, method: str, url, opened: bool) -> None:
338
+ """A file of the ui; on a capability host with no ``Referer``, so a page that
339
+ loads another site's resource never hands it the capability."""
1798
340
  if method != "GET":
1799
341
  return self._reply(405, {"error": "the static ui answers GET only"})
1800
342
  status, static_headers, payload = static_file(ui, url.path)
1801
- self._relay(status, static_headers.items(), payload, drop=FRAMING_HEADERS)
343
+ sealed = {**static_headers, **(NO_REFERRER if opened else {})}
344
+ self._relay(status, sealed.items(), payload, drop=FRAMING_HEADERS)
1802
345
 
1803
346
  def _serve_browser(self, session: Session, url, headers: dict[str, str]) -> None:
1804
347
  """``/cdp/<session token>`` and ``/live/<grant>`` on the session's
@@ -1816,8 +359,7 @@ def make_host_handler(
1816
359
  if kind == "cdp":
1817
360
  if not matches(credential, token):
1818
361
  return self._reply(401, {"error": "invalid session token"})
1819
- # The relay decodes the path segment, so a token with '/' or
1820
- # '=' (openssl rand -base64) must travel percent-encoded.
362
+ # The relay decodes the path segment: a token with '/' or '=' must travel percent-encoded.
1821
363
  upstream_path = f"/cdp/{urllib.parse.quote(relay.token, safe='')}"
1822
364
  else:
1823
365
  scope = verify_live_grant(credential, token)
@@ -1832,146 +374,17 @@ def make_host_handler(
1832
374
  except OSError as error:
1833
375
  self._reply(502, {"error": "browser unreachable", "detail": str(error)})
1834
376
 
1835
- def _proxy(self, sid: str, owner: str, lookup: OwnerLookup, method: str, headers: dict[str, str]) -> None:
1836
- """Send the request on to ``owner``. An owner that does not answer
1837
- at all may be a host that died with the session since recovered
1838
- elsewhere, so the platform is asked once more, uncached, and the
1839
- request goes to the owner it names now; only then is it a 502.
1840
- The retry stays one hop (the hop header goes with it)."""
1841
- websocket = headers.get("upgrade", "").lower() == "websocket"
1842
- raw = b"" if websocket else self._read()
1843
- try:
1844
- return self._forward(owner, method, headers, raw, websocket)
1845
- except OwnerUnreachable as stale:
1846
- current = lookup(sid, fresh=True)
1847
- if not current or current.rstrip("/") == owner.rstrip("/"):
1848
- return self._reply(502, {"error": "owner unreachable", "detail": str(stale)})
1849
- host.tally("requests.rerouted")
1850
- try:
1851
- self._forward(current, method, headers, raw, websocket)
1852
- except OwnerUnreachable as error:
1853
- self._reply(502, {"error": "owner unreachable", "detail": str(error)})
1854
-
1855
- def _forward(self, owner: str, method: str, headers: dict[str, str], raw: bytes, websocket: bool) -> None:
1856
- """One attempt at ``owner``; raises :class:`OwnerUnreachable` when it
1857
- did not answer, before anything has been written to the client."""
1858
- if websocket:
1859
- # A browser session on another host: the owner runs the
1860
- # credential check; this hop only carries bytes. ``tunnel``
1861
- # fails before the 101 or not at all.
1862
- owner_ws = "ws://" + owner.split("://", 1)[-1].rstrip("/")
1863
- self.close_connection = True
1864
- try:
1865
- tunnel(self.connection, owner_ws, self.path, {**headers, HOP_HEADER: "1"}, host=headers.get("host"))
1866
- except OSError as error:
1867
- raise OwnerUnreachable(str(error)) from error
1868
- return
1869
- target = f"{owner.rstrip('/')}{self.path}"
1870
- forward = {k: v for k, v in headers.items() if k not in ("host", "content-length")}
1871
- forward["host"] = headers.get("host", "")
1872
- forward[HOP_HEADER] = "1"
1873
- req = urllib.request.Request(target, data=raw or None, method=method, headers=forward)
1874
- try:
1875
- with _PROXY_OPENER.open(req, timeout=60) as r:
1876
- self._relay(r.status, r.headers.items(), r.read(), drop=PROXIED_DROP_HEADERS)
1877
- except urllib.error.HTTPError as e:
1878
- self._relay(e.code, e.headers.items(), e.read(), drop=PROXIED_DROP_HEADERS)
1879
- except (OSError, http.client.HTTPException) as error:
1880
- # URLError is an OSError: no connection, or none in time.
1881
- raise OwnerUnreachable(str(error)) from error
1882
-
1883
- def _relay(
1884
- self,
1885
- status: int,
1886
- headers: Iterable[tuple[str, str]],
1887
- payload: bytes,
1888
- *,
1889
- drop: frozenset[str],
1890
- ) -> None:
1891
- """Write another process's answer through as it came: same status,
1892
- same body bytes, every header the world set except ``drop`` (the
1893
- framing, which is this connection's, and from an owner the
1894
- hop-by-hop headers too). The world's contract, its ``page_headers``
1895
- included, reads the same from every replica."""
1896
- self.send_response(status)
1897
- content_type = None
1898
- for name, value in headers:
1899
- lower = name.lower()
1900
- if lower == "content-type":
1901
- content_type = value
1902
- elif lower not in drop:
1903
- self.send_header(name, value)
1904
- self.send_header("Content-Type", content_type or "application/json")
1905
- self.send_header("Content-Length", str(len(payload)))
1906
- self.end_headers()
1907
- self.wfile.write(payload)
1908
-
1909
377
  return Handler
1910
378
 
1911
379
 
1912
- def start_lease_sweeper(world_host: WorldHost, interval_s: float = LEASE_SWEEP_S) -> threading.Event:
1913
- """Expire leases every ``interval_s`` until the returned event is set."""
1914
- stop = threading.Event()
1915
-
1916
- def sweep() -> None:
1917
- while not stop.wait(interval_s):
1918
- for sid in world_host.expire_leases():
1919
- print(
1920
- f"world host: lease expired, closed session {sid}",
1921
- file=sys.stderr,
1922
- flush=True,
1923
- )
1924
- for version_id in world_host.reap_workers():
1925
- print(
1926
- f"world host: stopped idle worker for version {version_id}",
1927
- file=sys.stderr,
1928
- flush=True,
1929
- )
1930
-
1931
- threading.Thread(target=sweep, name="world-host-leases", daemon=True).start()
1932
- return stop
1933
-
1934
-
1935
- def heartbeat_once(world_host: WorldHost, platform_url: str, secret: str, timeout_s: float = 5.0) -> bool:
1936
- """Tell the platform what this host is and holds (:meth:`WorldHost.stats`).
1937
- The platform's registry is what places new sessions: a host it has not
1938
- heard from lately gets none. Returns whether the platform took it."""
1939
- body = json.dumps(world_host.stats()).encode("utf-8")
1940
- req = urllib.request.Request(
1941
- f"{platform_url.rstrip('/')}/api/public/world-hosts",
1942
- data=body,
1943
- method="POST",
1944
- headers={"content-type": "application/json", "x-host-secret": secret},
1945
- )
1946
- try:
1947
- with urllib.request.urlopen(req, timeout=timeout_s) as r:
1948
- return 200 <= r.status < 300
1949
- except (urllib.error.URLError, OSError, ValueError) as error:
1950
- print(f"world host: heartbeat failed: {error}", file=sys.stderr, flush=True)
1951
- return False
1952
-
1953
-
1954
- def start_heartbeat(
1955
- world_host: WorldHost, platform_url: str, secret: str, interval_s: float = HEARTBEAT_S
1956
- ) -> threading.Event:
1957
- """Heartbeat every ``interval_s`` until the returned event is set; the
1958
- first beat goes out at once so a fresh replica is placeable immediately,
1959
- and a last one goes out on stop so the registry sees it draining."""
1960
- stop = threading.Event()
1961
-
1962
- def beat() -> None:
1963
- heartbeat_once(world_host, platform_url, secret)
1964
- while not stop.wait(interval_s):
1965
- heartbeat_once(world_host, platform_url, secret)
1966
- heartbeat_once(world_host, platform_url, secret)
1967
-
1968
- threading.Thread(target=beat, name="world-host-heartbeat", daemon=True).start()
1969
- return stop
1970
-
1971
-
1972
380
  class HostServer(ThreadingHTTPServer):
1973
381
  """The listening socket plus what shutdown has to stop: the lease sweeper,
1974
- the heartbeat and the version workers."""
382
+ the heartbeat and the version workers. Its backlog holds a burst of
383
+ connections; their requests wait their turn in :class:`Admission`. It
384
+ counts the requests it is answering, so a drain stops only once the
385
+ requests sent on to the hosts its sessions moved to are answered."""
386
+
387
+ request_queue_size = 1024
1975
388
 
1976
389
  def __init__(
1977
390
  self,
@@ -1982,81 +395,104 @@ class HostServer(ThreadingHTTPServer):
1982
395
  heartbeat: tuple[str, str] | None = None,
1983
396
  heartbeat_s: float = HEARTBEAT_S,
1984
397
  drain_s: float = DEFAULT_DRAIN_S,
398
+ ledger: LedgerFlusher | None = None,
399
+ handoff: HandoffDoor | None = None,
1985
400
  ):
1986
401
  super().__init__(address, handler)
1987
402
  self.world_host = world_host
1988
403
  self.lease_sweeper = start_lease_sweeper(world_host)
1989
- #: ``(platform url, secret)`` to report to; None runs unregistered (one
1990
- #: replica behind WORLDS_HOST_URL needs no registry).
1991
- self.heartbeat = start_heartbeat(world_host, *heartbeat, interval_s=heartbeat_s) if heartbeat else None
1992
- self.drain_s = drain_s
404
+ self.heartbeat = Heartbeat(world_host, *heartbeat, interval_s=heartbeat_s).start() if heartbeat else None
405
+ world_host.ledger = ledger.start() if ledger else NoLedger()
406
+ self.handoff = DrainHandoff(world_host, handoff, drain_s)
407
+ self._answering = 0
408
+ self._draining = False
409
+ self._lock = threading.Lock()
410
+
411
+ def process_request_thread(self, request, client_address) -> None:
412
+ with self._lock:
413
+ self._answering += 1
414
+ try:
415
+ super().process_request_thread(request, client_address)
416
+ finally:
417
+ with self._lock:
418
+ self._answering -= 1
419
+
420
+ def idle(self) -> bool:
421
+ with self._lock:
422
+ return self._answering == 0
1993
423
 
1994
424
  def drain(self) -> None:
1995
- """SIGTERM: refuse new sessions, announce it, serve what is here until
1996
- it closes or the grace runs out, then stop. Runs off the signal thread
1997
- so ``serve_forever`` keeps answering meanwhile."""
425
+ """SIGTERM: refuse new sessions, tell the platform at once and keep beating
426
+ (a draining host is alive), hand every session off (:class:`DrainHandoff`),
427
+ then stop. Runs off the signal thread so ``serve_forever`` keeps answering
428
+ meanwhile; a second call does nothing."""
429
+ with self._lock:
430
+ if self._draining:
431
+ return
432
+ self._draining = True
1998
433
  self.world_host.drain()
1999
434
  if self.heartbeat is not None:
2000
- self.heartbeat.set() # its last beat carries ``draining: true``
435
+ self.heartbeat.wake()
436
+ threading.Thread(target=self._hand_off_then_stop, name="world-host-drain", daemon=True).start()
2001
437
 
2002
- def wait_then_stop() -> None:
2003
- deadline = time.time() + self.drain_s
2004
- while self.world_host.session_count() > 0 and time.time() < deadline:
2005
- time.sleep(0.5)
438
+ def _hand_off_then_stop(self) -> None:
439
+ try:
440
+ self.handoff.run(self.idle)
441
+ finally:
2006
442
  self.shutdown()
2007
443
 
2008
- threading.Thread(target=wait_then_stop, name="world-host-drain", daemon=True).start()
2009
-
2010
444
 
2011
445
  def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0") -> HostServer:
2012
446
  """The host as deployed, configured from the environment:
2013
447
 
2014
- ``WORLDS_HOST_SECRET`` gates the control plane; ``WORLDS_INGRESS_DOMAIN`` is
2015
- the domain the session hostnames live under; ``WORLDS_HOST_ADVERTISE_URL`` is
2016
- this replica's own address (``http://<pod ip>:<port>``), returned on open so
2017
- the platform pins the session to it; ``WORLDS_PLATFORM_URL`` lets a
2018
- non-owner replica ask the platform who owns a session and proxy one hop;
2019
- ``WORLDS_HOST_LEASE_S`` is how long an untouched session lives;
2020
- ``WORLDS_HOST_MAX_SESSIONS`` is how many it takes before answering 503;
2021
- ``WORLDS_HOST_HEARTBEAT_S`` is how often it reports to the platform (with
2022
- ``WORLDS_PLATFORM_URL`` and an advertised address, else no heartbeat);
2023
- ``WORLDS_HOST_DRAIN_S`` is how long a SIGTERM waits for sessions to close;
2024
- ``WORLDS_WORKER_MEMORY_MB`` caps each version worker's address space,
2025
- ``WORLDS_WORKER_BUDGET_MB`` is the memory all of them together are held to
2026
- and ``WORLDS_WORKER_FOOTPRINT_MB`` what each is counted at against it
2027
- (unset, its whole address-space cap; their count is the quotient, and one
2028
- more version answers 503 ``at_capacity``),
2029
- ``WORLDS_HOST_OBJECT_STORE`` is where every presigned URL it is handed
2030
- may point (required; ``any`` for a dev host),
2031
- ``WORLDS_HOST_MAX_BUNDLE_MB`` is the largest version bundle the cache
2032
- accepts, ``WORLDS_HOST_MAX_BODY_MB`` the largest request body it reads,
2033
- ``WORLDS_HOST_PLAIN_HTTP=1`` says the public surface is reached over plain
2034
- http (a dev host: the ui cookie is not marked ``Secure``) and
2035
- ``WORLDS_HOST_WORKERS=0`` runs without workers (handler worlds refused)."""
448
+ ``WORLDS_HOST_SECRET`` gates the control plane; ``WORLDS_INGRESS_DOMAIN`` is the domain the session hostnames
449
+ live under (required); ``WORLDS_HOST_ADVERTISE_URL`` is this replica's own address (``http://<pod ip>:<port>``),
450
+ returned on open so the platform pins the session to it; ``WORLDS_PLATFORM_URL`` lets a non-owner replica ask
451
+ the platform who owns a session and proxy one hop, and is where the request ledger goes; ``WORLDS_HOST_LEASE_S``
452
+ is how long an untouched session lives; ``WORLDS_HOST_MAX_SESSIONS`` is how many it takes before answering 503;
453
+ ``WORLDS_HOST_HEARTBEAT_S`` is how often it reports to the platform (with ``WORLDS_PLATFORM_URL`` and an
454
+ advertised address, else no heartbeat), and ``WORLDS_HOST_BUILD`` which pod template it runs, so a session
455
+ handed off goes to the newest one; ``WORLDS_HOST_DRAIN_S`` is how long a SIGTERM's drain may take to hand its
456
+ sessions to other hosts through the platform (with an advertised address) and serve what could not move;
457
+ ``WORLDS_WORKER_MEMORY_MB`` caps each version worker's address space, ``WORLDS_WORKER_BUDGET_MB`` is the memory
458
+ all of them together are held to and ``WORLDS_WORKER_FOOTPRINT_MB`` what each is counted at against it (unset,
459
+ its whole address-space cap; their count is the quotient, and one more version answers 503 ``at_capacity``),
460
+ ``WORLDS_HOST_OBJECT_STORE`` is where every presigned URL it is handed may point (required; ``any`` for a dev
461
+ host), ``WORLDS_HOST_MAX_BUNDLE_MB`` is the largest version bundle the cache accepts,
462
+ ``WORLDS_HOST_MAX_BODY_MB`` the largest request body it reads, ``WORLDS_HOST_LEDGER_SPILL_DIR`` (under the cache
463
+ root) and ``WORLDS_HOST_LEDGER_SPILL_MB`` are where and how much of the request ledger waits on disk for the
464
+ platform, ``WORLDS_HOST_PLAIN_HTTP=1`` says the public surface is reached over plain http (a dev host: the ui
465
+ cookie is not marked ``Secure``) and ``WORLDS_HOST_WORKERS=0`` runs without workers (handler worlds refused)."""
466
+ if not seal_process() and sys.platform.startswith("linux"):
467
+ raise SystemExit("world host: cannot make this process non-dumpable; its workers could read its secret")
2036
468
  secret = os.environ.get("WORLDS_HOST_SECRET") or ""
2037
469
  if not secret:
2038
470
  raise SystemExit("world host: WORLDS_HOST_SECRET is unset; refusing to start without a control-plane secret")
471
+ spill_dir, spill_mb = os.environ.get("WORLDS_HOST_LEDGER_SPILL_DIR"), os.environ.get("WORLDS_HOST_LEDGER_SPILL_MB")
472
+ platform_url = os.environ.get("WORLDS_PLATFORM_URL") or None
2039
473
  try:
2040
474
  object_store = parse_origins(os.environ.get("WORLDS_HOST_OBJECT_STORE"))
2041
- except ContractError as error:
475
+ worlds_domain = ControlPlane.domain_of(os.environ.get("WORLDS_INGRESS_DOMAIN"))
476
+ spill = LedgerSpill.at(Path(cache_root), spill_dir, spill_mb, secret)
477
+ spill = spill.open() if platform_url else spill
478
+ except (ContractError, ValueError, OSError) as error:
2042
479
  raise SystemExit(f"world host: {error}; refusing to start") from error
480
+ # Nothing this process starts inherits them; /proc/<pid>/environ keeps the exec-time copy, closed by the seal.
481
+ os.environ.pop("WORLDS_HOST_SECRET", None)
482
+ os.environ.pop("WORLDS_HOST_OBJECT_STORE", None)
2043
483
  advertise = os.environ.get("WORLDS_HOST_ADVERTISE_URL") or None
2044
- platform_url = os.environ.get("WORLDS_PLATFORM_URL") or None
2045
484
  lease_s = float(os.environ.get("WORLDS_HOST_LEASE_S") or DEFAULT_LEASE_S)
2046
- worlds_domain = os.environ.get("WORLDS_INGRESS_DOMAIN", "worlds.local")
2047
485
  workers = None
486
+ footprint = os.environ.get("WORLDS_WORKER_FOOTPRINT_MB")
2048
487
  if os.environ.get("WORLDS_HOST_WORKERS", "1") != "0":
2049
488
  workers = WorkerPool(
2050
489
  Path(cache_root),
2051
490
  worlds_domain=worlds_domain,
2052
491
  memory_mb=int(os.environ.get("WORLDS_WORKER_MEMORY_MB") or DEFAULT_WORKER_MEMORY_MB),
2053
492
  budget_mb=int(os.environ.get("WORLDS_WORKER_BUDGET_MB") or DEFAULT_WORKER_BUDGET_MB),
2054
- footprint_mb=(
2055
- int(os.environ["WORLDS_WORKER_FOOTPRINT_MB"])
2056
- if os.environ.get("WORLDS_WORKER_FOOTPRINT_MB")
2057
- else DEFAULT_WORKER_FOOTPRINT_MB
2058
- ),
493
+ footprint_mb=int(footprint) if footprint else DEFAULT_WORKER_FOOTPRINT_MB,
2059
494
  )
495
+ owners = PlatformOwners(platform_url, secret, advertise) if platform_url else None
2060
496
  world_host = WorldHost(
2061
497
  Path(cache_root),
2062
498
  lease_s=lease_s,
@@ -2065,22 +501,27 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
2065
501
  max_sessions=int(os.environ.get("WORLDS_HOST_MAX_SESSIONS") or DEFAULT_MAX_SESSIONS),
2066
502
  max_bundle_mb=int(os.environ.get("WORLDS_HOST_MAX_BUNDLE_MB") or DEFAULT_MAX_BUNDLE_MB),
2067
503
  object_store=object_store,
504
+ build=os.environ.get("WORLDS_HOST_BUILD") or None,
505
+ owners=owners,
2068
506
  )
2069
507
  handler = make_host_handler(
2070
508
  world_host,
2071
509
  secret=secret,
2072
510
  worlds_domain=worlds_domain,
2073
- owner_lookup=platform_owner_lookup(platform_url, secret, advertise) if platform_url else None,
511
+ owner_lookup=owners,
2074
512
  max_body_mb=int(os.environ.get("WORLDS_HOST_MAX_BODY_MB") or DEFAULT_MAX_BODY_MB),
2075
513
  secure_cookies=os.environ.get("WORLDS_HOST_PLAIN_HTTP", "0") != "1",
2076
514
  )
515
+ sink = PlatformLedger(platform_url, secret, host_url=advertise) if platform_url else None
2077
516
  return HostServer(
2078
517
  (host_addr, port),
2079
518
  handler,
2080
519
  world_host,
2081
520
  heartbeat=(platform_url, secret) if platform_url and advertise else None,
521
+ ledger=LedgerFlusher(world_host, sink, spill) if sink else None,
2082
522
  heartbeat_s=float(os.environ.get("WORLDS_HOST_HEARTBEAT_S") or HEARTBEAT_S),
2083
523
  drain_s=float(os.environ.get("WORLDS_HOST_DRAIN_S") or DEFAULT_DRAIN_S),
524
+ handoff=PlatformHandoff(platform_url, secret) if platform_url and advertise else None,
2084
525
  )
2085
526
 
2086
527
 
@@ -2093,7 +534,7 @@ def main(argv: list[str] | None = None) -> int:
2093
534
  args = parser.parse_args(argv)
2094
535
  server = serve(args.cache_root, port=args.port)
2095
536
  print(
2096
- f"world host listening on :{args.port}, cache {args.cache_root}",
537
+ f"world host listening on :{server.server_address[1]}, cache {args.cache_root}",
2097
538
  file=sys.stderr,
2098
539
  flush=True,
2099
540
  )
@@ -2101,9 +542,10 @@ def main(argv: list[str] | None = None) -> int:
2101
542
  try:
2102
543
  server.serve_forever()
2103
544
  finally:
545
+ server.world_host.ledger.stop(settle=server.world_host.finishing.settle)
2104
546
  server.lease_sweeper.set()
2105
547
  if server.heartbeat is not None:
2106
- server.heartbeat.set()
548
+ server.heartbeat.stop()
2107
549
  if server.world_host.workers is not None:
2108
550
  server.world_host.workers.stop_all()
2109
551
  server.server_close()