@withgateway/sdk 0.3.3 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +2221 -962
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2212 -953
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +81 -19
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +79 -17
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/index.d.mts +2 -2
- package/dist/index.d.ts +2 -2
- package/dist/index.js +903 -221
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +868 -212
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js +78 -16
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +76 -14
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.js +7 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +7 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +692 -410
- package/dist/worlds/index.d.ts +692 -410
- package/dist/worlds/index.js +740 -197
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +736 -193
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_materialize.py +5 -25
- package/runtime/gatewaysdk/environment/_world.py +58 -65
- package/runtime/gatewaysdk/environment/backing.py +20 -24
- package/runtime/gatewaysdk/environment/core.py +1 -16
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +174 -471
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +230 -18
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +201 -19
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/conform.py +6 -27
- package/runtime/gatewaysdk/environment/schema/connector.py +89 -152
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +14 -13
- package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
- package/runtime/gatewaysdk/environment/schema/database/guard.py +43 -36
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
- package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +25 -4
- package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +31 -116
- package/runtime/gatewaysdk/environment/schema/platform.py +15 -17
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +129 -176
- package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +31 -6
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +13 -13
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +23 -23
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +35 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +149 -29
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
- package/runtime/gatewaysdk/environment/schema/tools_world.py +84 -203
- package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
- package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
- package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +9 -11
- package/runtime/gatewaysdk/environment/world.py +1 -1
- package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/skills/connector-schema-authoring/SKILL.md +31 -6
- package/skills/connector-schema-authoring/connector-worlds.md +129 -7
- package/skills/world-data-ingestion/ROWS.md +12 -8
- package/skills/world-data-ingestion/SKILL.md +13 -13
- package/skills/worlds-getting-started/SKILL.md +23 -23
- package/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
- package/skills/worlds-getting-started/references/gotchas.md +35 -2
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
|
@@ -9,13 +9,13 @@ nothing per session in Kubernetes or the load balancer.
|
|
|
9
9
|
|
|
10
10
|
Two planes on one HTTP server:
|
|
11
11
|
|
|
12
|
-
* the **control plane** (``/__host
|
|
13
|
-
platform's half: open, close, reset, seed, state, grade,
|
|
14
|
-
synchronous — a ``seed`` or ``grade`` completes in the response, not over a
|
|
12
|
+
* the **control plane** (``/__host/*`` on the pod's own address, behind ``X-Host-Secret``;
|
|
13
|
+
:mod:`host_control`) is the platform's half: open, close, reset, seed, state, grade,
|
|
14
|
+
call. It is synchronous — a ``seed`` or ``grade`` completes in the response, not over a
|
|
15
15
|
poll — because the host holds the world in memory.
|
|
16
|
-
* the **data plane** (everything else
|
|
17
|
-
vendor routes, selected by session and checked against
|
|
18
|
-
answered in-process by :class:`WorldApp`.
|
|
16
|
+
* the **data plane** (everything else, every path under the worlds domain included) is the
|
|
17
|
+
end user's half: the world's own vendor routes, selected by session and checked against
|
|
18
|
+
that session's token, answered in-process by :class:`WorldApp`.
|
|
19
19
|
|
|
20
20
|
Every session carries a lease. Control calls and data-plane requests renew it;
|
|
21
21
|
a session the platform stops touching expires on the host and is unlinked, so
|
|
@@ -45,1399 +45,71 @@ guessing.
|
|
|
45
45
|
|
|
46
46
|
from __future__ import annotations
|
|
47
47
|
|
|
48
|
-
import base64
|
|
49
|
-
import hashlib
|
|
50
|
-
import hmac
|
|
51
|
-
import http.client
|
|
52
|
-
import json
|
|
53
48
|
import os
|
|
54
|
-
import re
|
|
55
|
-
import shutil
|
|
56
49
|
import signal
|
|
57
50
|
import sys
|
|
58
|
-
import tarfile
|
|
59
51
|
import threading
|
|
60
|
-
import time
|
|
61
|
-
import urllib.error
|
|
62
52
|
import urllib.parse
|
|
63
|
-
import urllib.request
|
|
64
|
-
import uuid
|
|
65
|
-
from collections import Counter
|
|
66
|
-
from collections.abc import Callable, Iterable, Iterator
|
|
67
|
-
from contextlib import contextmanager
|
|
68
|
-
from dataclasses import dataclass, field
|
|
69
53
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
70
54
|
from pathlib import Path
|
|
71
|
-
from typing import Any
|
|
72
|
-
|
|
73
|
-
from
|
|
74
|
-
from
|
|
75
|
-
from . import
|
|
76
|
-
from .
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
send_response,
|
|
85
|
-
with_query,
|
|
86
|
-
)
|
|
87
|
-
from .compiler import check_bundle
|
|
88
|
-
from .connector import Spec, load_spec
|
|
89
|
-
from .credentials import basic_user
|
|
90
|
-
from .database import WorldDatabase
|
|
91
|
-
from .dependencies import Dependency
|
|
92
|
-
from .handler_errors import carried, log_request_failure, request_failure_body
|
|
93
|
-
from .host_requests import BrowserRelay, DependencyRef, parse_dependency_refs
|
|
55
|
+
from typing import Any
|
|
56
|
+
|
|
57
|
+
from .api import Via, with_query
|
|
58
|
+
from .call_capture import JSON_TYPE, send_response
|
|
59
|
+
from .credentials import BearerAuth
|
|
60
|
+
from .door import PLATFORM_HEADER, Entry
|
|
61
|
+
from .handler_errors import log_request_failure, request_failure_body
|
|
62
|
+
from .host_control import ControlPlane
|
|
63
|
+
from .host_handoff import DrainHandoff, HandoffDoor, PlatformHandoff
|
|
64
|
+
from .host_loops import HEARTBEAT_S, Heartbeat, start_lease_sweeper
|
|
65
|
+
from .host_owners import CLOSED_SESSION, FRAMING_HEADERS, OwnerLookup, PlatformOwners
|
|
66
|
+
from .host_proxy import OwnerProxy
|
|
67
|
+
from .host_session import Session
|
|
94
68
|
from .host_surfaces import (
|
|
95
69
|
HOP_HEADER,
|
|
70
|
+
MAX_HOPS,
|
|
71
|
+
NO_REFERRER,
|
|
72
|
+
NO_SESSION_NAMED,
|
|
96
73
|
UiSurface,
|
|
97
74
|
api_target,
|
|
98
75
|
cookie_token,
|
|
99
76
|
cross_site,
|
|
100
|
-
|
|
77
|
+
hops,
|
|
101
78
|
matches,
|
|
102
79
|
mint_live_grant,
|
|
103
80
|
query_token,
|
|
104
81
|
session_cookie,
|
|
82
|
+
session_from_host,
|
|
105
83
|
static_file,
|
|
106
84
|
tunnel,
|
|
107
85
|
verify_live_grant,
|
|
108
86
|
without_query_token,
|
|
109
87
|
)
|
|
110
|
-
from .
|
|
111
|
-
from .
|
|
112
|
-
from .
|
|
113
|
-
from .
|
|
114
|
-
from .
|
|
115
|
-
from .
|
|
116
|
-
from .validation import ContractError, fail
|
|
117
|
-
from .verbs import SESSION_VERBS, session_verb, unknown_verb
|
|
88
|
+
from .host_transfers import parse_origins
|
|
89
|
+
from .host_versions import DEFAULT_MAX_BUNDLE_MB
|
|
90
|
+
from .host_workers import UNKNOWN_SESSION, refusal
|
|
91
|
+
from .request_ledger import LedgerFlusher, LedgerSpill, NoLedger, PlatformLedger
|
|
92
|
+
from .session_move import SessionMoved
|
|
93
|
+
from .validation import ContractError
|
|
118
94
|
from .workers import (
|
|
119
95
|
DEFAULT_WORKER_BUDGET_MB,
|
|
120
96
|
DEFAULT_WORKER_FOOTPRINT_MB,
|
|
121
97
|
DEFAULT_WORKER_MEMORY_MB,
|
|
122
|
-
Worker,
|
|
123
|
-
WorkerCapacity,
|
|
124
|
-
WorkerGone,
|
|
125
98
|
WorkerPool,
|
|
99
|
+
seal_process,
|
|
126
100
|
)
|
|
101
|
+
from .world_host import DEFAULT_LEASE_S, DEFAULT_MAX_SESSIONS, HOSTED_OPERATIONS, WorldHost
|
|
127
102
|
|
|
128
|
-
|
|
129
|
-
WORLD_SPEC = "schema/world.json"
|
|
130
|
-
DEFAULT_LEASE_S = 3600.0
|
|
131
|
-
LEASE_SWEEP_S = 30.0
|
|
132
|
-
OWNER_CACHE_S = 60.0
|
|
133
|
-
#: Long enough that a client hammering a closed session costs the platform one call at a time,
|
|
134
|
-
#: short enough that a session recovered onto another host is reached at once.
|
|
135
|
-
OWNER_MISS_S = 2.0
|
|
136
|
-
DEFAULT_MAX_SESSIONS = 1500
|
|
137
|
-
#: The platform treats exactly this code (with a 404) as "the host lost the session".
|
|
138
|
-
UNKNOWN_SESSION = "unknown_session"
|
|
139
|
-
HEARTBEAT_S = 10.0
|
|
140
|
-
DEFAULT_DRAIN_S = 45.0
|
|
141
|
-
#: Only the framing is dropped: every header the world set must pass through a relay.
|
|
142
|
-
FRAMING_HEADERS = frozenset({"content-length", "connection", "transfer-encoding", "server", "date"})
|
|
143
|
-
HOP_BY_HOP_HEADERS = frozenset(
|
|
144
|
-
{"keep-alive", "proxy-authenticate", "proxy-authorization", "te", "trailer", "upgrade", "proxy-connection"}
|
|
145
|
-
)
|
|
146
|
-
PROXIED_DROP_HEADERS = FRAMING_HEADERS | HOP_BY_HOP_HEADERS
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
150
|
-
"""An owner's redirect is the client's to follow, not the proxy's: the ui
|
|
151
|
-
token exchange answers 303 with the session cookie, and following it here
|
|
152
|
-
would drop the cookie and land on the owner's 401."""
|
|
103
|
+
DEFAULT_DRAIN_S = 240.0
|
|
153
104
|
|
|
154
|
-
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
155
|
-
return None
|
|
156
105
|
|
|
157
|
-
|
|
158
|
-
_PROXY_OPENER = urllib.request.build_opener(_NoRedirect)
|
|
106
|
+
_BEARER = BearerAuth()
|
|
159
107
|
#: Short-lived: the viewer's own grant was already checked, this one only has to survive the handshake.
|
|
160
108
|
LIVE_RELAY_GRANT_MS = 60_000
|
|
161
|
-
TRANSFER_TIMEOUT_S = 300.0
|
|
162
|
-
#: Bounded because the cache is one shared disk and a build's memory is the supervisor's.
|
|
163
|
-
DEFAULT_MAX_BUNDLE_MB = 2048
|
|
164
|
-
MAX_BUNDLE_MEMBERS = 200_000
|
|
165
109
|
#: A body is held in memory whole, in the supervisor shared by every session: a client must not size that allocation.
|
|
166
110
|
DEFAULT_MAX_BODY_MB = 32
|
|
167
111
|
|
|
168
112
|
|
|
169
|
-
OBJECT_STORE_ANY = "any"
|
|
170
|
-
_ORIGIN_HOST = re.compile(r"^[A-Za-z0-9.-]+(:[0-9]+)?$")
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
def parse_origins(value: str | None) -> frozenset[str] | None:
|
|
174
|
-
"""``WORLDS_HOST_OBJECT_STORE``: the origins (``scheme://host[:port]``,
|
|
175
|
-
comma-separated) the platform's presigned URLs may point at, or the word
|
|
176
|
-
``any`` for a dev host reading ``file://`` bundles (``None``: every URL
|
|
177
|
-
admitted). Unset is refused rather than read as ``any``: a deployed host
|
|
178
|
-
with no allowlist is a host whose secret can aim it anywhere, and a
|
|
179
|
-
template's placeholder (``https://<bucket>.s3...``) is refused by name."""
|
|
180
|
-
if value is None or not value.strip():
|
|
181
|
-
fail(
|
|
182
|
-
"host",
|
|
183
|
-
"WORLDS_HOST_OBJECT_STORE is unset; name the object store's origins "
|
|
184
|
-
f"(https://bucket.s3.amazonaws.com) or {OBJECT_STORE_ANY!r} for a dev host",
|
|
185
|
-
)
|
|
186
|
-
raise AssertionError # fail() never returns
|
|
187
|
-
if value.strip().lower() == OBJECT_STORE_ANY:
|
|
188
|
-
return None
|
|
189
|
-
origins = set()
|
|
190
|
-
for item in value.split(","):
|
|
191
|
-
item = item.strip()
|
|
192
|
-
if not item:
|
|
193
|
-
continue
|
|
194
|
-
parts = urllib.parse.urlsplit(item)
|
|
195
|
-
if parts.scheme not in ("http", "https") or not parts.netloc or parts.path not in ("", "/"):
|
|
196
|
-
fail(
|
|
197
|
-
"host",
|
|
198
|
-
f"WORLDS_HOST_OBJECT_STORE entries are origins like https://bucket.s3.amazonaws.com, not {item!r}",
|
|
199
|
-
)
|
|
200
|
-
if not _ORIGIN_HOST.match(parts.netloc):
|
|
201
|
-
fail(
|
|
202
|
-
"host",
|
|
203
|
-
f"WORLDS_HOST_OBJECT_STORE names a placeholder, not a host: {item!r}; set the real bucket origin",
|
|
204
|
-
)
|
|
205
|
-
origins.add(f"{parts.scheme}://{parts.netloc}".lower())
|
|
206
|
-
return frozenset(origins)
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
def check_object_store_url(url: object, origins: frozenset[str] | None, what: str) -> str:
|
|
210
|
-
"""The presigned URL the control plane handed us, once it is a string on one
|
|
211
|
-
of the object store's origins. The control plane is trusted with the host
|
|
212
|
-
secret, not with the host's network: a URL pointing anywhere else (the
|
|
213
|
-
metadata service, a cluster Service, the internet) is refused before a
|
|
214
|
-
byte moves, so a leaked secret cannot turn a session's state or the host's
|
|
215
|
-
reach into someone else's download."""
|
|
216
|
-
if not isinstance(url, str) or not url:
|
|
217
|
-
fail("host", f"{what} must be a presigned url")
|
|
218
|
-
raise AssertionError # fail() never returns
|
|
219
|
-
if origins is None:
|
|
220
|
-
return url
|
|
221
|
-
parts = urllib.parse.urlsplit(url)
|
|
222
|
-
origin = f"{parts.scheme}://{parts.netloc}".lower()
|
|
223
|
-
if origin not in origins:
|
|
224
|
-
fail("host", f"{what} must be on the object store ({', '.join(sorted(origins))}), not {origin}")
|
|
225
|
-
return url
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
def file_digest(path: Path) -> tuple[str, int]:
|
|
229
|
-
"""The hex sha256 and size of a file, streamed."""
|
|
230
|
-
digest = hashlib.sha256()
|
|
231
|
-
size = 0
|
|
232
|
-
with open(path, "rb") as fh:
|
|
233
|
-
for chunk in iter(lambda: fh.read(1 << 20), b""):
|
|
234
|
-
digest.update(chunk)
|
|
235
|
-
size += len(chunk)
|
|
236
|
-
return digest.hexdigest(), size
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
def download_verified(url: str, sha256: str, dest: Path) -> Path:
|
|
240
|
-
"""Fetch ``url`` into ``dest`` and refuse it unless its bytes hash to
|
|
241
|
-
``sha256``; a mismatch leaves nothing behind."""
|
|
242
|
-
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
243
|
-
staging = dest.with_name(f".{dest.name}.{uuid.uuid4().hex}")
|
|
244
|
-
try:
|
|
245
|
-
try:
|
|
246
|
-
with (
|
|
247
|
-
urllib.request.urlopen(urllib.request.Request(url), timeout=TRANSFER_TIMEOUT_S) as r,
|
|
248
|
-
open(staging, "wb") as out,
|
|
249
|
-
):
|
|
250
|
-
shutil.copyfileobj(r, out)
|
|
251
|
-
except Exception as error: # noqa: BLE001 - name the stage in the message
|
|
252
|
-
fail("host", f"could not download the session export: {error}")
|
|
253
|
-
refuse_digest_mismatch(staging, sha256)
|
|
254
|
-
os.replace(staging, dest)
|
|
255
|
-
finally:
|
|
256
|
-
staging.unlink(missing_ok=True)
|
|
257
|
-
return dest
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
def upload_file(url: str, path: Path, sha256: str, size: int, content_type: str) -> None:
|
|
261
|
-
"""PUT a file to a presigned URL, sending exactly the headers the platform
|
|
262
|
-
signed for it: content type, length and the base64 sha256 the object store
|
|
263
|
-
verifies on receipt."""
|
|
264
|
-
headers = {
|
|
265
|
-
"Content-Type": content_type,
|
|
266
|
-
"Content-Length": str(size),
|
|
267
|
-
"x-amz-checksum-sha256": base64.b64encode(bytes.fromhex(sha256)).decode("ascii"),
|
|
268
|
-
}
|
|
269
|
-
with open(path, "rb") as fh:
|
|
270
|
-
request = urllib.request.Request(url, data=fh, method="PUT", headers=headers)
|
|
271
|
-
try:
|
|
272
|
-
with urllib.request.urlopen(request, timeout=TRANSFER_TIMEOUT_S) as response:
|
|
273
|
-
response.read()
|
|
274
|
-
except urllib.error.HTTPError as error:
|
|
275
|
-
fail("host", f"the export upload was refused: HTTP {error.code} {error.read()[:200]!r}")
|
|
276
|
-
except urllib.error.URLError as error:
|
|
277
|
-
fail("host", f"could not upload the export: {error.reason}")
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
def world_tree(bundle: Path) -> Path:
|
|
281
|
-
"""The compiled schema world inside an extracted bundle (``WorldTree.world``:
|
|
282
|
-
``bench push`` packages a schema world as its api bundle with the tree
|
|
283
|
-
under ``server/world``; a bare tree is its own root). The host opens
|
|
284
|
-
compiled worlds only, so a tree without ``schema/world.json`` refuses."""
|
|
285
|
-
tree = WorldTree.from_dir(bundle)
|
|
286
|
-
if tree.has(tree.world_path(WORLD_SPEC)):
|
|
287
|
-
return tree.world
|
|
288
|
-
fail(
|
|
289
|
-
"host",
|
|
290
|
-
f"{bundle} holds no schema world the host can open (no {BUNDLED_WORLD}/{WORLD_SPEC} "
|
|
291
|
-
f"nor {WORLD_SPEC}); the host serves schema worlds and api bundles only",
|
|
292
|
-
)
|
|
293
|
-
raise AssertionError # fail() never returns
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
@dataclass
|
|
297
|
-
class CachedVersion:
|
|
298
|
-
"""One cached version, of either shape the host serves: an api bundle (a
|
|
299
|
-
connector ``spec`` and ``settings``, sessions cloned from ``snapshot``) or a
|
|
300
|
-
schema world stored as itself (``tools``: handlers, tasks and verifiers,
|
|
301
|
-
sessions copied from its template — and ``spec`` + ``settings`` too when
|
|
302
|
-
its tree carries connector.toml, the HTTP surface the same session serves
|
|
303
|
-
over the same store). ``snapshot`` and ``served`` are an api bundle's."""
|
|
304
|
-
|
|
305
|
-
version_id: str
|
|
306
|
-
bundle: Path
|
|
307
|
-
tree: Path
|
|
308
|
-
spec: Spec | None
|
|
309
|
-
settings: ApiSettings | None
|
|
310
|
-
snapshot: Path | None
|
|
311
|
-
engine: type[WorldDatabase]
|
|
312
|
-
#: None in the supervisor: only a worker imports world-authored code.
|
|
313
|
-
handlers: dict[str, Handler] | None = None
|
|
314
|
-
ui: UiSurface | None = None
|
|
315
|
-
tools: ToolsVersion | None = None
|
|
316
|
-
served: snapshots.ServedSnapshot | None = None
|
|
317
|
-
|
|
318
|
-
@property
|
|
319
|
-
def runtime_class(self) -> type[WorldRuntime]:
|
|
320
|
-
"""The one place a session's engine is chosen: a schema world runs as
|
|
321
|
-
the tree it is; an api bundle on the snapshot READ PATH that step 6
|
|
322
|
-
deletes with the bundle shape itself."""
|
|
323
|
-
return TreeRuntime if self.tools is not None else SnapshotRuntime
|
|
324
|
-
|
|
325
|
-
def router(self) -> Router:
|
|
326
|
-
"""A fresh router per session: routes are compiled from the shared spec,
|
|
327
|
-
but the oauth2 tokens a world mints must stay that session's own."""
|
|
328
|
-
assert self.spec is not None and self.settings is not None
|
|
329
|
-
return Router(self.spec, self.settings)
|
|
330
|
-
|
|
331
|
-
@property
|
|
332
|
-
def authored_code(self) -> str | None:
|
|
333
|
-
"""What world-authored code the version runs, None for none: only a
|
|
334
|
-
worker may open a version that runs any."""
|
|
335
|
-
return self.runtime_class.authored_code(self)
|
|
336
|
-
|
|
337
|
-
@property
|
|
338
|
-
def needs_worker(self) -> bool:
|
|
339
|
-
return self.authored_code is not None
|
|
340
|
-
|
|
341
|
-
def served_names(self, dependencies: dict[str, DependencyWorld]) -> ServedNames:
|
|
342
|
-
"""The tools a session task may allow, beside the routes it may not."""
|
|
343
|
-
return ServedNames.of(self.runtime_class.served_by(self, dependencies), self.spec)
|
|
344
|
-
|
|
345
|
-
@property
|
|
346
|
-
def declared_dependencies(self) -> dict[str, Dependency]:
|
|
347
|
-
return self.runtime_class.declared_dependencies(self)
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
class VersionCache:
|
|
351
|
-
"""Every world version this host has seen, on local disk. The first session
|
|
352
|
-
of a version pays for the download and extract; the rest clone its
|
|
353
|
-
snapshot. Nothing here is per session, so a hot version is cheap to serve
|
|
354
|
-
many times over."""
|
|
355
|
-
|
|
356
|
-
def __init__(self, root: Path, *, run_handlers: bool = False, max_bundle_mb: int = DEFAULT_MAX_BUNDLE_MB):
|
|
357
|
-
self.root = Path(root)
|
|
358
|
-
self.root.mkdir(parents=True, exist_ok=True)
|
|
359
|
-
#: Off in the supervisor: only a version worker imports world-authored code.
|
|
360
|
-
self.run_handlers = run_handlers
|
|
361
|
-
self.max_bundle_bytes = max_bundle_mb * (1 << 20)
|
|
362
|
-
self._lock = threading.Lock()
|
|
363
|
-
self._versions: dict[str, CachedVersion] = {}
|
|
364
|
-
self._builds = Lanes()
|
|
365
|
-
|
|
366
|
-
def ensure(
|
|
367
|
-
self,
|
|
368
|
-
version_id: str,
|
|
369
|
-
bundle_url: str,
|
|
370
|
-
snapshot: dict[str, str] | None,
|
|
371
|
-
) -> CachedVersion:
|
|
372
|
-
hit = self._cached(version_id)
|
|
373
|
-
if hit is not None:
|
|
374
|
-
return hit
|
|
375
|
-
with self._builds.turn(version_id):
|
|
376
|
-
return self._cached(version_id) or self._store(self._build(version_id, bundle_url, snapshot))
|
|
377
|
-
|
|
378
|
-
def _cached(self, version_id: str) -> CachedVersion | None:
|
|
379
|
-
with self._lock:
|
|
380
|
-
return self._versions.get(version_id)
|
|
381
|
-
|
|
382
|
-
def _store(self, cached: CachedVersion) -> CachedVersion:
|
|
383
|
-
with self._lock:
|
|
384
|
-
self._versions[cached.version_id] = cached
|
|
385
|
-
return cached
|
|
386
|
-
|
|
387
|
-
def has(self, version_id: str) -> bool:
|
|
388
|
-
with self._lock:
|
|
389
|
-
return version_id in self._versions
|
|
390
|
-
|
|
391
|
-
def version_ids(self) -> list[str]:
|
|
392
|
-
with self._lock:
|
|
393
|
-
return sorted(self._versions)
|
|
394
|
-
|
|
395
|
-
def _build(self, version_id: str, bundle_url: str, snapshot: dict[str, str] | None) -> CachedVersion:
|
|
396
|
-
bundle = self.root / version_id / "bundle"
|
|
397
|
-
if not bundle.is_dir():
|
|
398
|
-
self._download(bundle_url, bundle)
|
|
399
|
-
if is_tools_bundle(bundle):
|
|
400
|
-
tools = load_tools_version(
|
|
401
|
-
bundle, self.root / version_id / "template.db", snapshot, run_handlers=self.run_handlers
|
|
402
|
-
)
|
|
403
|
-
return CachedVersion(
|
|
404
|
-
version_id=version_id,
|
|
405
|
-
bundle=bundle,
|
|
406
|
-
tree=bundle,
|
|
407
|
-
spec=tools.spec,
|
|
408
|
-
settings=tools.settings,
|
|
409
|
-
snapshot=None,
|
|
410
|
-
engine=tools.engine,
|
|
411
|
-
handlers=tools.api_handlers,
|
|
412
|
-
ui=load_ui_surface(bundle) if tools.spec is not None else None,
|
|
413
|
-
tools=tools,
|
|
414
|
-
)
|
|
415
|
-
tree = world_tree(bundle)
|
|
416
|
-
contract = check_bundle(tree)
|
|
417
|
-
if snapshot:
|
|
418
|
-
snapshots.fetch(tree, contract, snapshot["url"], snapshot["sha256"], snapshot["digest"])
|
|
419
|
-
served = snapshots.ServedSnapshot.of(tree)
|
|
420
|
-
spec = load_spec(tree)
|
|
421
|
-
settings = load_api_settings(tree, spec)
|
|
422
|
-
handlers = load_handlers(settings) if self.run_handlers and settings.handlers is not None else None
|
|
423
|
-
snap = snapshots.default_snapshot(tree, contract)
|
|
424
|
-
if not snap.is_file():
|
|
425
|
-
fail(
|
|
426
|
-
"host",
|
|
427
|
-
f"version {version_id} has no compiled snapshot ({snap.relative_to(tree)}); "
|
|
428
|
-
"publish a data snapshot or pin one on the version",
|
|
429
|
-
)
|
|
430
|
-
return CachedVersion(
|
|
431
|
-
version_id=version_id,
|
|
432
|
-
bundle=bundle,
|
|
433
|
-
tree=tree,
|
|
434
|
-
spec=spec,
|
|
435
|
-
settings=settings,
|
|
436
|
-
snapshot=snap,
|
|
437
|
-
engine=WorldDatabase.for_contract(contract),
|
|
438
|
-
served=served,
|
|
439
|
-
handlers=handlers,
|
|
440
|
-
ui=load_ui_surface(bundle),
|
|
441
|
-
)
|
|
442
|
-
|
|
443
|
-
def _download(self, url: str, dest: Path) -> None:
|
|
444
|
-
"""Fetch and extract into a sibling staging directory, then rename it into
|
|
445
|
-
place: ``dest`` exists only once it is complete, so a download that dies
|
|
446
|
-
halfway is never mistaken for a cached bundle on the next open."""
|
|
447
|
-
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
448
|
-
staging = dest.parent / f".{dest.name}.{uuid.uuid4().hex}"
|
|
449
|
-
tgz = staging.with_suffix(".tar.gz")
|
|
450
|
-
limit = self.max_bundle_bytes
|
|
451
|
-
try:
|
|
452
|
-
try:
|
|
453
|
-
with (
|
|
454
|
-
urllib.request.urlopen(urllib.request.Request(url), timeout=TRANSFER_TIMEOUT_S) as r,
|
|
455
|
-
open(tgz, "wb") as out,
|
|
456
|
-
):
|
|
457
|
-
copied = 0
|
|
458
|
-
for chunk in iter(lambda: r.read(1 << 20), b""):
|
|
459
|
-
copied += len(chunk)
|
|
460
|
-
if copied > limit:
|
|
461
|
-
raise ValueError(f"the bundle is larger than {limit} bytes")
|
|
462
|
-
out.write(chunk)
|
|
463
|
-
except Exception as error: # noqa: BLE001 - name the stage in the message
|
|
464
|
-
fail("host", f"could not download the bundle: {error}")
|
|
465
|
-
staging.mkdir()
|
|
466
|
-
with tarfile.open(tgz, "r:gz") as t:
|
|
467
|
-
members = t.getmembers()
|
|
468
|
-
if len(members) > MAX_BUNDLE_MEMBERS:
|
|
469
|
-
fail("host", f"the bundle holds {len(members)} files, more than the {MAX_BUNDLE_MEMBERS} allowed")
|
|
470
|
-
extracted = sum(m.size for m in members)
|
|
471
|
-
if extracted > limit:
|
|
472
|
-
fail("host", f"the bundle extracts to {extracted} bytes, more than the {limit} allowed")
|
|
473
|
-
safe_tar_extract(t, staging)
|
|
474
|
-
os.replace(staging, dest)
|
|
475
|
-
finally:
|
|
476
|
-
tgz.unlink(missing_ok=True)
|
|
477
|
-
shutil.rmtree(staging, ignore_errors=True)
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
@dataclass
|
|
481
|
-
class Export:
|
|
482
|
-
"""A session's state copied to a file on this host, hashed, waiting for the
|
|
483
|
-
platform to hand over the presigned URL it goes to."""
|
|
484
|
-
|
|
485
|
-
export_id: str
|
|
486
|
-
path: Path
|
|
487
|
-
sha256: str
|
|
488
|
-
bytes: int
|
|
489
|
-
content_type: str
|
|
490
|
-
|
|
491
|
-
def describe(self) -> dict:
|
|
492
|
-
return {
|
|
493
|
-
"exportId": self.export_id,
|
|
494
|
-
"sha256": self.sha256,
|
|
495
|
-
"bytes": self.bytes,
|
|
496
|
-
"contentType": self.content_type,
|
|
497
|
-
}
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
@dataclass
|
|
501
|
-
class Session:
|
|
502
|
-
session_id: str
|
|
503
|
-
version_id: str
|
|
504
|
-
task: str | None
|
|
505
|
-
spec: Spec | None
|
|
506
|
-
token: str | None
|
|
507
|
-
bundle: Path
|
|
508
|
-
tree: Path
|
|
509
|
-
lease_until: float
|
|
510
|
-
engine: type[WorldDatabase]
|
|
511
|
-
runtime: WorldRuntime | None = None
|
|
512
|
-
worker: Worker | None = None
|
|
513
|
-
ui: UiSurface | None = None
|
|
514
|
-
browser: BrowserRelay | None = None
|
|
515
|
-
#: Teardown waits for zero, so a close or expiry never pulls the database from under a handler.
|
|
516
|
-
inflight: int = 0
|
|
517
|
-
#: Once set, the last request out closes the state.
|
|
518
|
-
closing: bool = False
|
|
519
|
-
opened_at: float = field(default_factory=time.time)
|
|
520
|
-
#: A renewal or an export is the platform looking, not the session working, and must not move it:
|
|
521
|
-
#: the platform's idle clock follows this.
|
|
522
|
-
touched_at: float = field(default_factory=time.time)
|
|
523
|
-
export: Export | None = None
|
|
524
|
-
restored: bool = False
|
|
525
|
-
task_spec: TaskSpec | None = None
|
|
526
|
-
exporting: threading.Lock = field(default_factory=threading.Lock, repr=False, compare=False)
|
|
527
|
-
|
|
528
|
-
@property
|
|
529
|
-
def app(self) -> WorldApp | None:
|
|
530
|
-
"""The world's routes over this session's store, when this process
|
|
531
|
-
serves it and the tree has any."""
|
|
532
|
-
return self.runtime.app if self.runtime is not None else None
|
|
533
|
-
|
|
534
|
-
@property
|
|
535
|
-
def task_key(self) -> tuple[str | None, dict | None]:
|
|
536
|
-
"""What an open retry must match: the bundled name or the spec itself."""
|
|
537
|
-
return self.task, self.task_spec.raw if self.task_spec is not None else None
|
|
538
|
-
|
|
539
|
-
def renew(self, lease_s: float) -> None:
|
|
540
|
-
self.lease_until = time.time() + lease_s
|
|
541
|
-
|
|
542
|
-
@property
|
|
543
|
-
def seconds(self) -> float:
|
|
544
|
-
return max(0.0, time.time() - self.opened_at)
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
class WorldHost:
|
|
548
|
-
"""The session registry and the control operations, in one of two roles:
|
|
549
|
-
|
|
550
|
-
* the **supervisor** (the deployed host): serves code-free worlds in
|
|
551
|
-
process and places worlds with handlers on its :class:`WorkerPool`;
|
|
552
|
-
* a **worker** (``run_handlers=True``, ``pinned_version=...``): serves one
|
|
553
|
-
version, handlers included, for its supervisor.
|
|
554
|
-
"""
|
|
555
|
-
|
|
556
|
-
def __init__(
|
|
557
|
-
self,
|
|
558
|
-
cache_root: Path,
|
|
559
|
-
*,
|
|
560
|
-
lease_s: float = DEFAULT_LEASE_S,
|
|
561
|
-
advertise_url: str | None = None,
|
|
562
|
-
workers: WorkerPool | None = None,
|
|
563
|
-
versions_root: Path | None = None,
|
|
564
|
-
run_handlers: bool = False,
|
|
565
|
-
pinned_version: str | None = None,
|
|
566
|
-
limits: dict[str, int] | None = None,
|
|
567
|
-
max_sessions: int = DEFAULT_MAX_SESSIONS,
|
|
568
|
-
max_bundle_mb: int = DEFAULT_MAX_BUNDLE_MB,
|
|
569
|
-
object_store: frozenset[str] | None = None,
|
|
570
|
-
):
|
|
571
|
-
#: Every presigned URL the control plane hands this host must point here; None admits any.
|
|
572
|
-
self.object_store = object_store
|
|
573
|
-
self.cache = VersionCache(
|
|
574
|
-
versions_root or Path(cache_root) / "versions", run_handlers=run_handlers, max_bundle_mb=max_bundle_mb
|
|
575
|
-
)
|
|
576
|
-
self.sessions_root = Path(cache_root) / "sessions"
|
|
577
|
-
# Sessions live in this process, so whatever a previous one left on disk is unreachable.
|
|
578
|
-
shutil.rmtree(self.sessions_root, ignore_errors=True)
|
|
579
|
-
self.sessions_root.mkdir(parents=True, exist_ok=True)
|
|
580
|
-
self.lease_s = lease_s
|
|
581
|
-
self.advertise_url = advertise_url
|
|
582
|
-
self.workers = workers
|
|
583
|
-
self.run_handlers = run_handlers
|
|
584
|
-
self.pinned_version = pinned_version
|
|
585
|
-
self.limits = limits
|
|
586
|
-
self.max_sessions = max_sessions
|
|
587
|
-
self.draining = False
|
|
588
|
-
self.started_at = time.time()
|
|
589
|
-
self._lock = threading.Lock()
|
|
590
|
-
self._sessions: dict[str, Session] = {}
|
|
591
|
-
#: Each holds a slot from admission, so concurrent opens cannot all pass the capacity check.
|
|
592
|
-
self._opening: set[str] = set()
|
|
593
|
-
#: Opens the platform closed before they finished: closed the moment they exist, not at lease end.
|
|
594
|
-
self._abandoned: set[str] = set()
|
|
595
|
-
self._opens = Lanes()
|
|
596
|
-
self._counters: Counter[str] = Counter()
|
|
597
|
-
self._session_seconds = 0.0
|
|
598
|
-
|
|
599
|
-
def open(
|
|
600
|
-
self,
|
|
601
|
-
session_id: str,
|
|
602
|
-
version_id: str,
|
|
603
|
-
bundle_url: str,
|
|
604
|
-
task: str | None,
|
|
605
|
-
token: str | None,
|
|
606
|
-
snapshot: dict[str, str] | None,
|
|
607
|
-
browser: dict[str, str] | None = None,
|
|
608
|
-
restore: dict[str, str] | None = None,
|
|
609
|
-
task_spec: Any = None,
|
|
610
|
-
dependencies: list[dict[str, Any]] | None = None,
|
|
611
|
-
) -> dict:
|
|
612
|
-
"""Open ``session_id`` on ``version_id``, either on the bundled ``task``
|
|
613
|
-
or on ``task_spec``, a session task of its own (:mod:`tasks`); exactly
|
|
614
|
-
one of the two. ``dependencies`` are the versions the world's
|
|
615
|
-
``[dependencies]`` resolved to, one ``{alias, versionId, bundleUrl,
|
|
616
|
-
snapshot?}`` per alias the manifest declares, opened beside it."""
|
|
617
|
-
if self.pinned_version is not None and version_id != self.pinned_version:
|
|
618
|
-
fail("host", f"this worker serves version {self.pinned_version} only, not {version_id}")
|
|
619
|
-
if (task is None) == (task_spec is None):
|
|
620
|
-
fail("host", "an open names a bundled task or carries a task spec, one of the two")
|
|
621
|
-
if task is not None and not isinstance(task, str):
|
|
622
|
-
fail("host", "task is the bundled task's name")
|
|
623
|
-
spec = parse_task_spec(task_spec) if task_spec is not None else None
|
|
624
|
-
deps = parse_dependency_refs(dependencies)
|
|
625
|
-
with self._lock:
|
|
626
|
-
existing = self._sessions.get(session_id)
|
|
627
|
-
if existing is None:
|
|
628
|
-
# A retry of an open already served is answered even while draining: refusing it would
|
|
629
|
-
# send the platform to open the same id elsewhere while this copy holds the live writes.
|
|
630
|
-
if self.draining:
|
|
631
|
-
fail("draining", "this host is draining and takes no new sessions")
|
|
632
|
-
self._admit(session_id)
|
|
633
|
-
if existing is not None:
|
|
634
|
-
# A retried open gets the same session, never a rebuild that drops its writes.
|
|
635
|
-
wanted = (version_id, (task, spec.raw if spec is not None else None), token)
|
|
636
|
-
if (existing.version_id, existing.task_key, existing.token) != wanted:
|
|
637
|
-
fail("host", f"session {session_id} is already open as another version, task or token")
|
|
638
|
-
return self._describe(existing)
|
|
639
|
-
try:
|
|
640
|
-
described = self._open(
|
|
641
|
-
session_id, version_id, bundle_url, task, spec, token, snapshot, browser, restore, deps
|
|
642
|
-
)
|
|
643
|
-
except Exception:
|
|
644
|
-
self.tally("open_failures")
|
|
645
|
-
with self._lock:
|
|
646
|
-
self._opening.discard(session_id)
|
|
647
|
-
self._abandoned.discard(session_id)
|
|
648
|
-
raise
|
|
649
|
-
with self._lock:
|
|
650
|
-
self._opening.discard(session_id)
|
|
651
|
-
abandoned = session_id in self._abandoned
|
|
652
|
-
self._abandoned.discard(session_id)
|
|
653
|
-
if abandoned:
|
|
654
|
-
self.close(session_id)
|
|
655
|
-
fail(UNKNOWN_SESSION, f"session {session_id} was closed while it opened")
|
|
656
|
-
return described
|
|
657
|
-
|
|
658
|
-
def _admit(self, session_id: str) -> None:
|
|
659
|
-
"""Reserve one of this host's ``max_sessions`` for ``session_id`` while it
|
|
660
|
-
opens. Under ``_lock``."""
|
|
661
|
-
if session_id in self._opening:
|
|
662
|
-
fail("host", f"session {session_id} is being opened already")
|
|
663
|
-
if len(self._sessions) + len(self._opening) >= self.max_sessions:
|
|
664
|
-
fail("at_capacity", f"this host holds its {self.max_sessions} sessions; place elsewhere")
|
|
665
|
-
self._opening.add(session_id)
|
|
666
|
-
|
|
667
|
-
def _open(
|
|
668
|
-
self,
|
|
669
|
-
session_id: str,
|
|
670
|
-
version_id: str,
|
|
671
|
-
bundle_url: str,
|
|
672
|
-
task: str | None,
|
|
673
|
-
spec: TaskSpec | None,
|
|
674
|
-
token: str | None,
|
|
675
|
-
snapshot: dict[str, str] | None,
|
|
676
|
-
browser: dict[str, str] | None,
|
|
677
|
-
restore: dict[str, str] | None,
|
|
678
|
-
deps: list[DependencyRef],
|
|
679
|
-
) -> dict:
|
|
680
|
-
cached = self.cache.ensure(version_id, *self._trusted_version(bundle_url, snapshot))
|
|
681
|
-
dependencies = self._dependency_worlds(cached, deps)
|
|
682
|
-
if cached.spec is not None:
|
|
683
|
-
unsupported = unsupported_auth(cached.spec)
|
|
684
|
-
if unsupported:
|
|
685
|
-
fail("host", unsupported)
|
|
686
|
-
if spec is not None:
|
|
687
|
-
cached.served_names(dependencies).check_allowed(spec.tools)
|
|
688
|
-
relay = BrowserRelay.parse(browser)
|
|
689
|
-
if cached.needs_worker and not self.run_handlers:
|
|
690
|
-
return self._open_on_worker(cached, session_id, bundle_url, task, spec, token, relay, restore, deps)
|
|
691
|
-
session_dir = self.sessions_root / session_id
|
|
692
|
-
with self._opens.turn(version_id):
|
|
693
|
-
runtime = WorldRuntime.open(
|
|
694
|
-
cached,
|
|
695
|
-
session_dir,
|
|
696
|
-
task=task,
|
|
697
|
-
spec=spec,
|
|
698
|
-
token=token,
|
|
699
|
-
dependencies=dependencies,
|
|
700
|
-
restore=lambda load: self._restore_into(restore, session_dir, load, cached.engine),
|
|
701
|
-
)
|
|
702
|
-
session = Session(
|
|
703
|
-
session_id=session_id,
|
|
704
|
-
version_id=version_id,
|
|
705
|
-
task=task,
|
|
706
|
-
spec=cached.spec,
|
|
707
|
-
token=token,
|
|
708
|
-
bundle=cached.bundle,
|
|
709
|
-
tree=cached.tree,
|
|
710
|
-
lease_until=time.time() + self.lease_s,
|
|
711
|
-
engine=cached.engine,
|
|
712
|
-
runtime=runtime,
|
|
713
|
-
ui=cached.ui,
|
|
714
|
-
browser=relay,
|
|
715
|
-
restored=runtime.restored,
|
|
716
|
-
task_spec=spec,
|
|
717
|
-
)
|
|
718
|
-
self._register(session)
|
|
719
|
-
return self._describe(session)
|
|
720
|
-
|
|
721
|
-
def _open_on_worker(
|
|
722
|
-
self,
|
|
723
|
-
cached: CachedVersion,
|
|
724
|
-
session_id: str,
|
|
725
|
-
bundle_url: str,
|
|
726
|
-
task: str | None,
|
|
727
|
-
spec: TaskSpec | None,
|
|
728
|
-
token: str | None,
|
|
729
|
-
relay: BrowserRelay | None,
|
|
730
|
-
restore: dict[str, str] | None,
|
|
731
|
-
deps: list[DependencyRef],
|
|
732
|
-
) -> dict:
|
|
733
|
-
"""Place the session on the version's worker. The worker opens it from
|
|
734
|
-
the bundle this process already cached (no snapshot to fetch: the file
|
|
735
|
-
is on disk), and the supervisor records only where it went. The ui's
|
|
736
|
-
static files and the browser tunnel stay the supervisor's; the ui's
|
|
737
|
-
api-prefix calls are relayed as they came, so the worker logs them
|
|
738
|
-
``via: ui``. An export to restore from is fetched and verified here
|
|
739
|
-
too, then handed to the worker as a local file with its sha256, which
|
|
740
|
-
the worker checks again before it opens the file.
|
|
741
|
-
A worker every one of whose peers holds sessions cannot be started
|
|
742
|
-
past the pool's memory budget: that is ``at_capacity``, and the
|
|
743
|
-
platform places the session on another replica."""
|
|
744
|
-
if self.workers is None:
|
|
745
|
-
fail(
|
|
746
|
-
"host",
|
|
747
|
-
f"version {cached.version_id} runs world-authored code ({cached.authored_code}); "
|
|
748
|
-
"this host runs none in its process and has no version workers",
|
|
749
|
-
)
|
|
750
|
-
try:
|
|
751
|
-
worker = self.workers.get(cached.version_id)
|
|
752
|
-
except WorkerCapacity as error:
|
|
753
|
-
fail("at_capacity", str(error))
|
|
754
|
-
# Claimed before placement, so another open cannot stop this worker as session-less meanwhile.
|
|
755
|
-
with self._lock:
|
|
756
|
-
worker.sessions.add(session_id)
|
|
757
|
-
source = None
|
|
758
|
-
try:
|
|
759
|
-
source = self._restore_source(restore, self._worker_session_dir(worker, session_id))
|
|
760
|
-
with self._opens.turn(cached.version_id):
|
|
761
|
-
self._worker_control(
|
|
762
|
-
worker,
|
|
763
|
-
"POST",
|
|
764
|
-
"/__host/sessions",
|
|
765
|
-
{
|
|
766
|
-
"sessionId": session_id,
|
|
767
|
-
"versionId": cached.version_id,
|
|
768
|
-
"bundleUrl": bundle_url,
|
|
769
|
-
"task": task,
|
|
770
|
-
"taskSpec": spec.raw if spec is not None else None,
|
|
771
|
-
"token": token,
|
|
772
|
-
"restore": source.for_worker() if source is not None else None,
|
|
773
|
-
"dependencies": [dep.raw for dep in deps],
|
|
774
|
-
},
|
|
775
|
-
)
|
|
776
|
-
except BaseException:
|
|
777
|
-
with self._lock:
|
|
778
|
-
worker.sessions.discard(session_id)
|
|
779
|
-
if not worker.sessions:
|
|
780
|
-
worker.idle_since = time.time()
|
|
781
|
-
raise
|
|
782
|
-
finally:
|
|
783
|
-
if source is not None:
|
|
784
|
-
source.path.unlink(missing_ok=True)
|
|
785
|
-
session = Session(
|
|
786
|
-
session_id=session_id,
|
|
787
|
-
version_id=cached.version_id,
|
|
788
|
-
task=task,
|
|
789
|
-
spec=cached.spec,
|
|
790
|
-
token=token,
|
|
791
|
-
bundle=cached.bundle,
|
|
792
|
-
tree=cached.tree,
|
|
793
|
-
lease_until=time.time() + self.lease_s,
|
|
794
|
-
engine=cached.engine,
|
|
795
|
-
worker=worker,
|
|
796
|
-
ui=cached.ui,
|
|
797
|
-
browser=relay,
|
|
798
|
-
restored=source is not None,
|
|
799
|
-
task_spec=spec,
|
|
800
|
-
)
|
|
801
|
-
self._register(session)
|
|
802
|
-
return self._describe(session)
|
|
803
|
-
|
|
804
|
-
def _worker_session_dir(self, worker: Worker, session_id: str) -> Path:
|
|
805
|
-
"""Where a worker keeps a session's files: its own ``WorldHost`` runs
|
|
806
|
-
under ``workers/<version>`` of this host's cache root (:mod:`worker`)."""
|
|
807
|
-
assert self.workers is not None
|
|
808
|
-
return self.workers.root / "workers" / worker.version_id / "sessions" / session_id
|
|
809
|
-
|
|
810
|
-
def _restore_into(
|
|
811
|
-
self,
|
|
812
|
-
restore: dict[str, str] | None,
|
|
813
|
-
session_dir: Path,
|
|
814
|
-
load: Callable[[SessionExport], None],
|
|
815
|
-
engine: type[WorldDatabase],
|
|
816
|
-
) -> bool:
|
|
817
|
-
"""Load the export ``restore`` names into a freshly opened world, its
|
|
818
|
-
bytes checked against the recorded sha256 here, where they are opened,
|
|
819
|
-
then drop the fetched file. False when there was nothing to restore."""
|
|
820
|
-
source = self._restore_source(restore, session_dir)
|
|
821
|
-
if source is None:
|
|
822
|
-
return False
|
|
823
|
-
try:
|
|
824
|
-
load(SessionExport.verified(source, engine))
|
|
825
|
-
finally:
|
|
826
|
-
source.path.unlink(missing_ok=True)
|
|
827
|
-
return True
|
|
828
|
-
|
|
829
|
-
def _restore_source(self, restore: dict[str, str] | None, session_dir: Path) -> ExportFile | None:
|
|
830
|
-
"""The export a session continues from, as a local file and the sha256
|
|
831
|
-
the platform recorded for it, or None for a fresh session. The platform
|
|
832
|
-
names it by ``{url, sha256}`` and the file is fetched into the session's
|
|
833
|
-
directory and verified; a supervisor names it to its worker by ``{path,
|
|
834
|
-
sha256}``, and only the supervisor reaches a worker's control socket."""
|
|
835
|
-
if not restore:
|
|
836
|
-
return None
|
|
837
|
-
path = restore.get("path")
|
|
838
|
-
if path is not None:
|
|
839
|
-
return self._worker_restore_source(Path(path), restore.get("sha256"))
|
|
840
|
-
url, sha256 = restore.get("url"), restore.get("sha256")
|
|
841
|
-
if not isinstance(url, str) or not isinstance(sha256, str) or len(sha256) != 64:
|
|
842
|
-
fail("host", "restore must be {url: <presigned export url>, sha256: <hex digest>}")
|
|
843
|
-
raise AssertionError # fail() never returns
|
|
844
|
-
check_object_store_url(url, self.object_store, "restore.url")
|
|
845
|
-
return ExportFile(download_verified(url, sha256, session_dir / "restore.db"), sha256)
|
|
846
|
-
|
|
847
|
-
def _worker_restore_source(self, path: Path, sha256: object) -> ExportFile:
|
|
848
|
-
if not self.run_handlers:
|
|
849
|
-
fail("host", "restore by path is a worker's; name the export by {url, sha256}")
|
|
850
|
-
if not path.is_file():
|
|
851
|
-
fail("host", f"restore file {path} is missing")
|
|
852
|
-
if not isinstance(sha256, str) or len(sha256) != 64:
|
|
853
|
-
fail("host", "restore by path names the export's sha256 beside it")
|
|
854
|
-
raise AssertionError # fail() never returns
|
|
855
|
-
return ExportFile(path, sha256)
|
|
856
|
-
|
|
857
|
-
def _trusted_version(self, bundle_url: str, snapshot: dict[str, str] | None) -> tuple[str, dict[str, str] | None]:
|
|
858
|
-
check_object_store_url(bundle_url, self.object_store, "bundleUrl")
|
|
859
|
-
if snapshot:
|
|
860
|
-
check_object_store_url(snapshot.get("url"), self.object_store, "snapshot.url")
|
|
861
|
-
return bundle_url, snapshot
|
|
862
|
-
|
|
863
|
-
def _register(self, session: Session) -> None:
|
|
864
|
-
with self._lock:
|
|
865
|
-
replaced = self._retire(self._sessions.pop(session.session_id, None))
|
|
866
|
-
self._sessions[session.session_id] = session
|
|
867
|
-
self._counters["opened"] += 1
|
|
868
|
-
if replaced is not None:
|
|
869
|
-
self._finish(replaced, reason="replaced")
|
|
870
|
-
|
|
871
|
-
def _describe(self, session: Session) -> dict:
|
|
872
|
-
return {
|
|
873
|
-
"sessionId": session.session_id,
|
|
874
|
-
"versionId": session.version_id,
|
|
875
|
-
"task": session.task,
|
|
876
|
-
"taskName": session.task if session.task_spec is None else session.task_spec.name,
|
|
877
|
-
"taskSource": "bundled" if session.task_spec is None else "inline",
|
|
878
|
-
"tools": (
|
|
879
|
-
list(session.task_spec.tools)
|
|
880
|
-
if session.task_spec is not None and session.task_spec.tools is not None
|
|
881
|
-
else None
|
|
882
|
-
),
|
|
883
|
-
"ready": True,
|
|
884
|
-
"hostUrl": self.advertise_url,
|
|
885
|
-
"leaseSeconds": self.lease_s,
|
|
886
|
-
"leaseUntil": session.lease_until,
|
|
887
|
-
"workerPid": session.worker.pid if session.worker is not None else None,
|
|
888
|
-
"ui": {"entry": session.ui.entry, "apiPath": session.ui.api_path} if session.ui is not None else None,
|
|
889
|
-
"browser": session.browser is not None,
|
|
890
|
-
"restored": session.restored,
|
|
891
|
-
"openedAt": session.opened_at,
|
|
892
|
-
"sessionSeconds": session.seconds,
|
|
893
|
-
"touchedAt": session.touched_at,
|
|
894
|
-
}
|
|
895
|
-
|
|
896
|
-
def close(self, session_id: str) -> dict:
|
|
897
|
-
with self._lock:
|
|
898
|
-
session = self._sessions.pop(session_id, None)
|
|
899
|
-
idle = self._retire(session)
|
|
900
|
-
if session is None and session_id in self._opening:
|
|
901
|
-
self._abandoned.add(session_id)
|
|
902
|
-
if idle is not None:
|
|
903
|
-
self._finish(idle, reason="closed")
|
|
904
|
-
if session is None:
|
|
905
|
-
return {"closed": True, "openedAt": None, "sessionSeconds": 0.0}
|
|
906
|
-
return {"closed": True, "openedAt": session.opened_at, "sessionSeconds": session.seconds}
|
|
907
|
-
|
|
908
|
-
def set_browser(self, session_id: str, browser: Any) -> dict:
|
|
909
|
-
"""Point the session's browser surface at another pool relay: the one
|
|
910
|
-
it had died and the platform leased a replacement. Connections open
|
|
911
|
-
on the old relay end with it; every connection after this reaches the
|
|
912
|
-
new one. A session opened without a browser never gains one here."""
|
|
913
|
-
relay = BrowserRelay.parse(browser)
|
|
914
|
-
if relay is None:
|
|
915
|
-
fail("host", "browser must be {url: 'ws://host:port', token: <relay token>}")
|
|
916
|
-
with self._lock:
|
|
917
|
-
session = self._sessions.get(session_id)
|
|
918
|
-
if session is None:
|
|
919
|
-
fail(UNKNOWN_SESSION, f"no session {session_id} on this host")
|
|
920
|
-
if session.browser is None:
|
|
921
|
-
fail("host", f"session {session_id} was opened without a browser surface")
|
|
922
|
-
session.browser = relay
|
|
923
|
-
return {"sessionId": session_id, "browser": True}
|
|
924
|
-
|
|
925
|
-
def export(self, session_id: str, known_sha256: str | None = None) -> dict:
|
|
926
|
-
"""Copy the session's state to a file on this host and hash it, one export of
|
|
927
|
-
a session at a time; it waits for :meth:`upload_export`, and a new one discards
|
|
928
|
-
the last. A copy hashing to ``known_sha256`` (the platform's last export of the
|
|
929
|
-
session) is thrown away and reported ``unchanged``: nothing to upload."""
|
|
930
|
-
with self.using(session_id, touch=False) as session, session.exporting:
|
|
931
|
-
engine = session.engine
|
|
932
|
-
if session.worker is not None:
|
|
933
|
-
answer = self._worker_control(session.worker, "POST", f"/__host/sessions/{session_id}/export", {})
|
|
934
|
-
path = self._worker_export_path(session, answer)
|
|
935
|
-
else:
|
|
936
|
-
exports = self.sessions_root / session_id / "exports"
|
|
937
|
-
shutil.rmtree(exports, ignore_errors=True)
|
|
938
|
-
path = exports / f"{uuid.uuid4().hex}{Path(engine.snapshot_name).suffix}"
|
|
939
|
-
assert session.runtime is not None
|
|
940
|
-
session.runtime.export(path)
|
|
941
|
-
sha256, size = file_digest(path)
|
|
942
|
-
if known_sha256 is not None and sha256 == known_sha256:
|
|
943
|
-
path.unlink(missing_ok=True)
|
|
944
|
-
with self._lock:
|
|
945
|
-
session.export = None
|
|
946
|
-
self._counters["exports_unchanged"] += 1
|
|
947
|
-
return {"exportId": None, "sha256": sha256, "bytes": size, "unchanged": True}
|
|
948
|
-
export = Export(export_id=path.stem, path=path, sha256=sha256, bytes=size, content_type=engine.content_type)
|
|
949
|
-
with self._lock:
|
|
950
|
-
session.export = export
|
|
951
|
-
self._counters["exports"] += 1
|
|
952
|
-
described = export.describe()
|
|
953
|
-
if self.run_handlers:
|
|
954
|
-
described["path"] = str(path) # for the supervisor, which uploads on the worker's behalf
|
|
955
|
-
return described
|
|
956
|
-
|
|
957
|
-
def _worker_export_path(self, session: Session, answer: dict) -> Path:
|
|
958
|
-
"""The file a worker's export landed in, admitted only from inside that
|
|
959
|
-
worker's directory for this session: the worker runs tenant code, so
|
|
960
|
-
what it names is checked, and the bytes are hashed here, not trusted."""
|
|
961
|
-
assert session.worker is not None
|
|
962
|
-
exports = self._worker_session_dir(session.worker, session.session_id) / "exports"
|
|
963
|
-
path = Path(str(answer.get("path") or ""))
|
|
964
|
-
try:
|
|
965
|
-
resolved = path.resolve(strict=True)
|
|
966
|
-
resolved.relative_to(exports.resolve())
|
|
967
|
-
except (OSError, ValueError):
|
|
968
|
-
fail("host", f"the worker's export is not where a worker's exports go: {path}")
|
|
969
|
-
raise AssertionError # fail() never returns
|
|
970
|
-
return resolved
|
|
971
|
-
|
|
972
|
-
def upload_export(self, session_id: str, export_id: str, url: str) -> dict:
|
|
973
|
-
"""PUT the export :meth:`export` took to the presigned URL the platform
|
|
974
|
-
signed for its hash and size; the local copy goes once it is up."""
|
|
975
|
-
check_object_store_url(url, self.object_store, "url")
|
|
976
|
-
with self.using(session_id, touch=False) as session:
|
|
977
|
-
export = session.export
|
|
978
|
-
if export is None or export.export_id != export_id:
|
|
979
|
-
fail("host", f"no export {export_id} for session {session_id}; take one first")
|
|
980
|
-
raise AssertionError # fail() never returns
|
|
981
|
-
upload_file(url, export.path, export.sha256, export.bytes, export.content_type)
|
|
982
|
-
export.path.unlink(missing_ok=True)
|
|
983
|
-
with self._lock:
|
|
984
|
-
session.export = None
|
|
985
|
-
self._counters["exports_uploaded"] += 1
|
|
986
|
-
return {"uploaded": True, **export.describe()}
|
|
987
|
-
|
|
988
|
-
def renew(self, session_id: str) -> dict:
|
|
989
|
-
with self.using(session_id, touch=False) as session:
|
|
990
|
-
return self._describe(session)
|
|
991
|
-
|
|
992
|
-
def expire_leases(self, now: float | None = None) -> list[str]:
|
|
993
|
-
"""Close every session whose lease has run out; returns their ids. The
|
|
994
|
-
check and the removal happen under one lock, so a renewal that lands
|
|
995
|
-
in between keeps its session, and a session with a request in flight
|
|
996
|
-
is left for the next sweep (the request renewed it anyway)."""
|
|
997
|
-
now = time.time() if now is None else now
|
|
998
|
-
with self._lock:
|
|
999
|
-
expired = [s for s in self._sessions.values() if s.lease_until <= now and s.inflight == 0]
|
|
1000
|
-
for session in expired:
|
|
1001
|
-
del self._sessions[session.session_id]
|
|
1002
|
-
self._retire(session)
|
|
1003
|
-
for session in expired:
|
|
1004
|
-
self._finish(session, reason="expired")
|
|
1005
|
-
return [s.session_id for s in expired]
|
|
1006
|
-
|
|
1007
|
-
def _retire(self, session: Session | None) -> Session | None:
|
|
1008
|
-
"""Mark a session that has left the registry; returns it when nobody is
|
|
1009
|
-
inside it, so the caller closes the state now. Otherwise the last
|
|
1010
|
-
request out does (see :meth:`_release`). Call under the lock."""
|
|
1011
|
-
if session is None:
|
|
1012
|
-
return None
|
|
1013
|
-
session.closing = True
|
|
1014
|
-
return session if session.inflight == 0 else None
|
|
1015
|
-
|
|
1016
|
-
def _finish(self, session: Session, *, reason: str = "closed") -> None:
|
|
1017
|
-
"""Free a session that has left the registry and has nobody inside it:
|
|
1018
|
-
close its state here, or tell its worker to."""
|
|
1019
|
-
with self._lock:
|
|
1020
|
-
self._counters[f"closed.{reason}"] += 1
|
|
1021
|
-
self._session_seconds += session.seconds
|
|
1022
|
-
if session.worker is None:
|
|
1023
|
-
assert session.runtime is not None
|
|
1024
|
-
session.runtime.close()
|
|
1025
|
-
return
|
|
1026
|
-
worker = session.worker
|
|
1027
|
-
with self._lock:
|
|
1028
|
-
worker.sessions.discard(session.session_id)
|
|
1029
|
-
if not worker.sessions:
|
|
1030
|
-
worker.idle_since = time.time()
|
|
1031
|
-
try:
|
|
1032
|
-
self._worker_control(worker, "DELETE", f"/__host/sessions/{session.session_id}", None)
|
|
1033
|
-
except (WorkerGone, ContractError):
|
|
1034
|
-
pass # the worker is gone, or already forgot it: either way the session is over
|
|
1035
|
-
|
|
1036
|
-
def _worker_control(self, worker: Worker, method: str, path: str, body: dict | None) -> dict:
|
|
1037
|
-
"""One control call to a worker, its answer as this host would give it:
|
|
1038
|
-
the worker's refusal is re-raised as ours, and a worker that has died
|
|
1039
|
-
takes every session it held with it."""
|
|
1040
|
-
raw = json.dumps(body).encode("utf-8") if body is not None else b""
|
|
1041
|
-
try:
|
|
1042
|
-
status, _, payload = worker.request(method, path, {"content-type": "application/json"}, raw, control=True)
|
|
1043
|
-
except WorkerGone:
|
|
1044
|
-
self.lose_worker(worker)
|
|
1045
|
-
raise
|
|
1046
|
-
answer = json.loads(payload or b"null")
|
|
1047
|
-
if status != 200:
|
|
1048
|
-
detail = answer if isinstance(answer, dict) else {}
|
|
1049
|
-
fail(str(detail.get("error") or "host"), str(detail.get("message") or detail.get("detail") or answer))
|
|
1050
|
-
return answer if isinstance(answer, dict) else {"result": answer}
|
|
1051
|
-
|
|
1052
|
-
def lose_worker(self, worker: Worker) -> list[str]:
|
|
1053
|
-
"""A worker exited: forget it and every session it held. Returns the
|
|
1054
|
-
session ids, which are now gone from this host (the platform learns of
|
|
1055
|
-
it on the next renew)."""
|
|
1056
|
-
if self.workers is not None:
|
|
1057
|
-
self.workers.drop(worker.version_id)
|
|
1058
|
-
with self._lock:
|
|
1059
|
-
lost = [sid for sid, s in self._sessions.items() if s.worker is worker]
|
|
1060
|
-
for sid in lost:
|
|
1061
|
-
session = self._sessions.pop(sid)
|
|
1062
|
-
session.closing = True
|
|
1063
|
-
self._counters["closed.lost"] += 1
|
|
1064
|
-
self._session_seconds += session.seconds
|
|
1065
|
-
worker.sessions.clear()
|
|
1066
|
-
return lost
|
|
1067
|
-
|
|
1068
|
-
def reap_workers(self) -> list[str]:
|
|
1069
|
-
"""Stop version workers that have had no session for a while."""
|
|
1070
|
-
return self.workers.reap_idle() if self.workers is not None else []
|
|
1071
|
-
|
|
1072
|
-
def session_count(self) -> int:
|
|
1073
|
-
with self._lock:
|
|
1074
|
-
return len(self._sessions)
|
|
1075
|
-
|
|
1076
|
-
def worker_count(self) -> int:
|
|
1077
|
-
return len(self.workers._workers) if self.workers is not None else 0
|
|
1078
|
-
|
|
1079
|
-
def warm(self, version_id: str, bundle_url: str, snapshot: dict[str, str] | None) -> dict:
|
|
1080
|
-
"""Cache a version before any session asks for it, so the first open
|
|
1081
|
-
pays nothing. Idempotent: a version already here is reported, not
|
|
1082
|
-
rebuilt."""
|
|
1083
|
-
already = self.cache.has(version_id)
|
|
1084
|
-
if not already:
|
|
1085
|
-
self.cache.ensure(version_id, *self._trusted_version(bundle_url, snapshot))
|
|
1086
|
-
return {"versionId": version_id, "cached": True, "alreadyCached": already}
|
|
1087
|
-
|
|
1088
|
-
def drain(self) -> None:
|
|
1089
|
-
"""Stop taking sessions; the ones here are served until they close."""
|
|
1090
|
-
self.draining = True
|
|
1091
|
-
|
|
1092
|
-
def tally(self, key: str) -> None:
|
|
1093
|
-
with self._lock:
|
|
1094
|
-
self._counters[key] += 1
|
|
1095
|
-
|
|
1096
|
-
def stats(self) -> dict:
|
|
1097
|
-
"""What this host is and holds, for the heartbeat and ``/__host/stats``:
|
|
1098
|
-
the placement inputs (address, load, capacity, draining, which versions
|
|
1099
|
-
are already cached) plus the lifetime counters."""
|
|
1100
|
-
with self._lock:
|
|
1101
|
-
sessions = len(self._sessions)
|
|
1102
|
-
versions_in_use = sorted({s.version_id for s in self._sessions.values()})
|
|
1103
|
-
counters = dict(self._counters)
|
|
1104
|
-
session_seconds = self._session_seconds
|
|
1105
|
-
return {
|
|
1106
|
-
"hostUrl": self.advertise_url,
|
|
1107
|
-
"sessions": sessions,
|
|
1108
|
-
"capacity": self.max_sessions,
|
|
1109
|
-
"draining": self.draining,
|
|
1110
|
-
"workers": self.worker_count(),
|
|
1111
|
-
"maxWorkers": self.workers.max_workers if self.workers is not None else 0,
|
|
1112
|
-
"versions": self.cache.version_ids(),
|
|
1113
|
-
"versionsInUse": versions_in_use,
|
|
1114
|
-
"uptimeSeconds": time.time() - self.started_at,
|
|
1115
|
-
"counters": counters,
|
|
1116
|
-
"sessionSeconds": session_seconds,
|
|
1117
|
-
"pid": os.getpid(),
|
|
1118
|
-
"limits": self.limits,
|
|
1119
|
-
}
|
|
1120
|
-
|
|
1121
|
-
def metrics_text(self) -> str:
|
|
1122
|
-
"""The prometheus exposition of :meth:`stats`. ``worlds_host_sessions``
|
|
1123
|
-
is the number the fleet scales on (sessions per host, not cpu)."""
|
|
1124
|
-
stats = self.stats()
|
|
1125
|
-
counters: dict[str, int] = stats["counters"]
|
|
1126
|
-
lines = [
|
|
1127
|
-
"# TYPE worlds_host_sessions gauge",
|
|
1128
|
-
f"worlds_host_sessions {stats['sessions']}",
|
|
1129
|
-
"# TYPE worlds_host_capacity gauge",
|
|
1130
|
-
f"worlds_host_capacity {stats['capacity']}",
|
|
1131
|
-
"# TYPE worlds_host_draining gauge",
|
|
1132
|
-
f"worlds_host_draining {int(stats['draining'])}",
|
|
1133
|
-
"# TYPE worlds_host_workers gauge",
|
|
1134
|
-
f"worlds_host_workers {stats['workers']}",
|
|
1135
|
-
"# TYPE worlds_host_versions_cached gauge",
|
|
1136
|
-
f"worlds_host_versions_cached {len(stats['versions'])}",
|
|
1137
|
-
"# TYPE worlds_host_uptime_seconds gauge",
|
|
1138
|
-
f"worlds_host_uptime_seconds {stats['uptimeSeconds']:.0f}",
|
|
1139
|
-
"# TYPE worlds_host_sessions_opened_total counter",
|
|
1140
|
-
f"worlds_host_sessions_opened_total {counters.get('opened', 0)}",
|
|
1141
|
-
"# TYPE worlds_host_open_failures_total counter",
|
|
1142
|
-
f"worlds_host_open_failures_total {counters.get('open_failures', 0)}",
|
|
1143
|
-
"# TYPE worlds_host_session_seconds_total counter",
|
|
1144
|
-
f"worlds_host_session_seconds_total {stats['sessionSeconds']:.0f}",
|
|
1145
|
-
"# TYPE worlds_host_exports_total counter",
|
|
1146
|
-
f"worlds_host_exports_total {counters.get('exports', 0)}",
|
|
1147
|
-
"# TYPE worlds_host_exports_uploaded_total counter",
|
|
1148
|
-
f"worlds_host_exports_uploaded_total {counters.get('exports_uploaded', 0)}",
|
|
1149
|
-
"# TYPE worlds_host_sessions_closed_total counter",
|
|
1150
|
-
]
|
|
1151
|
-
for reason in ("closed", "expired", "lost", "replaced"):
|
|
1152
|
-
lines.append(
|
|
1153
|
-
f'worlds_host_sessions_closed_total{{reason="{reason}"}} {counters.get(f"closed.{reason}", 0)}'
|
|
1154
|
-
)
|
|
1155
|
-
lines.append("# TYPE worlds_host_requests_total counter")
|
|
1156
|
-
for surface in ("api", "ui", "browser", "proxied"):
|
|
1157
|
-
lines.append(f'worlds_host_requests_total{{surface="{surface}"}} {counters.get(f"requests.{surface}", 0)}')
|
|
1158
|
-
return "\n".join(lines) + "\n"
|
|
1159
|
-
|
|
1160
|
-
def _dependency_worlds(self, cached: CachedVersion, deps: list[DependencyRef]) -> dict[str, DependencyWorld]:
|
|
1161
|
-
"""The dependency versions a session opens beside ``cached``, cached the
|
|
1162
|
-
way the root is (a worker imports their handlers too). Every alias the
|
|
1163
|
-
root's manifest declares must be resolved, and nothing else: a version
|
|
1164
|
-
without ``[dependencies]`` opened with any is refused, as is a
|
|
1165
|
-
dependency that is not a schema world."""
|
|
1166
|
-
declared = cached.declared_dependencies
|
|
1167
|
-
if not deps:
|
|
1168
|
-
if declared:
|
|
1169
|
-
names = ", ".join(sorted(declared))
|
|
1170
|
-
fail("host", f"version {cached.version_id} declares dependencies ({names}); the open resolved none")
|
|
1171
|
-
return {}
|
|
1172
|
-
worlds: dict[str, DependencyWorld] = {}
|
|
1173
|
-
for ref in deps:
|
|
1174
|
-
dependency = declared.get(ref.alias)
|
|
1175
|
-
if dependency is None:
|
|
1176
|
-
known = ", ".join(sorted(declared)) or "none"
|
|
1177
|
-
fail("host", f"version {cached.version_id} declares no dependency '{ref.alias}' (it declares: {known})")
|
|
1178
|
-
raise AssertionError # fail() never returns
|
|
1179
|
-
dep = self.cache.ensure(ref.version_id, *self._trusted_version(ref.bundle_url, ref.snapshot))
|
|
1180
|
-
if dep.tools is None:
|
|
1181
|
-
fail("host", f"dependency '{ref.alias}' ({ref.version_id}) is an api bundle, not a schema world")
|
|
1182
|
-
raise AssertionError # fail() never returns
|
|
1183
|
-
worlds[ref.alias] = DependencyWorld.of(dependency, dep.tools)
|
|
1184
|
-
cached.served_names(worlds) # refuses a root tool that reads as a dependency's
|
|
1185
|
-
return worlds
|
|
1186
|
-
|
|
1187
|
-
@contextmanager
|
|
1188
|
-
def using(self, session_id: str, *, touch: bool = True) -> Iterator[Session]:
|
|
1189
|
-
"""The session for the duration of one request: its lease renewed (any
|
|
1190
|
-
traffic keeps it alive), its ``touched_at`` moved unless ``touch`` is
|
|
1191
|
-
off (a renewal or export observing the session, not using it), and its
|
|
1192
|
-
teardown held off until the block ends."""
|
|
1193
|
-
with self._lock:
|
|
1194
|
-
session = self._sessions.get(session_id)
|
|
1195
|
-
if session is not None:
|
|
1196
|
-
session.renew(self.lease_s)
|
|
1197
|
-
if touch:
|
|
1198
|
-
session.touched_at = time.time()
|
|
1199
|
-
session.inflight += 1
|
|
1200
|
-
if session is None:
|
|
1201
|
-
fail(UNKNOWN_SESSION, f"no session {session_id} on this host")
|
|
1202
|
-
raise AssertionError # fail() never returns
|
|
1203
|
-
try:
|
|
1204
|
-
yield session
|
|
1205
|
-
finally:
|
|
1206
|
-
self._release(session)
|
|
1207
|
-
|
|
1208
|
-
def _release(self, session: Session) -> None:
|
|
1209
|
-
with self._lock:
|
|
1210
|
-
session.inflight -= 1
|
|
1211
|
-
last_out = session.closing and session.inflight == 0
|
|
1212
|
-
if last_out:
|
|
1213
|
-
self._finish(session)
|
|
1214
|
-
|
|
1215
|
-
def owns(self, session_id: str) -> bool:
|
|
1216
|
-
with self._lock:
|
|
1217
|
-
return session_id in self._sessions
|
|
1218
|
-
|
|
1219
|
-
def _control(self, session: Session, method: str, verb: str, body: dict | None) -> dict:
|
|
1220
|
-
return self._worker_control(session.worker, method, f"/__host/sessions/{session.session_id}/{verb}", body)
|
|
1221
|
-
|
|
1222
|
-
def _verb(self, session_id: str, method: str, verb: str, body: dict | None, local: Callable[[Any], dict]) -> dict:
|
|
1223
|
-
"""One session verb: forwarded to the session's worker when it has
|
|
1224
|
-
one, else answered by its in-process runtime through ``local``."""
|
|
1225
|
-
with self.using(session_id) as session:
|
|
1226
|
-
if session.worker is not None:
|
|
1227
|
-
return self._control(session, method, verb, body)
|
|
1228
|
-
assert session.runtime is not None
|
|
1229
|
-
return local(session.runtime)
|
|
1230
|
-
|
|
1231
|
-
def tools(self, session_id: str) -> dict:
|
|
1232
|
-
return self._verb(session_id, "GET", "tools", None, lambda runtime: {"tools": runtime.served()})
|
|
1233
|
-
|
|
1234
|
-
def task(self, session_id: str) -> dict:
|
|
1235
|
-
return self._verb(session_id, "GET", "task", None, lambda runtime: runtime.task_card())
|
|
1236
|
-
|
|
1237
|
-
def reset(self, session_id: str) -> dict:
|
|
1238
|
-
return self._verb(session_id, "POST", "reset", {}, lambda runtime: runtime.reset())
|
|
1239
|
-
|
|
1240
|
-
def seed(self, session_id: str, rows: dict, mode: str = "append", redact: str = "off") -> dict:
|
|
1241
|
-
args = {"rows": rows, "mode": mode, **({"redact": redact} if redact != "off" else {})}
|
|
1242
|
-
return self._verb(session_id, "POST", "seed", {"args": args}, lambda runtime: runtime.seed(rows, mode, redact))
|
|
1243
|
-
|
|
1244
|
-
def state(self, session_id: str) -> dict:
|
|
1245
|
-
return self._verb(session_id, "GET", "state", None, lambda runtime: runtime.state())
|
|
1246
|
-
|
|
1247
|
-
def evidence(self, session_id: str) -> dict:
|
|
1248
|
-
"""The session's state as a grader reads it (rows keyed by primary key,
|
|
1249
|
-
the calls): what the platform gathers from each world of a multi-world
|
|
1250
|
-
task and hands another world's ``grade`` as ``context``."""
|
|
1251
|
-
return self._verb(session_id, "GET", "evidence", None, lambda runtime: runtime.grader_state())
|
|
1252
|
-
|
|
1253
|
-
def grade(self, session_id: str, report: Optional[str] = None, context: Optional[dict] = None) -> dict:
|
|
1254
|
-
"""Grade the session. ``report`` is the agent's own account of the run — the
|
|
1255
|
-
text a rubric judge reads; here it is put beside the state as ``report`` so a
|
|
1256
|
-
deterministic grader (assertions, verify.py) can check what the agent CLAIMED
|
|
1257
|
-
against what the calls show, without a judge. ``context`` is the other
|
|
1258
|
-
worlds' evidence the task's cross-world verifiers read (``{alias: state}``)."""
|
|
1259
|
-
body = {key: value for key, value in (("report", report), ("context", context)) if value is not None}
|
|
1260
|
-
return self._verb(session_id, "POST", "grade", body, lambda runtime: runtime.grade(report, context))
|
|
1261
|
-
|
|
1262
|
-
def call(self, session_id: str, tool: str, args: dict) -> dict:
|
|
1263
|
-
"""One of the session's declared tools by name, a dependency's as
|
|
1264
|
-
``alias.tool``; a route's name is refused (routes answer over HTTP)."""
|
|
1265
|
-
with self.using(session_id) as session:
|
|
1266
|
-
if session.worker is not None:
|
|
1267
|
-
return self._control(session, "POST", "call", {"tool": tool, "args": args})
|
|
1268
|
-
assert session.runtime is not None
|
|
1269
|
-
return session.runtime.call(tool, args)
|
|
1270
|
-
|
|
1271
|
-
|
|
1272
|
-
#: ``local.py`` binds the same rows to its own operations, so the two cannot drift.
|
|
1273
|
-
HOSTED_OPERATIONS: dict[str, Callable[..., dict]] = {
|
|
1274
|
-
"GET tools": WorldHost.tools,
|
|
1275
|
-
"GET task": WorldHost.task,
|
|
1276
|
-
"GET state": WorldHost.state,
|
|
1277
|
-
"GET evidence": WorldHost.evidence,
|
|
1278
|
-
"POST grade": WorldHost.grade,
|
|
1279
|
-
"POST reset": WorldHost.reset,
|
|
1280
|
-
"POST seed": WorldHost.seed,
|
|
1281
|
-
"POST call": WorldHost.call,
|
|
1282
|
-
"POST close": WorldHost.close,
|
|
1283
|
-
"POST renew": WorldHost.renew,
|
|
1284
|
-
"POST export": WorldHost.export,
|
|
1285
|
-
"DELETE <session>": WorldHost.close,
|
|
1286
|
-
}
|
|
1287
|
-
assert set(HOSTED_OPERATIONS) == {verb.route for verb in SESSION_VERBS}, "the host serves every session verb"
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
def public_host(host_header: str | None, worlds_domain: str) -> bool:
|
|
1291
|
-
"""True when ``Host`` names something under the worlds domain: a session,
|
|
1292
|
-
or any other label the wildcard ALB forwarded. That is the public side.
|
|
1293
|
-
Prometheus, the kubelet and the ALB health check address the pod itself
|
|
1294
|
-
(``ip:port``), which is not."""
|
|
1295
|
-
name = (host_header or "").split(":")[0].rstrip(".").lower()
|
|
1296
|
-
domain = worlds_domain.rstrip(".").lower()
|
|
1297
|
-
return bool(domain) and (name == domain or name.endswith("." + domain))
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
def session_from_host(host_header: str | None) -> tuple[str, str] | None:
|
|
1301
|
-
"""``w-<sid>-api.<domain>`` / ``w-<sid>.<domain>`` → (session id, surface).
|
|
1302
|
-
Returns None when the host does not name a session."""
|
|
1303
|
-
if not host_header:
|
|
1304
|
-
return None
|
|
1305
|
-
label = host_header.split(":")[0].split(".")[0]
|
|
1306
|
-
if not label.startswith("w-"):
|
|
1307
|
-
return None
|
|
1308
|
-
rest = label[2:]
|
|
1309
|
-
if rest.endswith("-api"):
|
|
1310
|
-
return rest[: -len("-api")], "api"
|
|
1311
|
-
if rest.endswith("-cdp"):
|
|
1312
|
-
return rest[: -len("-cdp")], "browser"
|
|
1313
|
-
return rest, "ui"
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
#: Explicit so a new ``connector.SCHEMES`` entry is refused at open rather than admitted with no credential.
|
|
1317
|
-
SUPPORTED_AUTH_SCHEMES = ("none", "header", "bearer", "basic", "query", "oauth2")
|
|
1318
|
-
|
|
1319
|
-
|
|
1320
|
-
def unsupported_auth(spec: Spec) -> str | None:
|
|
1321
|
-
if spec.scheme in SUPPORTED_AUTH_SCHEMES:
|
|
1322
|
-
return None
|
|
1323
|
-
return (
|
|
1324
|
-
f"the host cannot present auth.scheme '{spec.scheme}' to a world; it serves {', '.join(SUPPORTED_AUTH_SCHEMES)}"
|
|
1325
|
-
)
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
def world_credential(session: Session) -> tuple[dict[str, str], dict[str, str]]:
|
|
1329
|
-
"""(headers, query) carrying the world's own credential for this session.
|
|
1330
|
-
A caller that satisfies the host's session-token gate has already been
|
|
1331
|
-
authorized; the host then presents the credential the WORLD itself expects
|
|
1332
|
-
(the session token in the vendor's shape, or an oauth2 world's grant), so
|
|
1333
|
-
the emulated vendor's own auth check passes."""
|
|
1334
|
-
assert session.runtime is not None, "a session on a worker presents its credential there"
|
|
1335
|
-
return session.runtime.credential()
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
def bearer_of(headers: dict[str, str]) -> str | None:
|
|
1339
|
-
auth = headers.get("authorization", "")
|
|
1340
|
-
return auth[7:] if auth.lower().startswith("bearer ") else None
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
def vendor_shaped_token(spec: Spec, headers: dict[str, str], token: str) -> bool:
|
|
1344
|
-
"""Whether the request carries ``token`` the way the world's own auth scheme
|
|
1345
|
-
carries a credential: the declared header for ``header``, the user of a
|
|
1346
|
-
Basic authorization for ``basic``. A ``query`` scheme is not honoured here (a
|
|
1347
|
-
token in the URL outlives the request in logs); ``bearer`` is the gate's
|
|
1348
|
-
own shape and ``oauth2`` has its grant."""
|
|
1349
|
-
if spec.scheme == "header":
|
|
1350
|
-
presented = headers.get((spec.auth_name or "x-api-key").lower())
|
|
1351
|
-
return presented is not None and hmac.compare_digest(presented, token)
|
|
1352
|
-
if spec.scheme == "basic":
|
|
1353
|
-
user = basic_user(headers.get("authorization", ""))
|
|
1354
|
-
return user is not None and hmac.compare_digest(user, token)
|
|
1355
|
-
return False
|
|
1356
|
-
|
|
1357
|
-
|
|
1358
|
-
def admits(session: Session, path: str, headers: dict[str, str]) -> str | None:
|
|
1359
|
-
"""How the host's gate admits a data-plane request, or None to refuse.
|
|
1360
|
-
|
|
1361
|
-
``"session"``: the caller presented the session token as a bearer and the
|
|
1362
|
-
host will present the world's credential for it. Only the header counts: a
|
|
1363
|
-
token in the URL would outlive the request in access logs and referrers.
|
|
1364
|
-
``"world"``: an oauth2 caller doing the vendor's own dance — the grant
|
|
1365
|
-
request (the world checks ``client_secret`` against the session token) or
|
|
1366
|
-
a bearer this session's world minted — whose headers pass through as sent.
|
|
1367
|
-
"""
|
|
1368
|
-
if session.token is None:
|
|
1369
|
-
return "session"
|
|
1370
|
-
bearer = bearer_of(headers)
|
|
1371
|
-
if bearer is not None and hmac.compare_digest(bearer, session.token):
|
|
1372
|
-
return "session"
|
|
1373
|
-
if session.spec is not None and vendor_shaped_token(session.spec, headers, session.token):
|
|
1374
|
-
# The session token in the vendor's own header shape, never in the URL.
|
|
1375
|
-
return "session"
|
|
1376
|
-
if session.spec is not None and session.spec.scheme == "oauth2":
|
|
1377
|
-
if session.app is None:
|
|
1378
|
-
return "world" # the world's tokens live in its worker, which runs this same gate
|
|
1379
|
-
token_path = session.app.settings.token_path
|
|
1380
|
-
if token_path and session.app._strip(path) == token_path:
|
|
1381
|
-
return "world"
|
|
1382
|
-
if bearer is not None and session.app.router.authorized(headers, {}, session.token):
|
|
1383
|
-
return "world"
|
|
1384
|
-
return None
|
|
1385
|
-
|
|
1386
|
-
|
|
1387
|
-
class OwnerLookup(Protocol):
|
|
1388
|
-
"""Which host owns a session this replica does not: its control-plane
|
|
1389
|
-
URL, or None for a closed or unknown one. ``fresh`` bypasses whatever the
|
|
1390
|
-
lookup remembers, for a caller that has just found the remembered owner
|
|
1391
|
-
dead."""
|
|
1392
|
-
|
|
1393
|
-
def __call__(self, sid: str, *, fresh: bool = False) -> str | None: ...
|
|
1394
|
-
|
|
1395
|
-
|
|
1396
|
-
def platform_owner_lookup(platform_url: str, secret: str | None, self_url: str | None) -> OwnerLookup:
|
|
1397
|
-
"""Ask the platform which host owns a session the caller reached us for.
|
|
1398
|
-
``GET /api/public/world-sessions/{sid}/host`` is host-secret authenticated
|
|
1399
|
-
and answers ``{hostUrl}``; a 404 is a closed or unknown session. Answers are
|
|
1400
|
-
held briefly so a busy session does not cost one platform call per request.
|
|
1401
|
-
Our own address answers None: a session we do not hold but the platform
|
|
1402
|
-
says is ours is gone, not one hop away."""
|
|
1403
|
-
cache: dict[str, tuple[float, str | None]] = {}
|
|
1404
|
-
lock = threading.Lock()
|
|
1405
|
-
base = platform_url.rstrip("/")
|
|
1406
|
-
|
|
1407
|
-
def lookup(sid: str, *, fresh: bool = False) -> str | None:
|
|
1408
|
-
now = time.time()
|
|
1409
|
-
if not fresh:
|
|
1410
|
-
with lock:
|
|
1411
|
-
hit = cache.get(sid)
|
|
1412
|
-
if hit and hit[0] > now:
|
|
1413
|
-
return hit[1]
|
|
1414
|
-
owner: str | None = None
|
|
1415
|
-
req = urllib.request.Request(
|
|
1416
|
-
f"{base}/api/public/world-sessions/{urllib.parse.quote(sid, safe='')}/host",
|
|
1417
|
-
headers={"X-Host-Secret": secret or ""},
|
|
1418
|
-
)
|
|
1419
|
-
try:
|
|
1420
|
-
with urllib.request.urlopen(req, timeout=5) as r:
|
|
1421
|
-
answer = json.loads(r.read() or b"{}")
|
|
1422
|
-
url = answer.get("hostUrl") if isinstance(answer, dict) else None
|
|
1423
|
-
owner = url if isinstance(url, str) and url else None
|
|
1424
|
-
except urllib.error.HTTPError as e:
|
|
1425
|
-
if e.code != 404:
|
|
1426
|
-
raise
|
|
1427
|
-
if owner and self_url and owner.rstrip("/") == self_url.rstrip("/"):
|
|
1428
|
-
owner = None
|
|
1429
|
-
with lock:
|
|
1430
|
-
cache[sid] = (now + (OWNER_CACHE_S if owner else OWNER_MISS_S), owner)
|
|
1431
|
-
return owner
|
|
1432
|
-
|
|
1433
|
-
return lookup
|
|
1434
|
-
|
|
1435
|
-
|
|
1436
|
-
class OwnerUnreachable(Exception):
|
|
1437
|
-
"""The owner a request was proxied to did not answer: no connection, or
|
|
1438
|
-
none in time. Not an answer of any status."""
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
113
|
class BodyRefused(Exception):
|
|
1442
114
|
"""A request body the host will not read: the answer's status and error."""
|
|
1443
115
|
|
|
@@ -1456,25 +128,27 @@ def make_host_handler(
|
|
|
1456
128
|
max_body_mb: int = DEFAULT_MAX_BODY_MB,
|
|
1457
129
|
secure_cookies: bool = True,
|
|
1458
130
|
):
|
|
1459
|
-
"""The request handler class for one host
|
|
1460
|
-
|
|
1461
|
-
control plane at all rather than an open one. ``secure_cookies`` is off
|
|
131
|
+
"""The request handler class for one host; its control plane is :class:`ControlPlane`, which
|
|
132
|
+
refuses an empty ``secret`` or ``worlds_domain``. ``secure_cookies`` is off
|
|
1462
133
|
only for a host whose public surface is reached over plain http (a dev
|
|
1463
134
|
host); it is the deployment's word, not the request's, since a client
|
|
1464
135
|
writes the forwarding headers a request arrives with."""
|
|
1465
|
-
|
|
1466
|
-
raise ValueError("the world host needs a control-plane secret (WORLDS_HOST_SECRET)")
|
|
136
|
+
control = ControlPlane(host, secret, HOSTED_OPERATIONS, worlds_domain)
|
|
1467
137
|
if max_body_mb <= 0:
|
|
1468
138
|
raise ValueError(f"the request body cap must be positive (WORLDS_HOST_MAX_BODY_MB={max_body_mb})")
|
|
1469
139
|
max_body_bytes = max_body_mb * (1 << 20)
|
|
1470
140
|
|
|
1471
|
-
class Handler(
|
|
141
|
+
class Handler(OwnerProxy):
|
|
1472
142
|
server_version = "gateway-world-host/1"
|
|
1473
|
-
|
|
1474
|
-
def log_message(self, format: str, *args: object) -> None:
|
|
1475
|
-
pass
|
|
143
|
+
world_host = host
|
|
1476
144
|
|
|
1477
145
|
def _read(self) -> bytes:
|
|
146
|
+
"""The request's body, read once: a request sent on after its session moved carries it."""
|
|
147
|
+
if self._body is None:
|
|
148
|
+
self._body = self._read_body()
|
|
149
|
+
return self._body
|
|
150
|
+
|
|
151
|
+
def _read_body(self) -> bytes:
|
|
1478
152
|
declared = self.headers.get("Content-Length")
|
|
1479
153
|
if declared is None:
|
|
1480
154
|
return b""
|
|
@@ -1488,45 +162,36 @@ def make_host_handler(
|
|
|
1488
162
|
raise BodyRefused(413, f"request body over {max_body_mb} MiB")
|
|
1489
163
|
return self.rfile.read(length) if length else b""
|
|
1490
164
|
|
|
1491
|
-
def _reply(self, status: int, body: Any, extra: dict[str, str] | None = None) -> None:
|
|
1492
|
-
|
|
165
|
+
def _reply(self, status: int, body: Any, extra: dict[str, str] | None = None, errors: str = JSON_TYPE) -> None:
|
|
166
|
+
try:
|
|
167
|
+
send_response(self, status, body, extra, errors)
|
|
168
|
+
except ConnectionError:
|
|
169
|
+
self.close_connection = True
|
|
1493
170
|
|
|
1494
171
|
def _headers(self) -> dict[str, str]:
|
|
1495
172
|
return {k.lower(): v for k, v in self.headers.items()}
|
|
1496
173
|
|
|
1497
|
-
def do_GET(self):
|
|
1498
|
-
self._dispatch("GET")
|
|
1499
|
-
|
|
1500
|
-
def do_POST(self):
|
|
1501
|
-
self._dispatch("POST")
|
|
1502
|
-
|
|
1503
|
-
def do_PUT(self):
|
|
1504
|
-
self._dispatch("PUT")
|
|
1505
|
-
|
|
1506
|
-
def do_PATCH(self):
|
|
1507
|
-
self._dispatch("PATCH")
|
|
1508
|
-
|
|
1509
|
-
def do_DELETE(self):
|
|
1510
|
-
self._dispatch("DELETE")
|
|
1511
|
-
|
|
1512
174
|
def _dispatch(self, method: str) -> None:
|
|
175
|
+
self._body: bytes | None = None
|
|
1513
176
|
url = urllib.parse.urlsplit(self.path)
|
|
1514
|
-
|
|
1515
|
-
# stay on the pod address or behind the control-plane secret.
|
|
1516
|
-
if url.path == "/healthz":
|
|
177
|
+
if url.path == "/healthz" and control.internal(self.path, self.headers):
|
|
1517
178
|
return self._reply(200, {"ok": True})
|
|
1518
|
-
if url.path == "/metrics" and
|
|
179
|
+
if url.path == "/metrics" and control.serves(self.path, self.headers):
|
|
1519
180
|
payload = host.metrics_text().encode("utf-8")
|
|
1520
181
|
return self._relay(
|
|
1521
182
|
200, [("Content-Type", "text/plain; version=0.0.4; charset=utf-8")], payload, drop=FRAMING_HEADERS
|
|
1522
183
|
)
|
|
1523
184
|
try:
|
|
1524
|
-
if url.path.startswith("/__host/"):
|
|
185
|
+
if url.path.startswith("/__host/") and control.serves(self.path, self.headers):
|
|
1525
186
|
return self._control(method, url)
|
|
1526
187
|
return self._data(method, url)
|
|
188
|
+
except SessionMoved as moved:
|
|
189
|
+
headers = self._headers()
|
|
190
|
+
if hops(headers) >= MAX_HOPS:
|
|
191
|
+
return self._reply(404, CLOSED_SESSION)
|
|
192
|
+
return self._proxy(moved.session_id, moved.owner, owner_lookup, method, headers)
|
|
1527
193
|
except BodyRefused as refused:
|
|
1528
|
-
# The body was not read, so the connection cannot carry another
|
|
1529
|
-
# request: answer and hang up.
|
|
194
|
+
# The body was not read, so the connection cannot carry another request: answer and hang up.
|
|
1530
195
|
self.close_connection = True
|
|
1531
196
|
return self._reply(refused.status, {"error": refused.error}, {"Connection": "close"})
|
|
1532
197
|
except Exception as error:
|
|
@@ -1534,94 +199,27 @@ def make_host_handler(
|
|
|
1534
199
|
return self._reply(500, request_failure_body(error))
|
|
1535
200
|
|
|
1536
201
|
def _control(self, method: str, url) -> None:
|
|
1537
|
-
if not
|
|
202
|
+
if not control.admits(self.headers.get("X-Host-Secret")):
|
|
1538
203
|
return self._reply(403, {"error": "forbidden"})
|
|
1539
|
-
|
|
1540
|
-
try:
|
|
1541
|
-
body = json.loads(raw) if raw else {}
|
|
1542
|
-
except ValueError as error:
|
|
1543
|
-
return self._reply(400, {"error": "malformed json", "detail": str(error)})
|
|
1544
|
-
if not isinstance(body, dict):
|
|
1545
|
-
return self._reply(400, {"error": "malformed json", "detail": "the body must be a json object"})
|
|
1546
|
-
parts = url.path.strip("/").split("/")
|
|
1547
|
-
try:
|
|
1548
|
-
if method == "GET" and parts == ["__host", "stats"]:
|
|
1549
|
-
return self._reply(200, host.stats())
|
|
1550
|
-
if method == "POST" and parts == ["__host", "drain"]:
|
|
1551
|
-
host.drain()
|
|
1552
|
-
return self._reply(200, {"draining": True, "sessions": host.session_count()})
|
|
1553
|
-
if method == "GET" and parts == ["__host", "versions"]:
|
|
1554
|
-
return self._reply(200, {"versions": host.cache.version_ids()})
|
|
1555
|
-
if method == "POST" and parts == ["__host", "versions"]:
|
|
1556
|
-
return self._reply(
|
|
1557
|
-
200,
|
|
1558
|
-
host.warm(body["versionId"], body["bundleUrl"], body.get("snapshot")),
|
|
1559
|
-
)
|
|
1560
|
-
if method == "POST" and parts == ["__host", "sessions"]:
|
|
1561
|
-
return self._reply(
|
|
1562
|
-
200,
|
|
1563
|
-
host.open(
|
|
1564
|
-
session_id=body["sessionId"],
|
|
1565
|
-
version_id=body["versionId"],
|
|
1566
|
-
bundle_url=body["bundleUrl"],
|
|
1567
|
-
task=body.get("task"),
|
|
1568
|
-
token=body.get("token"),
|
|
1569
|
-
snapshot=body.get("snapshot"),
|
|
1570
|
-
browser=body.get("browser"),
|
|
1571
|
-
restore=body.get("restore"),
|
|
1572
|
-
task_spec=body.get("taskSpec"),
|
|
1573
|
-
dependencies=body.get("dependencies"),
|
|
1574
|
-
),
|
|
1575
|
-
)
|
|
1576
|
-
if len(parts) >= 3 and parts[:2] == ["__host", "sessions"]:
|
|
1577
|
-
sid = parts[2]
|
|
1578
|
-
verb = parts[3] if len(parts) > 3 else None
|
|
1579
|
-
if method == "PUT" and verb == "browser" and len(parts) == 4:
|
|
1580
|
-
return self._reply(200, host.set_browser(sid, body.get("browser")))
|
|
1581
|
-
if method == "POST" and verb == "export" and len(parts) == 5:
|
|
1582
|
-
url = body.get("url")
|
|
1583
|
-
if not isinstance(url, str) or not url.startswith(("https://", "http://")):
|
|
1584
|
-
return self._reply(400, {"error": "url must be the presigned upload url"})
|
|
1585
|
-
return self._reply(200, host.upload_export(sid, parts[4], url))
|
|
1586
|
-
row = session_verb(method, verb)
|
|
1587
|
-
if row is None or len(parts) > 4:
|
|
1588
|
-
return self._reply(
|
|
1589
|
-
404, unknown_verb(method, verb, hosted=True, prefix=f"/__host/sessions/{sid}/")
|
|
1590
|
-
)
|
|
1591
|
-
return self._reply(200, carried(HOSTED_OPERATIONS[row.route](host, sid, **row.read(body))))
|
|
1592
|
-
return self._reply(404, {"error": "unknown control route"})
|
|
1593
|
-
except ContractError as error:
|
|
1594
|
-
code = error.detail.get("code", "host")
|
|
1595
|
-
# 503 sends the platform to another replica; 404 is the one answer it reads as "lost".
|
|
1596
|
-
if code in ("draining", "at_capacity"):
|
|
1597
|
-
status = 503
|
|
1598
|
-
elif code == UNKNOWN_SESSION:
|
|
1599
|
-
status = 404
|
|
1600
|
-
else:
|
|
1601
|
-
status = 400
|
|
1602
|
-
return self._reply(
|
|
1603
|
-
status, {"error": code, "detail": str(error), "message": str(error.detail.get("message", ""))}
|
|
1604
|
-
)
|
|
1605
|
-
except KeyError as error:
|
|
1606
|
-
return self._reply(400, {"error": "missing field", "field": str(error)})
|
|
1607
|
-
except WorkerGone as error:
|
|
1608
|
-
return self._reply(502, {"error": "worker exited", "detail": str(error)})
|
|
204
|
+
return self._reply(*control.answer(method, url.path, self._read()))
|
|
1609
205
|
|
|
1610
206
|
def _data(self, method: str, url) -> None:
|
|
1611
207
|
headers = self._headers()
|
|
1612
208
|
named = session_from_host(headers.get("host"))
|
|
1613
209
|
if named is None:
|
|
1614
|
-
return self._reply(404,
|
|
1615
|
-
sid, surface = named
|
|
210
|
+
return self._reply(404, NO_SESSION_NAMED)
|
|
211
|
+
sid, surface = named.session_id, named.surface
|
|
1616
212
|
if not host.owns(sid):
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
owner
|
|
1620
|
-
if owner and owner_lookup:
|
|
213
|
+
looked_up = owner_lookup if owner_lookup and not hops(headers) else None
|
|
214
|
+
owner = host.forwarding.owner(sid) or (looked_up(sid) if looked_up else None)
|
|
215
|
+
if owner and hops(headers) < MAX_HOPS:
|
|
1621
216
|
host.tally("requests.proxied")
|
|
1622
217
|
return self._proxy(sid, owner, owner_lookup, method, headers)
|
|
1623
|
-
return self._reply(404,
|
|
218
|
+
return self._reply(404, CLOSED_SESSION)
|
|
219
|
+
if not host.reached_by(named):
|
|
220
|
+
return self._reply(404, NO_SESSION_NAMED)
|
|
1624
221
|
host.tally(f"requests.{surface}")
|
|
222
|
+
opened = named.capability is not None
|
|
1625
223
|
try:
|
|
1626
224
|
with host.using(sid) as session:
|
|
1627
225
|
if session.spec is None:
|
|
@@ -1631,84 +229,119 @@ def make_host_handler(
|
|
|
1631
229
|
if surface == "browser":
|
|
1632
230
|
self._serve_browser(session, url, headers)
|
|
1633
231
|
elif surface == "ui":
|
|
1634
|
-
self._serve_ui(session, method, url, headers)
|
|
232
|
+
self._serve_ui(session, method, url, headers, opened)
|
|
1635
233
|
else:
|
|
1636
|
-
self._serve(session, method, url, headers)
|
|
1637
|
-
except ContractError:
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
234
|
+
self._serve(session, method, url, headers, opened)
|
|
235
|
+
except ContractError as error:
|
|
236
|
+
owner = host.forwarding.owner(sid) if error.detail.get("code") == UNKNOWN_SESSION else None
|
|
237
|
+
if owner is not None:
|
|
238
|
+
raise SessionMoved(sid, owner) from error
|
|
239
|
+
self._refused(error)
|
|
240
|
+
|
|
241
|
+
def _refused(self, error: ContractError) -> None:
|
|
242
|
+
if error.detail.get("code") == UNKNOWN_SESSION:
|
|
243
|
+
return self._reply(404, CLOSED_SESSION)
|
|
244
|
+
self._reply(*refusal(error))
|
|
245
|
+
|
|
246
|
+
def _serve(self, session: Session, method: str, url, headers: dict[str, str], opened: bool) -> None:
|
|
247
|
+
preflight = session.preflight(method, headers)
|
|
248
|
+
if preflight is not None:
|
|
249
|
+
return self._reply(*preflight)
|
|
250
|
+
assert session.door is not None, "a session with an http surface has a door"
|
|
251
|
+
door = session.door.opened() if opened else session.door
|
|
252
|
+
entry = door.session_entry(method, url.path, url.query, headers)
|
|
253
|
+
if entry is None:
|
|
254
|
+
status, body, errors = door.session_refusal
|
|
255
|
+
return self._reply(status, body, None, errors)
|
|
256
|
+
if door.by_platform_bearer(entry, headers):
|
|
257
|
+
host.tally("requests.platform_bearer")
|
|
258
|
+
headers = door.with_platform_header(headers)
|
|
1645
259
|
if session.worker is not None:
|
|
1646
260
|
return self._on_worker(session, method, headers)
|
|
1647
|
-
self._serve_api(session, method, self.path, headers,
|
|
261
|
+
self._serve_api(session, method, self.path, headers, entry, Via.API, opened)
|
|
1648
262
|
|
|
1649
263
|
def _on_worker(self, session: Session, method: str, headers: dict[str, str]) -> None:
|
|
1650
264
|
"""Relay the request as it came, Host and path unchanged, to the
|
|
1651
|
-
session's version worker
|
|
1652
|
-
door (api or ui) on its
|
|
1653
|
-
|
|
1654
|
-
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
host.
|
|
1658
|
-
return self._reply(502, {"error": "worker exited", "detail": str(error)})
|
|
265
|
+
session's version worker, once it is admitted on the version: the
|
|
266
|
+
worker runs the same gate and picks the same door (api or ui) on its
|
|
267
|
+
own copy of the session."""
|
|
268
|
+
assert host.placement is not None
|
|
269
|
+
raw = self._read()
|
|
270
|
+
with host.admitted(session, HOP_HEADER in headers):
|
|
271
|
+
status, answer_headers, payload = host.placement.relay(session, method, self.path, headers, raw)
|
|
1659
272
|
return self._relay(status, answer_headers, payload, drop=FRAMING_HEADERS)
|
|
1660
273
|
|
|
1661
274
|
def _serve_api(
|
|
1662
|
-
self,
|
|
275
|
+
self,
|
|
276
|
+
session: Session,
|
|
277
|
+
method: str,
|
|
278
|
+
target: str,
|
|
279
|
+
headers: dict[str, str],
|
|
280
|
+
entry: Entry,
|
|
281
|
+
via: Via,
|
|
282
|
+
opened: bool = False,
|
|
1663
283
|
) -> None:
|
|
1664
284
|
"""Answer ``target`` (a path plus query) from the session's world in
|
|
1665
285
|
this process, once the gate has admitted it; the call is logged as
|
|
1666
|
-
having come in by ``via``, the door this host took it in by.
|
|
1667
|
-
|
|
286
|
+
having come in by ``via``, the door this host took it in by. A caller
|
|
287
|
+
the platform admitted is presented to the world with the session's
|
|
288
|
+
credential the vendor's way, and the platform's own header never
|
|
289
|
+
reaches it, so the world's own check and its handlers see what the
|
|
290
|
+
vendor's client would send. A caller on an open session's capability
|
|
291
|
+
host is presented nothing: world code never hands the session token to
|
|
292
|
+
someone who holds only the capability. A link the world signs points at
|
|
293
|
+
the session's own api host, where its signature alone admits it."""
|
|
294
|
+
assert session.app is not None and session.door is not None
|
|
1668
295
|
raw = self._read()
|
|
1669
|
-
world_headers = headers
|
|
1670
|
-
if
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
credential_headers, credential_query = world_credential(session)
|
|
1674
|
-
world_headers = {**headers, **credential_headers}
|
|
296
|
+
world_headers = {name: value for name, value in headers.items() if name != PLATFORM_HEADER}
|
|
297
|
+
if entry is Entry.PLATFORM and not opened:
|
|
298
|
+
credential_headers, credential_query = session.door.present()
|
|
299
|
+
world_headers = {**world_headers, **credential_headers}
|
|
1675
300
|
target = with_query(target, credential_query)
|
|
1676
|
-
default_host =
|
|
1677
|
-
|
|
301
|
+
default_host = session.api_host(worlds_domain, opened)
|
|
302
|
+
link_host = session.api_host(worlds_domain, opened=False)
|
|
303
|
+
with host.admitted(session, HOP_HEADER in headers):
|
|
304
|
+
answer = session.app.handle(
|
|
305
|
+
method, target, world_headers, raw, default_host, via=via, opened=opened, link_host=link_host
|
|
306
|
+
)
|
|
307
|
+
self._reply(*answer, session.app.settings.error_content_type)
|
|
1678
308
|
|
|
1679
|
-
def _serve_ui(self, session: Session, method: str, url, headers: dict[str, str]) -> None:
|
|
309
|
+
def _serve_ui(self, session: Session, method: str, url, headers: dict[str, str], opened: bool) -> None:
|
|
1680
310
|
ui = session.ui
|
|
1681
311
|
if ui is None:
|
|
1682
312
|
return self._reply(404, {"error": "this world has no static ui"})
|
|
1683
|
-
token = session.token
|
|
313
|
+
token = None if opened else session.token
|
|
1684
314
|
if token is not None and matches(query_token(url.query), token):
|
|
1685
315
|
return self._trade_token_for_cookie(url, token)
|
|
1686
|
-
if token is not None and not (
|
|
316
|
+
if token is not None and not (
|
|
317
|
+
matches(cookie_token(headers), token) or matches(_BEARER.read(headers, {}), token)
|
|
318
|
+
):
|
|
1687
319
|
return self._reply(401, {"error": "invalid session token"})
|
|
1688
320
|
route = api_target(ui, url.path)
|
|
1689
321
|
if route is None:
|
|
1690
|
-
return self._serve_static(ui, method, url)
|
|
322
|
+
return self._serve_static(ui, method, url, opened)
|
|
1691
323
|
if cross_site(headers, lambda origin: urllib.parse.urlsplit(origin).netloc == headers.get("host")):
|
|
1692
324
|
return self._reply(403, {"error": "the ui's api prefix answers its own page, not another site"})
|
|
1693
325
|
if session.worker is not None:
|
|
1694
326
|
return self._on_worker(session, method, headers)
|
|
1695
327
|
target = urllib.parse.urlunsplit(("", "", route, url.query, ""))
|
|
1696
328
|
api_headers = {k: v for k, v in headers.items() if k not in ("cookie", "authorization")}
|
|
1697
|
-
api_headers["host"] =
|
|
1698
|
-
|
|
1699
|
-
api_headers["authorization"] = f"Bearer {token}"
|
|
1700
|
-
return self._serve_api(session, method, target, api_headers, "session", Via.UI)
|
|
329
|
+
api_headers["host"] = session.api_host(worlds_domain, opened)
|
|
330
|
+
return self._serve_api(session, method, target, api_headers, Entry.PLATFORM, Via.UI, opened)
|
|
1701
331
|
|
|
1702
332
|
def _trade_token_for_cookie(self, url, token: str) -> None:
|
|
1703
333
|
landing = without_query_token(url)
|
|
1704
334
|
cookie = session_cookie(token, secure=secure_cookies, max_age_s=host.lease_s)
|
|
1705
335
|
self._reply(303, {"location": landing}, {"Location": landing, "Set-Cookie": cookie})
|
|
1706
336
|
|
|
1707
|
-
def _serve_static(self, ui: UiSurface, method: str, url) -> None:
|
|
337
|
+
def _serve_static(self, ui: UiSurface, method: str, url, opened: bool) -> None:
|
|
338
|
+
"""A file of the ui; on a capability host with no ``Referer``, so a page that
|
|
339
|
+
loads another site's resource never hands it the capability."""
|
|
1708
340
|
if method != "GET":
|
|
1709
341
|
return self._reply(405, {"error": "the static ui answers GET only"})
|
|
1710
342
|
status, static_headers, payload = static_file(ui, url.path)
|
|
1711
|
-
|
|
343
|
+
sealed = {**static_headers, **(NO_REFERRER if opened else {})}
|
|
344
|
+
self._relay(status, sealed.items(), payload, drop=FRAMING_HEADERS)
|
|
1712
345
|
|
|
1713
346
|
def _serve_browser(self, session: Session, url, headers: dict[str, str]) -> None:
|
|
1714
347
|
"""``/cdp/<session token>`` and ``/live/<grant>`` on the session's
|
|
@@ -1726,8 +359,7 @@ def make_host_handler(
|
|
|
1726
359
|
if kind == "cdp":
|
|
1727
360
|
if not matches(credential, token):
|
|
1728
361
|
return self._reply(401, {"error": "invalid session token"})
|
|
1729
|
-
# The relay decodes the path segment
|
|
1730
|
-
# '=' (openssl rand -base64) must travel percent-encoded.
|
|
362
|
+
# The relay decodes the path segment: a token with '/' or '=' must travel percent-encoded.
|
|
1731
363
|
upstream_path = f"/cdp/{urllib.parse.quote(relay.token, safe='')}"
|
|
1732
364
|
else:
|
|
1733
365
|
scope = verify_live_grant(credential, token)
|
|
@@ -1742,146 +374,17 @@ def make_host_handler(
|
|
|
1742
374
|
except OSError as error:
|
|
1743
375
|
self._reply(502, {"error": "browser unreachable", "detail": str(error)})
|
|
1744
376
|
|
|
1745
|
-
def _proxy(self, sid: str, owner: str, lookup: OwnerLookup, method: str, headers: dict[str, str]) -> None:
|
|
1746
|
-
"""Send the request on to ``owner``. An owner that does not answer
|
|
1747
|
-
at all may be a host that died with the session since recovered
|
|
1748
|
-
elsewhere, so the platform is asked once more, uncached, and the
|
|
1749
|
-
request goes to the owner it names now; only then is it a 502.
|
|
1750
|
-
The retry stays one hop (the hop header goes with it)."""
|
|
1751
|
-
websocket = headers.get("upgrade", "").lower() == "websocket"
|
|
1752
|
-
raw = b"" if websocket else self._read()
|
|
1753
|
-
try:
|
|
1754
|
-
return self._forward(owner, method, headers, raw, websocket)
|
|
1755
|
-
except OwnerUnreachable as stale:
|
|
1756
|
-
current = lookup(sid, fresh=True)
|
|
1757
|
-
if not current or current.rstrip("/") == owner.rstrip("/"):
|
|
1758
|
-
return self._reply(502, {"error": "owner unreachable", "detail": str(stale)})
|
|
1759
|
-
host.tally("requests.rerouted")
|
|
1760
|
-
try:
|
|
1761
|
-
self._forward(current, method, headers, raw, websocket)
|
|
1762
|
-
except OwnerUnreachable as error:
|
|
1763
|
-
self._reply(502, {"error": "owner unreachable", "detail": str(error)})
|
|
1764
|
-
|
|
1765
|
-
def _forward(self, owner: str, method: str, headers: dict[str, str], raw: bytes, websocket: bool) -> None:
|
|
1766
|
-
"""One attempt at ``owner``; raises :class:`OwnerUnreachable` when it
|
|
1767
|
-
did not answer, before anything has been written to the client."""
|
|
1768
|
-
if websocket:
|
|
1769
|
-
# A browser session on another host: the owner runs the
|
|
1770
|
-
# credential check; this hop only carries bytes. ``tunnel``
|
|
1771
|
-
# fails before the 101 or not at all.
|
|
1772
|
-
owner_ws = "ws://" + owner.split("://", 1)[-1].rstrip("/")
|
|
1773
|
-
self.close_connection = True
|
|
1774
|
-
try:
|
|
1775
|
-
tunnel(self.connection, owner_ws, self.path, {**headers, HOP_HEADER: "1"}, host=headers.get("host"))
|
|
1776
|
-
except OSError as error:
|
|
1777
|
-
raise OwnerUnreachable(str(error)) from error
|
|
1778
|
-
return
|
|
1779
|
-
target = f"{owner.rstrip('/')}{self.path}"
|
|
1780
|
-
forward = {k: v for k, v in headers.items() if k not in ("host", "content-length")}
|
|
1781
|
-
forward["host"] = headers.get("host", "")
|
|
1782
|
-
forward[HOP_HEADER] = "1"
|
|
1783
|
-
req = urllib.request.Request(target, data=raw or None, method=method, headers=forward)
|
|
1784
|
-
try:
|
|
1785
|
-
with _PROXY_OPENER.open(req, timeout=60) as r:
|
|
1786
|
-
self._relay(r.status, r.headers.items(), r.read(), drop=PROXIED_DROP_HEADERS)
|
|
1787
|
-
except urllib.error.HTTPError as e:
|
|
1788
|
-
self._relay(e.code, e.headers.items(), e.read(), drop=PROXIED_DROP_HEADERS)
|
|
1789
|
-
except (OSError, http.client.HTTPException) as error:
|
|
1790
|
-
# URLError is an OSError: no connection, or none in time.
|
|
1791
|
-
raise OwnerUnreachable(str(error)) from error
|
|
1792
|
-
|
|
1793
|
-
def _relay(
|
|
1794
|
-
self,
|
|
1795
|
-
status: int,
|
|
1796
|
-
headers: Iterable[tuple[str, str]],
|
|
1797
|
-
payload: bytes,
|
|
1798
|
-
*,
|
|
1799
|
-
drop: frozenset[str],
|
|
1800
|
-
) -> None:
|
|
1801
|
-
"""Write another process's answer through as it came: same status,
|
|
1802
|
-
same body bytes, every header the world set except ``drop`` (the
|
|
1803
|
-
framing, which is this connection's, and from an owner the
|
|
1804
|
-
hop-by-hop headers too). The world's contract, its ``page_headers``
|
|
1805
|
-
included, reads the same from every replica."""
|
|
1806
|
-
self.send_response(status)
|
|
1807
|
-
content_type = None
|
|
1808
|
-
for name, value in headers:
|
|
1809
|
-
lower = name.lower()
|
|
1810
|
-
if lower == "content-type":
|
|
1811
|
-
content_type = value
|
|
1812
|
-
elif lower not in drop:
|
|
1813
|
-
self.send_header(name, value)
|
|
1814
|
-
self.send_header("Content-Type", content_type or "application/json")
|
|
1815
|
-
self.send_header("Content-Length", str(len(payload)))
|
|
1816
|
-
self.end_headers()
|
|
1817
|
-
self.wfile.write(payload)
|
|
1818
|
-
|
|
1819
377
|
return Handler
|
|
1820
378
|
|
|
1821
379
|
|
|
1822
|
-
def start_lease_sweeper(world_host: WorldHost, interval_s: float = LEASE_SWEEP_S) -> threading.Event:
|
|
1823
|
-
"""Expire leases every ``interval_s`` until the returned event is set."""
|
|
1824
|
-
stop = threading.Event()
|
|
1825
|
-
|
|
1826
|
-
def sweep() -> None:
|
|
1827
|
-
while not stop.wait(interval_s):
|
|
1828
|
-
for sid in world_host.expire_leases():
|
|
1829
|
-
print(
|
|
1830
|
-
f"world host: lease expired, closed session {sid}",
|
|
1831
|
-
file=sys.stderr,
|
|
1832
|
-
flush=True,
|
|
1833
|
-
)
|
|
1834
|
-
for version_id in world_host.reap_workers():
|
|
1835
|
-
print(
|
|
1836
|
-
f"world host: stopped idle worker for version {version_id}",
|
|
1837
|
-
file=sys.stderr,
|
|
1838
|
-
flush=True,
|
|
1839
|
-
)
|
|
1840
|
-
|
|
1841
|
-
threading.Thread(target=sweep, name="world-host-leases", daemon=True).start()
|
|
1842
|
-
return stop
|
|
1843
|
-
|
|
1844
|
-
|
|
1845
|
-
def heartbeat_once(world_host: WorldHost, platform_url: str, secret: str, timeout_s: float = 5.0) -> bool:
|
|
1846
|
-
"""Tell the platform what this host is and holds (:meth:`WorldHost.stats`).
|
|
1847
|
-
The platform's registry is what places new sessions: a host it has not
|
|
1848
|
-
heard from lately gets none. Returns whether the platform took it."""
|
|
1849
|
-
body = json.dumps(world_host.stats()).encode("utf-8")
|
|
1850
|
-
req = urllib.request.Request(
|
|
1851
|
-
f"{platform_url.rstrip('/')}/api/public/world-hosts",
|
|
1852
|
-
data=body,
|
|
1853
|
-
method="POST",
|
|
1854
|
-
headers={"content-type": "application/json", "x-host-secret": secret},
|
|
1855
|
-
)
|
|
1856
|
-
try:
|
|
1857
|
-
with urllib.request.urlopen(req, timeout=timeout_s) as r:
|
|
1858
|
-
return 200 <= r.status < 300
|
|
1859
|
-
except (urllib.error.URLError, OSError, ValueError) as error:
|
|
1860
|
-
print(f"world host: heartbeat failed: {error}", file=sys.stderr, flush=True)
|
|
1861
|
-
return False
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
def start_heartbeat(
|
|
1865
|
-
world_host: WorldHost, platform_url: str, secret: str, interval_s: float = HEARTBEAT_S
|
|
1866
|
-
) -> threading.Event:
|
|
1867
|
-
"""Heartbeat every ``interval_s`` until the returned event is set; the
|
|
1868
|
-
first beat goes out at once so a fresh replica is placeable immediately,
|
|
1869
|
-
and a last one goes out on stop so the registry sees it draining."""
|
|
1870
|
-
stop = threading.Event()
|
|
1871
|
-
|
|
1872
|
-
def beat() -> None:
|
|
1873
|
-
heartbeat_once(world_host, platform_url, secret)
|
|
1874
|
-
while not stop.wait(interval_s):
|
|
1875
|
-
heartbeat_once(world_host, platform_url, secret)
|
|
1876
|
-
heartbeat_once(world_host, platform_url, secret)
|
|
1877
|
-
|
|
1878
|
-
threading.Thread(target=beat, name="world-host-heartbeat", daemon=True).start()
|
|
1879
|
-
return stop
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
380
|
class HostServer(ThreadingHTTPServer):
|
|
1883
381
|
"""The listening socket plus what shutdown has to stop: the lease sweeper,
|
|
1884
|
-
the heartbeat and the version workers.
|
|
382
|
+
the heartbeat and the version workers. Its backlog holds a burst of
|
|
383
|
+
connections; their requests wait their turn in :class:`Admission`. It
|
|
384
|
+
counts the requests it is answering, so a drain stops only once the
|
|
385
|
+
requests sent on to the hosts its sessions moved to are answered."""
|
|
386
|
+
|
|
387
|
+
request_queue_size = 1024
|
|
1885
388
|
|
|
1886
389
|
def __init__(
|
|
1887
390
|
self,
|
|
@@ -1892,79 +395,104 @@ class HostServer(ThreadingHTTPServer):
|
|
|
1892
395
|
heartbeat: tuple[str, str] | None = None,
|
|
1893
396
|
heartbeat_s: float = HEARTBEAT_S,
|
|
1894
397
|
drain_s: float = DEFAULT_DRAIN_S,
|
|
398
|
+
ledger: LedgerFlusher | None = None,
|
|
399
|
+
handoff: HandoffDoor | None = None,
|
|
1895
400
|
):
|
|
1896
401
|
super().__init__(address, handler)
|
|
1897
402
|
self.world_host = world_host
|
|
1898
403
|
self.lease_sweeper = start_lease_sweeper(world_host)
|
|
1899
|
-
self.heartbeat =
|
|
1900
|
-
|
|
404
|
+
self.heartbeat = Heartbeat(world_host, *heartbeat, interval_s=heartbeat_s).start() if heartbeat else None
|
|
405
|
+
world_host.ledger = ledger.start() if ledger else NoLedger()
|
|
406
|
+
self.handoff = DrainHandoff(world_host, handoff, drain_s)
|
|
407
|
+
self._answering = 0
|
|
408
|
+
self._draining = False
|
|
409
|
+
self._lock = threading.Lock()
|
|
410
|
+
|
|
411
|
+
def process_request_thread(self, request, client_address) -> None:
|
|
412
|
+
with self._lock:
|
|
413
|
+
self._answering += 1
|
|
414
|
+
try:
|
|
415
|
+
super().process_request_thread(request, client_address)
|
|
416
|
+
finally:
|
|
417
|
+
with self._lock:
|
|
418
|
+
self._answering -= 1
|
|
419
|
+
|
|
420
|
+
def idle(self) -> bool:
|
|
421
|
+
with self._lock:
|
|
422
|
+
return self._answering == 0
|
|
1901
423
|
|
|
1902
424
|
def drain(self) -> None:
|
|
1903
|
-
"""SIGTERM: refuse new sessions,
|
|
1904
|
-
|
|
1905
|
-
so ``serve_forever`` keeps answering
|
|
425
|
+
"""SIGTERM: refuse new sessions, tell the platform at once and keep beating
|
|
426
|
+
(a draining host is alive), hand every session off (:class:`DrainHandoff`),
|
|
427
|
+
then stop. Runs off the signal thread so ``serve_forever`` keeps answering
|
|
428
|
+
meanwhile; a second call does nothing."""
|
|
429
|
+
with self._lock:
|
|
430
|
+
if self._draining:
|
|
431
|
+
return
|
|
432
|
+
self._draining = True
|
|
1906
433
|
self.world_host.drain()
|
|
1907
434
|
if self.heartbeat is not None:
|
|
1908
|
-
self.heartbeat.
|
|
435
|
+
self.heartbeat.wake()
|
|
436
|
+
threading.Thread(target=self._hand_off_then_stop, name="world-host-drain", daemon=True).start()
|
|
1909
437
|
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
438
|
+
def _hand_off_then_stop(self) -> None:
|
|
439
|
+
try:
|
|
440
|
+
self.handoff.run(self.idle)
|
|
441
|
+
finally:
|
|
1914
442
|
self.shutdown()
|
|
1915
443
|
|
|
1916
|
-
threading.Thread(target=wait_then_stop, name="world-host-drain", daemon=True).start()
|
|
1917
|
-
|
|
1918
444
|
|
|
1919
445
|
def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0") -> HostServer:
|
|
1920
446
|
"""The host as deployed, configured from the environment:
|
|
1921
447
|
|
|
1922
|
-
``WORLDS_HOST_SECRET`` gates the control plane; ``WORLDS_INGRESS_DOMAIN`` is
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
the platform
|
|
1926
|
-
|
|
1927
|
-
``
|
|
1928
|
-
``
|
|
1929
|
-
``
|
|
1930
|
-
|
|
1931
|
-
``
|
|
1932
|
-
``
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
``
|
|
1938
|
-
|
|
1939
|
-
``
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
http (a dev host: the ui cookie is not marked ``Secure``) and
|
|
1943
|
-
``WORLDS_HOST_WORKERS=0`` runs without workers (handler worlds refused)."""
|
|
448
|
+
``WORLDS_HOST_SECRET`` gates the control plane; ``WORLDS_INGRESS_DOMAIN`` is the domain the session hostnames
|
|
449
|
+
live under (required); ``WORLDS_HOST_ADVERTISE_URL`` is this replica's own address (``http://<pod ip>:<port>``),
|
|
450
|
+
returned on open so the platform pins the session to it; ``WORLDS_PLATFORM_URL`` lets a non-owner replica ask
|
|
451
|
+
the platform who owns a session and proxy one hop, and is where the request ledger goes; ``WORLDS_HOST_LEASE_S``
|
|
452
|
+
is how long an untouched session lives; ``WORLDS_HOST_MAX_SESSIONS`` is how many it takes before answering 503;
|
|
453
|
+
``WORLDS_HOST_HEARTBEAT_S`` is how often it reports to the platform (with ``WORLDS_PLATFORM_URL`` and an
|
|
454
|
+
advertised address, else no heartbeat), and ``WORLDS_HOST_BUILD`` which pod template it runs, so a session
|
|
455
|
+
handed off goes to the newest one; ``WORLDS_HOST_DRAIN_S`` is how long a SIGTERM's drain may take to hand its
|
|
456
|
+
sessions to other hosts through the platform (with an advertised address) and serve what could not move;
|
|
457
|
+
``WORLDS_WORKER_MEMORY_MB`` caps each version worker's address space, ``WORLDS_WORKER_BUDGET_MB`` is the memory
|
|
458
|
+
all of them together are held to and ``WORLDS_WORKER_FOOTPRINT_MB`` what each is counted at against it (unset,
|
|
459
|
+
its whole address-space cap; their count is the quotient, and one more version answers 503 ``at_capacity``),
|
|
460
|
+
``WORLDS_HOST_OBJECT_STORE`` is where every presigned URL it is handed may point (required; ``any`` for a dev
|
|
461
|
+
host), ``WORLDS_HOST_MAX_BUNDLE_MB`` is the largest version bundle the cache accepts,
|
|
462
|
+
``WORLDS_HOST_MAX_BODY_MB`` the largest request body it reads, ``WORLDS_HOST_LEDGER_SPILL_DIR`` (under the cache
|
|
463
|
+
root) and ``WORLDS_HOST_LEDGER_SPILL_MB`` are where and how much of the request ledger waits on disk for the
|
|
464
|
+
platform, ``WORLDS_HOST_PLAIN_HTTP=1`` says the public surface is reached over plain http (a dev host: the ui
|
|
465
|
+
cookie is not marked ``Secure``) and ``WORLDS_HOST_WORKERS=0`` runs without workers (handler worlds refused)."""
|
|
466
|
+
if not seal_process() and sys.platform.startswith("linux"):
|
|
467
|
+
raise SystemExit("world host: cannot make this process non-dumpable; its workers could read its secret")
|
|
1944
468
|
secret = os.environ.get("WORLDS_HOST_SECRET") or ""
|
|
1945
469
|
if not secret:
|
|
1946
470
|
raise SystemExit("world host: WORLDS_HOST_SECRET is unset; refusing to start without a control-plane secret")
|
|
471
|
+
spill_dir, spill_mb = os.environ.get("WORLDS_HOST_LEDGER_SPILL_DIR"), os.environ.get("WORLDS_HOST_LEDGER_SPILL_MB")
|
|
472
|
+
platform_url = os.environ.get("WORLDS_PLATFORM_URL") or None
|
|
1947
473
|
try:
|
|
1948
474
|
object_store = parse_origins(os.environ.get("WORLDS_HOST_OBJECT_STORE"))
|
|
1949
|
-
|
|
475
|
+
worlds_domain = ControlPlane.domain_of(os.environ.get("WORLDS_INGRESS_DOMAIN"))
|
|
476
|
+
spill = LedgerSpill.at(Path(cache_root), spill_dir, spill_mb, secret)
|
|
477
|
+
spill = spill.open() if platform_url else spill
|
|
478
|
+
except (ContractError, ValueError, OSError) as error:
|
|
1950
479
|
raise SystemExit(f"world host: {error}; refusing to start") from error
|
|
480
|
+
# Nothing this process starts inherits them; /proc/<pid>/environ keeps the exec-time copy, closed by the seal.
|
|
481
|
+
os.environ.pop("WORLDS_HOST_SECRET", None)
|
|
482
|
+
os.environ.pop("WORLDS_HOST_OBJECT_STORE", None)
|
|
1951
483
|
advertise = os.environ.get("WORLDS_HOST_ADVERTISE_URL") or None
|
|
1952
|
-
platform_url = os.environ.get("WORLDS_PLATFORM_URL") or None
|
|
1953
484
|
lease_s = float(os.environ.get("WORLDS_HOST_LEASE_S") or DEFAULT_LEASE_S)
|
|
1954
|
-
worlds_domain = os.environ.get("WORLDS_INGRESS_DOMAIN", "worlds.local")
|
|
1955
485
|
workers = None
|
|
486
|
+
footprint = os.environ.get("WORLDS_WORKER_FOOTPRINT_MB")
|
|
1956
487
|
if os.environ.get("WORLDS_HOST_WORKERS", "1") != "0":
|
|
1957
488
|
workers = WorkerPool(
|
|
1958
489
|
Path(cache_root),
|
|
1959
490
|
worlds_domain=worlds_domain,
|
|
1960
491
|
memory_mb=int(os.environ.get("WORLDS_WORKER_MEMORY_MB") or DEFAULT_WORKER_MEMORY_MB),
|
|
1961
492
|
budget_mb=int(os.environ.get("WORLDS_WORKER_BUDGET_MB") or DEFAULT_WORKER_BUDGET_MB),
|
|
1962
|
-
footprint_mb=(
|
|
1963
|
-
int(os.environ["WORLDS_WORKER_FOOTPRINT_MB"])
|
|
1964
|
-
if os.environ.get("WORLDS_WORKER_FOOTPRINT_MB")
|
|
1965
|
-
else DEFAULT_WORKER_FOOTPRINT_MB
|
|
1966
|
-
),
|
|
493
|
+
footprint_mb=int(footprint) if footprint else DEFAULT_WORKER_FOOTPRINT_MB,
|
|
1967
494
|
)
|
|
495
|
+
owners = PlatformOwners(platform_url, secret, advertise) if platform_url else None
|
|
1968
496
|
world_host = WorldHost(
|
|
1969
497
|
Path(cache_root),
|
|
1970
498
|
lease_s=lease_s,
|
|
@@ -1973,22 +501,27 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
|
|
|
1973
501
|
max_sessions=int(os.environ.get("WORLDS_HOST_MAX_SESSIONS") or DEFAULT_MAX_SESSIONS),
|
|
1974
502
|
max_bundle_mb=int(os.environ.get("WORLDS_HOST_MAX_BUNDLE_MB") or DEFAULT_MAX_BUNDLE_MB),
|
|
1975
503
|
object_store=object_store,
|
|
504
|
+
build=os.environ.get("WORLDS_HOST_BUILD") or None,
|
|
505
|
+
owners=owners,
|
|
1976
506
|
)
|
|
1977
507
|
handler = make_host_handler(
|
|
1978
508
|
world_host,
|
|
1979
509
|
secret=secret,
|
|
1980
510
|
worlds_domain=worlds_domain,
|
|
1981
|
-
owner_lookup=
|
|
511
|
+
owner_lookup=owners,
|
|
1982
512
|
max_body_mb=int(os.environ.get("WORLDS_HOST_MAX_BODY_MB") or DEFAULT_MAX_BODY_MB),
|
|
1983
513
|
secure_cookies=os.environ.get("WORLDS_HOST_PLAIN_HTTP", "0") != "1",
|
|
1984
514
|
)
|
|
515
|
+
sink = PlatformLedger(platform_url, secret, host_url=advertise) if platform_url else None
|
|
1985
516
|
return HostServer(
|
|
1986
517
|
(host_addr, port),
|
|
1987
518
|
handler,
|
|
1988
519
|
world_host,
|
|
1989
520
|
heartbeat=(platform_url, secret) if platform_url and advertise else None,
|
|
521
|
+
ledger=LedgerFlusher(world_host, sink, spill) if sink else None,
|
|
1990
522
|
heartbeat_s=float(os.environ.get("WORLDS_HOST_HEARTBEAT_S") or HEARTBEAT_S),
|
|
1991
523
|
drain_s=float(os.environ.get("WORLDS_HOST_DRAIN_S") or DEFAULT_DRAIN_S),
|
|
524
|
+
handoff=PlatformHandoff(platform_url, secret) if platform_url and advertise else None,
|
|
1992
525
|
)
|
|
1993
526
|
|
|
1994
527
|
|
|
@@ -2001,7 +534,7 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
2001
534
|
args = parser.parse_args(argv)
|
|
2002
535
|
server = serve(args.cache_root, port=args.port)
|
|
2003
536
|
print(
|
|
2004
|
-
f"world host listening on :{
|
|
537
|
+
f"world host listening on :{server.server_address[1]}, cache {args.cache_root}",
|
|
2005
538
|
file=sys.stderr,
|
|
2006
539
|
flush=True,
|
|
2007
540
|
)
|
|
@@ -2009,9 +542,10 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
2009
542
|
try:
|
|
2010
543
|
server.serve_forever()
|
|
2011
544
|
finally:
|
|
545
|
+
server.world_host.ledger.stop(settle=server.world_host.finishing.settle)
|
|
2012
546
|
server.lease_sweeper.set()
|
|
2013
547
|
if server.heartbeat is not None:
|
|
2014
|
-
server.heartbeat.
|
|
548
|
+
server.heartbeat.stop()
|
|
2015
549
|
if server.world_host.workers is not None:
|
|
2016
550
|
server.world_host.workers.stop_all()
|
|
2017
551
|
server.server_close()
|