@withgateway/sdk 0.6.0 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/index.d.mts +581 -0
- package/dist/agents/index.d.ts +581 -0
- package/dist/agents/index.js +1473 -0
- package/dist/agents/index.js.map +1 -0
- package/dist/agents/index.mjs +1424 -0
- package/dist/agents/index.mjs.map +1 -0
- package/dist/apiError-DHpfbmdx.d.mts +17 -0
- package/dist/apiError-DHpfbmdx.d.ts +17 -0
- package/dist/cli.js +11611 -6845
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +10090 -5325
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +3 -1
- package/dist/files/index.d.ts +3 -1
- package/dist/files/index.js +371 -102
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +371 -102
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.js +346 -136
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +346 -136
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-LA-PMRJT.d.ts → index-DIN14Lhg.d.ts} +1 -1
- package/dist/{index-Di9nNqOY.d.mts → index-DuB-eQ0Z.d.mts} +1 -1
- package/dist/index.d.mts +7 -4
- package/dist/index.d.ts +7 -4
- package/dist/index.js +3177 -748
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2987 -590
- package/dist/index.mjs.map +1 -1
- package/dist/media-CQUC7TkS.d.mts +39 -0
- package/dist/media-DQnBloaX.d.ts +39 -0
- package/dist/traces/index.d.mts +3 -1
- package/dist/traces/index.d.ts +3 -1
- package/dist/traces/index.js +371 -102
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +371 -102
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +3 -2
- package/dist/tracing/index.d.ts +3 -2
- package/dist/tracing/index.js +257 -16
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +257 -16
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/transport-CpRe7zgd.d.mts +123 -0
- package/dist/transport-CpRe7zgd.d.ts +123 -0
- package/dist/worlds/index.d.mts +1986 -1746
- package/dist/worlds/index.d.ts +1986 -1746
- package/dist/worlds/index.js +1470 -290
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1465 -286
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +6 -1
- package/runtime/gatewaysdk/environment/bundle_manifest.py +531 -0
- package/runtime/gatewaysdk/environment/factory_documents.py +670 -0
- package/runtime/gatewaysdk/environment/factory_rules.py +973 -0
- package/runtime/gatewaysdk/environment/factory_shapes.py +253 -0
- package/runtime/gatewaysdk/environment/factory_spine.py +1155 -0
- package/runtime/gatewaysdk/environment/factory_spine_shapes.py +480 -0
- package/runtime/gatewaysdk/environment/schema/__main__.py +2 -2
- package/runtime/gatewaysdk/environment/schema/_slack_fidelity.py +8 -23
- package/runtime/gatewaysdk/environment/schema/admission.py +61 -59
- package/runtime/gatewaysdk/environment/schema/api.py +3 -4
- package/runtime/gatewaysdk/environment/schema/assertions.py +7 -1
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +20 -0
- package/runtime/gatewaysdk/environment/schema/cast.py +133 -67
- package/runtime/gatewaysdk/environment/schema/compiler.py +16 -17
- package/runtime/gatewaysdk/environment/schema/database/base.py +2 -2
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +58 -4
- package/runtime/gatewaysdk/environment/schema/host.py +94 -12
- package/runtime/gatewaysdk/environment/schema/host_browser.py +48 -0
- package/runtime/gatewaysdk/environment/schema/host_control.py +35 -2
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +4 -1
- package/runtime/gatewaysdk/environment/schema/host_loops.py +27 -4
- package/runtime/gatewaysdk/environment/schema/host_metrics.py +69 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +35 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +20 -2
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +100 -19
- package/runtime/gatewaysdk/environment/schema/host_workers.py +172 -50
- package/runtime/gatewaysdk/environment/schema/ingest.py +12 -3
- package/runtime/gatewaysdk/environment/schema/journal.py +7 -2
- package/runtime/gatewaysdk/environment/schema/local.py +32 -7
- package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
- package/runtime/gatewaysdk/environment/schema/measured_tests.py +187 -0
- package/runtime/gatewaysdk/environment/schema/page_checks.py +123 -0
- package/runtime/gatewaysdk/environment/schema/platform.py +80 -270
- package/runtime/gatewaysdk/environment/schema/platform_query.py +4 -3
- package/runtime/gatewaysdk/environment/schema/platform_templates.py +420 -0
- package/runtime/gatewaysdk/environment/schema/process_memory.py +76 -0
- package/runtime/gatewaysdk/environment/schema/redaction.py +13 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +4 -1
- package/runtime/gatewaysdk/environment/schema/run_parameters.py +25 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +57 -5
- package/runtime/gatewaysdk/environment/schema/scaffold.py +386 -219
- package/runtime/gatewaysdk/environment/schema/session_files.py +13 -2
- package/runtime/gatewaysdk/environment/schema/shape_catalog.py +493 -0
- package/runtime/gatewaysdk/environment/schema/shape_declarations.py +219 -0
- package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +1 -1
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +2 -2
- package/runtime/gatewaysdk/environment/schema/skills/factory-orchestrator/SKILL.md +147 -0
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +8 -6
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +9 -9
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +33 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +6 -6
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +17 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/new-world-sandbox.md +1 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/shape-catalog.md +59 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/ui-map.md +3 -2
- package/runtime/gatewaysdk/environment/schema/slack.py +55 -10
- package/runtime/gatewaysdk/environment/schema/snapshot.py +28 -2
- package/runtime/gatewaysdk/environment/schema/tasks.py +25 -2
- package/runtime/gatewaysdk/environment/schema/template_record.py +131 -0
- package/runtime/gatewaysdk/environment/schema/tools_world.py +7 -5
- package/runtime/gatewaysdk/environment/schema/ui.py +54 -33
- package/runtime/gatewaysdk/environment/schema/ui_interactions.py +419 -0
- package/runtime/gatewaysdk/environment/schema/ui_recorder.js +895 -0
- package/runtime/gatewaysdk/environment/schema/ui_replay.py +366 -0
- package/runtime/gatewaysdk/environment/schema/ui_replay_record.LICENSE +25 -0
- package/runtime/gatewaysdk/environment/schema/ui_replay_record.js +109 -0
- package/runtime/gatewaysdk/environment/schema/validation.py +24 -1
- package/runtime/gatewaysdk/environment/schema/worker.py +12 -5
- package/runtime/gatewaysdk/environment/schema/worker_key.py +38 -0
- package/runtime/gatewaysdk/environment/schema/worker_peaks.py +38 -0
- package/runtime/gatewaysdk/environment/schema/worker_watch.py +50 -0
- package/runtime/gatewaysdk/environment/schema/workers.py +245 -69
- package/runtime/gatewaysdk/environment/schema/world_host.py +96 -73
- package/runtime/gatewaysdk/environment/schema/world_tests.py +3 -1
- package/runtime/gatewaysdk/environment/secret_values.py +1 -1
- package/runtime/gatewaysdk/environment/world.py +34 -43
- package/runtime/gatewaysdk/environment/world_session_contract.py +7 -34
- package/skills/benchmarks-getting-started/SKILL.md +1 -1
- package/skills/connector-schema-authoring/SKILL.md +2 -2
- package/skills/connector-schema-authoring/connector-worlds.md +3 -2
- package/skills/connector-schema-authoring/world-schema.md +36 -23
- package/skills/factory-orchestrator/SKILL.md +147 -0
- package/skills/world-data-ingestion/ROWS.md +5 -1
- package/skills/world-data-ingestion/SKILL.md +8 -6
- package/skills/worlds-getting-started/SKILL.md +9 -9
- package/skills/worlds-getting-started/references/cli-mcp-map.md +33 -2
- package/skills/worlds-getting-started/references/gotchas.md +6 -6
- package/skills/worlds-getting-started/references/journal-checks.md +17 -2
- package/skills/worlds-getting-started/references/new-world-sandbox.md +1 -1
- package/skills/worlds-getting-started/references/shape-catalog.md +59 -0
- package/skills/worlds-getting-started/references/ui-map.md +3 -2
- package/dist/media-BIApiPzS.d.mts +0 -102
- package/dist/media-BIApiPzS.d.ts +0 -102
|
@@ -28,6 +28,9 @@ import json
|
|
|
28
28
|
import math
|
|
29
29
|
import re
|
|
30
30
|
import sys
|
|
31
|
+
from collections.abc import Iterable, Iterator
|
|
32
|
+
from contextlib import contextmanager
|
|
33
|
+
from contextvars import ContextVar
|
|
31
34
|
from typing import Any, Callable
|
|
32
35
|
|
|
33
36
|
from .validation import ContractError, fail
|
|
@@ -52,6 +55,7 @@ class Refusal(Exception):
|
|
|
52
55
|
|
|
53
56
|
_DETAIL_LIMIT = 2000
|
|
54
57
|
_FIELD_LIMIT = 256
|
|
58
|
+
ENGINE = "engine"
|
|
55
59
|
_SURROGATE = re.compile("[\ud800-\udfff]")
|
|
56
60
|
|
|
57
61
|
|
|
@@ -89,7 +93,45 @@ def log_contract_refusal(world: str, operation: str, error: ContractError) -> No
|
|
|
89
93
|
|
|
90
94
|
def request_failure_body(error: BaseException) -> dict:
|
|
91
95
|
"""The 500 body for a failure met answering a request, outside any handler."""
|
|
92
|
-
return {"error":
|
|
96
|
+
return {"error": ENGINE, "detail": f"{type(error).__name__}: {_detail(error)}"[:_DETAIL_LIMIT]}
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
class OutOfMemory:
|
|
100
|
+
"""A request whose world code raised ``MemoryError``, or ``SystemError`` (what C code raises when an
|
|
101
|
+
allocation fails without setting one), noted in that request's own context where the raise becomes
|
|
102
|
+
its answer (:func:`failure_answer`, the host's request handler) and never read from the answer, so
|
|
103
|
+
no data a world serves can claim one. A worker's front end says so to its supervisor in ``HEADER``
|
|
104
|
+
on that answer, and sends no other header of that name; the supervisor recycles the worker only
|
|
105
|
+
when the kernel counts it near its cap."""
|
|
106
|
+
|
|
107
|
+
HEADER = "X-Gateway-Out-Of-Memory"
|
|
108
|
+
_noted: ContextVar[bool] = ContextVar("gateway_out_of_memory", default=False)
|
|
109
|
+
|
|
110
|
+
@classmethod
|
|
111
|
+
@contextmanager
|
|
112
|
+
def request(cls) -> Iterator[None]:
|
|
113
|
+
token = cls._noted.set(False)
|
|
114
|
+
try:
|
|
115
|
+
yield
|
|
116
|
+
finally:
|
|
117
|
+
cls._noted.reset(token)
|
|
118
|
+
|
|
119
|
+
@classmethod
|
|
120
|
+
def note(cls, error: BaseException) -> None:
|
|
121
|
+
if isinstance(error, (MemoryError, SystemError)):
|
|
122
|
+
cls._noted.set(True)
|
|
123
|
+
|
|
124
|
+
@classmethod
|
|
125
|
+
def noted(cls) -> bool:
|
|
126
|
+
return cls._noted.get()
|
|
127
|
+
|
|
128
|
+
@classmethod
|
|
129
|
+
def named(cls, header: str) -> bool:
|
|
130
|
+
return header.lower() == cls.HEADER.lower()
|
|
131
|
+
|
|
132
|
+
@classmethod
|
|
133
|
+
def in_headers(cls, headers: Iterable[tuple[str, str]]) -> bool:
|
|
134
|
+
return any(cls.named(name) for name, _ in headers)
|
|
93
135
|
|
|
94
136
|
|
|
95
137
|
def engine_failure_body(handler: str, error: BaseException) -> dict:
|
|
@@ -99,6 +141,7 @@ def engine_failure_body(handler: str, error: BaseException) -> dict:
|
|
|
99
141
|
|
|
100
142
|
def failure_answer(handler: str, error: Exception) -> tuple[int, dict]:
|
|
101
143
|
"""A handler's raise as its ``(status, body)``: a refusal, its author's bug or the engine's failure."""
|
|
144
|
+
OutOfMemory.note(error)
|
|
102
145
|
if isinstance(error, Refusal):
|
|
103
146
|
return refusal_answer(handler, error)
|
|
104
147
|
if isinstance(error, HANDLER_BUG_ERRORS):
|
|
@@ -127,13 +170,24 @@ def unencodable(body: Any) -> Exception | None:
|
|
|
127
170
|
return None
|
|
128
171
|
|
|
129
172
|
|
|
173
|
+
_HEADER_NAME = re.compile(r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+")
|
|
174
|
+
_LINE_BREAK = re.compile(r"[\r\n\x00]")
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def header_line(name: str, value: str) -> bool:
|
|
178
|
+
"""Whether ``name: value`` is one header line: a token for a name, no CR, LF or NUL in the value, so no
|
|
179
|
+
answer can add a line of its own (a forged ``OutOfMemory.HEADER`` included)."""
|
|
180
|
+
return bool(_HEADER_NAME.fullmatch(name)) and not _LINE_BREAK.search(value)
|
|
181
|
+
|
|
182
|
+
|
|
130
183
|
def _unsendable_header(headers: dict[str, str] | None) -> Exception | None:
|
|
131
184
|
try:
|
|
132
185
|
for name, value in (headers or {}).items():
|
|
133
186
|
f"{name}: {value}".encode("latin-1")
|
|
134
187
|
except UnicodeEncodeError as error:
|
|
135
|
-
return error
|
|
136
|
-
|
|
188
|
+
return ValueError(f"a header is not latin-1: {error}")
|
|
189
|
+
split = next((name for name, value in (headers or {}).items() if not header_line(name, value)), None)
|
|
190
|
+
return None if split is None else ValueError(f"header {split!r} is not one line: a token name, no CR, LF or NUL")
|
|
137
191
|
|
|
138
192
|
|
|
139
193
|
def unencodable_answer(handler: str, response: tuple) -> tuple[int, dict] | None:
|
|
@@ -143,7 +197,7 @@ def unencodable_answer(handler: str, response: tuple) -> tuple[int, dict] | None
|
|
|
143
197
|
return 400, handler_bug_body(handler, ValueError(f"an answer names a status from 200 to 599, not {status!r}"))
|
|
144
198
|
header = _unsendable_header(headers)
|
|
145
199
|
if header is not None:
|
|
146
|
-
return 400, handler_bug_body(handler, header
|
|
200
|
+
return 400, handler_bug_body(handler, header)
|
|
147
201
|
error = unencodable(body)
|
|
148
202
|
return None if error is None else (400, handler_bug_body(handler, error, "the answer is not JSON: "))
|
|
149
203
|
|
|
@@ -50,15 +50,18 @@ import signal
|
|
|
50
50
|
import sys
|
|
51
51
|
import threading
|
|
52
52
|
import urllib.parse
|
|
53
|
+
from functools import partial
|
|
53
54
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
54
55
|
from pathlib import Path
|
|
55
56
|
from typing import Any
|
|
56
57
|
|
|
57
58
|
from .api import Via, with_query
|
|
58
59
|
from .call_capture import JSON_TYPE, send_response
|
|
60
|
+
from .call_redaction import CallRedaction
|
|
59
61
|
from .credentials import BearerAuth
|
|
60
62
|
from .door import PLATFORM_HEADER, Entry
|
|
61
|
-
from .handler_errors import log_request_failure, request_failure_body
|
|
63
|
+
from .handler_errors import OutOfMemory, header_line, log_request_failure, request_failure_body
|
|
64
|
+
from .host_browser import RelayNavigation, log_relay
|
|
62
65
|
from .host_control import ControlPlane
|
|
63
66
|
from .host_handoff import DrainHandoff, HandoffDoor, PlatformHandoff
|
|
64
67
|
from .host_loops import HEARTBEAT_S, Heartbeat, start_lease_sweeper
|
|
@@ -90,10 +93,14 @@ from .host_versions import DEFAULT_MAX_BUNDLE_MB
|
|
|
90
93
|
from .host_workers import UNKNOWN_SESSION, refusal
|
|
91
94
|
from .request_ledger import LedgerFlusher, LedgerSpill, NoLedger, PlatformLedger
|
|
92
95
|
from .session_move import SessionMoved
|
|
96
|
+
from .ui_interactions import RECORDER_PATH, UiRecorder
|
|
97
|
+
from .ui_replay import REPLAY_PATH, REPLAY_SCRIPT_PATH, REPLAY_STOP_S, ReplayShipper, UiReplay
|
|
93
98
|
from .validation import ContractError
|
|
94
99
|
from .workers import (
|
|
95
100
|
DEFAULT_WORKER_BUDGET_MB,
|
|
101
|
+
DEFAULT_WORKER_FILE_MB,
|
|
96
102
|
DEFAULT_WORKER_FOOTPRINT_MB,
|
|
103
|
+
DEFAULT_WORKER_MEMORY_CEILING_MB,
|
|
97
104
|
DEFAULT_WORKER_MEMORY_MB,
|
|
98
105
|
WorkerPool,
|
|
99
106
|
seal_process,
|
|
@@ -127,13 +134,15 @@ def make_host_handler(
|
|
|
127
134
|
*,
|
|
128
135
|
max_body_mb: int = DEFAULT_MAX_BODY_MB,
|
|
129
136
|
secure_cookies: bool = True,
|
|
137
|
+
navigation: RelayNavigation | None = None,
|
|
138
|
+
replays: ReplayShipper | None = None,
|
|
130
139
|
):
|
|
131
140
|
"""The request handler class for one host; its control plane is :class:`ControlPlane`, which
|
|
132
141
|
refuses an empty ``secret`` or ``worlds_domain``. ``secure_cookies`` is off
|
|
133
142
|
only for a host whose public surface is reached over plain http (a dev
|
|
134
143
|
host); it is the deployment's word, not the request's, since a client
|
|
135
144
|
writes the forwarding headers a request arrives with."""
|
|
136
|
-
control = ControlPlane(host, secret, HOSTED_OPERATIONS, worlds_domain)
|
|
145
|
+
control = ControlPlane(host, secret, HOSTED_OPERATIONS, worlds_domain, navigation)
|
|
137
146
|
if max_body_mb <= 0:
|
|
138
147
|
raise ValueError(f"the request body cap must be positive (WORLDS_HOST_MAX_BODY_MB={max_body_mb})")
|
|
139
148
|
max_body_bytes = max_body_mb * (1 << 20)
|
|
@@ -172,6 +181,20 @@ def make_host_handler(
|
|
|
172
181
|
return {k.lower(): v for k, v in self.headers.items()}
|
|
173
182
|
|
|
174
183
|
def _dispatch(self, method: str) -> None:
|
|
184
|
+
with OutOfMemory.request():
|
|
185
|
+
self._answer(method)
|
|
186
|
+
|
|
187
|
+
def send_header(self, keyword: str, value: str) -> None:
|
|
188
|
+
if OutOfMemory.named(keyword) or not header_line(keyword, str(value)):
|
|
189
|
+
return
|
|
190
|
+
super().send_header(keyword, value)
|
|
191
|
+
|
|
192
|
+
def end_headers(self) -> None:
|
|
193
|
+
if host.run_handlers and OutOfMemory.noted():
|
|
194
|
+
super().send_header(OutOfMemory.HEADER, "1")
|
|
195
|
+
super().end_headers()
|
|
196
|
+
|
|
197
|
+
def _answer(self, method: str) -> None:
|
|
175
198
|
self._body: bytes | None = None
|
|
176
199
|
url = urllib.parse.urlsplit(self.path)
|
|
177
200
|
if url.path == "/healthz" and control.internal(self.path, self.headers):
|
|
@@ -195,6 +218,7 @@ def make_host_handler(
|
|
|
195
218
|
self.close_connection = True
|
|
196
219
|
return self._reply(refused.status, {"error": refused.error}, {"Connection": "close"})
|
|
197
220
|
except Exception as error:
|
|
221
|
+
OutOfMemory.note(error)
|
|
198
222
|
log_request_failure(error, method, url.path)
|
|
199
223
|
return self._reply(500, request_failure_body(error))
|
|
200
224
|
|
|
@@ -318,12 +342,23 @@ def make_host_handler(
|
|
|
318
342
|
):
|
|
319
343
|
return self._reply(401, {"error": "invalid session token"})
|
|
320
344
|
route = api_target(ui, url.path)
|
|
321
|
-
if
|
|
322
|
-
return self.
|
|
345
|
+
if url.path == RECORDER_PATH:
|
|
346
|
+
return self._serve_file(method, opened, UiRecorder.script)
|
|
347
|
+
if url.path == REPLAY_SCRIPT_PATH:
|
|
348
|
+
return self._serve_file(method, opened, partial(UiReplay.script, replays))
|
|
349
|
+
static = route is None and not UiRecorder.serves(url.path)
|
|
350
|
+
if static and session.worker is not None and UiRecorder.is_page(url.path):
|
|
351
|
+
return self._on_worker(session, method, headers)
|
|
352
|
+
if static:
|
|
353
|
+
return self._serve_static(session, ui, method, url, opened)
|
|
323
354
|
if cross_site(headers, lambda origin: urllib.parse.urlsplit(origin).netloc == headers.get("host")):
|
|
324
355
|
return self._reply(403, {"error": "the ui's api prefix answers its own page, not another site"})
|
|
356
|
+
if url.path == REPLAY_PATH:
|
|
357
|
+
return self._record_replay(session, method)
|
|
325
358
|
if session.worker is not None:
|
|
326
359
|
return self._on_worker(session, method, headers)
|
|
360
|
+
if route is None:
|
|
361
|
+
return self._record_interactions(session, method, url, headers)
|
|
327
362
|
target = urllib.parse.urlunsplit(("", "", route, url.query, ""))
|
|
328
363
|
api_headers = {k: v for k, v in headers.items() if k not in ("cookie", "authorization")}
|
|
329
364
|
api_headers["host"] = session.api_host(worlds_domain, opened)
|
|
@@ -334,20 +369,49 @@ def make_host_handler(
|
|
|
334
369
|
cookie = session_cookie(token, secure=secure_cookies, max_age_s=host.lease_s)
|
|
335
370
|
self._reply(303, {"location": landing}, {"Location": landing, "Set-Cookie": cookie})
|
|
336
371
|
|
|
337
|
-
def _serve_static(self, ui: UiSurface, method: str, url, opened: bool) -> None:
|
|
338
|
-
"""A file of the ui;
|
|
339
|
-
|
|
372
|
+
def _serve_static(self, session: Session, ui: UiSurface, method: str, url, opened: bool) -> None:
|
|
373
|
+
"""A file of the ui; a page carries the recorder and is noted as served, a note for the
|
|
374
|
+
ledger alone (no grader reads it), so a page is never held by the move fence."""
|
|
340
375
|
if method != "GET":
|
|
341
376
|
return self._reply(405, {"error": "the static ui answers GET only"})
|
|
342
|
-
status,
|
|
343
|
-
|
|
377
|
+
status, file_headers, payload = static_file(ui, url.path)
|
|
378
|
+
page = UiRecorder.page(status, file_headers, payload)
|
|
379
|
+
if page is not None and session.runtime is not None:
|
|
380
|
+
UiRecorder.served(url.path, session.runtime.calls)
|
|
381
|
+
self._send_file(opened, status, file_headers, payload if page is None else page)
|
|
382
|
+
|
|
383
|
+
def _serve_file(self, method: str, opened: bool, answer) -> None:
|
|
384
|
+
if method != "GET":
|
|
385
|
+
return self._reply(405, {"error": "the static ui answers GET only"})
|
|
386
|
+
self._send_file(opened, *answer())
|
|
387
|
+
|
|
388
|
+
def _send_file(self, opened: bool, status: int, headers: dict[str, str], payload: bytes) -> None:
|
|
389
|
+
"""On a capability host with no ``Referer``, so a page that loads another site's
|
|
390
|
+
resource never hands it the capability."""
|
|
391
|
+
sealed = {**headers, **(NO_REFERRER if opened else {})}
|
|
344
392
|
self._relay(status, sealed.items(), payload, drop=FRAMING_HEADERS)
|
|
345
393
|
|
|
394
|
+
def _record_interactions(self, session: Session, method: str, url, headers: dict[str, str]) -> None:
|
|
395
|
+
if session.runtime is None:
|
|
396
|
+
return self._reply(404, {"error": "this session's world is not served here"})
|
|
397
|
+
raw = self._read()
|
|
398
|
+
with host.admitted(session, HOP_HEADER in headers):
|
|
399
|
+
answer = UiRecorder.record(method, url.path, raw, session.runtime.calls)
|
|
400
|
+
self._reply(*answer)
|
|
401
|
+
|
|
402
|
+
def _record_replay(self, session: Session, method: str) -> None:
|
|
403
|
+
"""The host that holds the session keeps its page's recording, a worker's session too, scrubbed
|
|
404
|
+
as the call log scrubs (:class:`CallRedaction`)."""
|
|
405
|
+
scrub = CallRedaction.of(session.spec, session.token).scrub
|
|
406
|
+
self._reply(*UiReplay.take(method, self._read(), session.session_id, scrub, replays))
|
|
407
|
+
|
|
346
408
|
def _serve_browser(self, session: Session, url, headers: dict[str, str]) -> None:
|
|
347
409
|
"""``/cdp/<session token>`` and ``/live/<grant>`` on the session's
|
|
348
410
|
cdp host: check the credential against the SESSION, then tunnel
|
|
349
411
|
the websocket to the leased relay with the POOL credential the
|
|
350
|
-
relay knows. The relay's token never leaves the host.
|
|
412
|
+
relay knows. The relay's token never leaves the host. The relay is
|
|
413
|
+
bound to the session's hosts first, so one that restarted and lost
|
|
414
|
+
its binding is never reached unbound."""
|
|
351
415
|
relay = session.browser
|
|
352
416
|
if relay is None:
|
|
353
417
|
return self._reply(404, {"error": "this session has no browser surface"})
|
|
@@ -368,11 +432,16 @@ def make_host_handler(
|
|
|
368
432
|
upstream_path = f"/live/{mint_live_grant(relay.token, LIVE_RELAY_GRANT_MS, scope)}"
|
|
369
433
|
if headers.get("upgrade", "").lower() != "websocket":
|
|
370
434
|
return self._reply(400, {"error": "the browser surface is a websocket"})
|
|
435
|
+
try:
|
|
436
|
+
control.navigation.bind(relay, session.browser_hosts(control.domain))
|
|
437
|
+
except ContractError as error:
|
|
438
|
+
return self._reply(502, {"error": "browser unbound", "detail": str(error)})
|
|
371
439
|
self.close_connection = True
|
|
372
440
|
try:
|
|
373
441
|
tunnel(self.connection, relay.url, upstream_path, headers)
|
|
374
442
|
except OSError as error:
|
|
375
|
-
|
|
443
|
+
log_relay("browser relay unreachable", relay, error=str(error))
|
|
444
|
+
self._reply(502, {"error": "browser unreachable"})
|
|
376
445
|
|
|
377
446
|
return Handler
|
|
378
447
|
|
|
@@ -397,12 +466,14 @@ class HostServer(ThreadingHTTPServer):
|
|
|
397
466
|
drain_s: float = DEFAULT_DRAIN_S,
|
|
398
467
|
ledger: LedgerFlusher | None = None,
|
|
399
468
|
handoff: HandoffDoor | None = None,
|
|
469
|
+
replays: ReplayShipper | None = None,
|
|
400
470
|
):
|
|
401
471
|
super().__init__(address, handler)
|
|
402
472
|
self.world_host = world_host
|
|
403
473
|
self.lease_sweeper = start_lease_sweeper(world_host)
|
|
404
474
|
self.heartbeat = Heartbeat(world_host, *heartbeat, interval_s=heartbeat_s).start() if heartbeat else None
|
|
405
475
|
world_host.ledger = ledger.start() if ledger else NoLedger()
|
|
476
|
+
self.replays = replays.start() if replays else None
|
|
406
477
|
self.handoff = DrainHandoff(world_host, handoff, drain_s)
|
|
407
478
|
self._answering = 0
|
|
408
479
|
self._draining = False
|
|
@@ -457,6 +528,8 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
|
|
|
457
528
|
``WORLDS_WORKER_MEMORY_MB`` caps each version worker's address space, ``WORLDS_WORKER_BUDGET_MB`` is the memory
|
|
458
529
|
all of them together are held to and ``WORLDS_WORKER_FOOTPRINT_MB`` what each is counted at against it (unset,
|
|
459
530
|
its whole address-space cap; their count is the quotient, and one more version answers 503 ``at_capacity``),
|
|
531
|
+
``WORLDS_WORKER_MEMORY_CEILING_MB`` the most address space a worker measured to need more is given (unset,
|
|
532
|
+
the cap), ``WORLDS_WORKER_FILE_MB`` is the largest file a worker may write (a session's database, so a world state's data),
|
|
460
533
|
``WORLDS_HOST_OBJECT_STORE`` is where every presigned URL it is handed may point (required; ``any`` for a dev
|
|
461
534
|
host), ``WORLDS_HOST_MAX_BUNDLE_MB`` is the largest version bundle the cache accepts,
|
|
462
535
|
``WORLDS_HOST_MAX_BODY_MB`` the largest request body it reads, ``WORLDS_HOST_LEDGER_SPILL_DIR`` (under the cache
|
|
@@ -484,6 +557,7 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
|
|
|
484
557
|
lease_s = float(os.environ.get("WORLDS_HOST_LEASE_S") or DEFAULT_LEASE_S)
|
|
485
558
|
workers = None
|
|
486
559
|
footprint = os.environ.get("WORLDS_WORKER_FOOTPRINT_MB")
|
|
560
|
+
ceiling = os.environ.get("WORLDS_WORKER_MEMORY_CEILING_MB")
|
|
487
561
|
if os.environ.get("WORLDS_HOST_WORKERS", "1") != "0":
|
|
488
562
|
workers = WorkerPool(
|
|
489
563
|
Path(cache_root),
|
|
@@ -491,6 +565,8 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
|
|
|
491
565
|
memory_mb=int(os.environ.get("WORLDS_WORKER_MEMORY_MB") or DEFAULT_WORKER_MEMORY_MB),
|
|
492
566
|
budget_mb=int(os.environ.get("WORLDS_WORKER_BUDGET_MB") or DEFAULT_WORKER_BUDGET_MB),
|
|
493
567
|
footprint_mb=int(footprint) if footprint else DEFAULT_WORKER_FOOTPRINT_MB,
|
|
568
|
+
file_mb=int(os.environ.get("WORLDS_WORKER_FILE_MB") or DEFAULT_WORKER_FILE_MB),
|
|
569
|
+
ceiling_mb=int(ceiling) if ceiling else DEFAULT_WORKER_MEMORY_CEILING_MB,
|
|
494
570
|
)
|
|
495
571
|
owners = PlatformOwners(platform_url, secret, advertise) if platform_url else None
|
|
496
572
|
world_host = WorldHost(
|
|
@@ -504,6 +580,7 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
|
|
|
504
580
|
build=os.environ.get("WORLDS_HOST_BUILD") or None,
|
|
505
581
|
owners=owners,
|
|
506
582
|
)
|
|
583
|
+
replays = ReplayShipper(platform_url, secret) if platform_url else None
|
|
507
584
|
handler = make_host_handler(
|
|
508
585
|
world_host,
|
|
509
586
|
secret=secret,
|
|
@@ -511,6 +588,7 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
|
|
|
511
588
|
owner_lookup=owners,
|
|
512
589
|
max_body_mb=int(os.environ.get("WORLDS_HOST_MAX_BODY_MB") or DEFAULT_MAX_BODY_MB),
|
|
513
590
|
secure_cookies=os.environ.get("WORLDS_HOST_PLAIN_HTTP", "0") != "1",
|
|
591
|
+
replays=replays,
|
|
514
592
|
)
|
|
515
593
|
sink = PlatformLedger(platform_url, secret, host_url=advertise) if platform_url else None
|
|
516
594
|
return HostServer(
|
|
@@ -522,6 +600,7 @@ def serve(cache_root: str | Path, *, port: int = 8080, host_addr: str = "0.0.0.0
|
|
|
522
600
|
heartbeat_s=float(os.environ.get("WORLDS_HOST_HEARTBEAT_S") or HEARTBEAT_S),
|
|
523
601
|
drain_s=float(os.environ.get("WORLDS_HOST_DRAIN_S") or DEFAULT_DRAIN_S),
|
|
524
602
|
handoff=PlatformHandoff(platform_url, secret) if platform_url and advertise else None,
|
|
603
|
+
replays=replays,
|
|
525
604
|
)
|
|
526
605
|
|
|
527
606
|
|
|
@@ -534,7 +613,8 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
534
613
|
args = parser.parse_args(argv)
|
|
535
614
|
server = serve(args.cache_root, port=args.port)
|
|
536
615
|
print(
|
|
537
|
-
f"world host listening on :{server.server_address[1]}, cache {args.cache_root}"
|
|
616
|
+
f"world host listening on :{server.server_address[1]}, cache {args.cache_root} "
|
|
617
|
+
f"(session copies: {server.world_host.cache.clone_mode})",
|
|
538
618
|
file=sys.stderr,
|
|
539
619
|
flush=True,
|
|
540
620
|
)
|
|
@@ -543,6 +623,8 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
543
623
|
server.serve_forever()
|
|
544
624
|
finally:
|
|
545
625
|
server.world_host.ledger.stop(settle=server.world_host.finishing.settle)
|
|
626
|
+
if server.replays is not None:
|
|
627
|
+
server.replays.stop(REPLAY_STOP_S)
|
|
546
628
|
server.lease_sweeper.set()
|
|
547
629
|
if server.heartbeat is not None:
|
|
548
630
|
server.heartbeat.stop()
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Binds a leased pool browser to the one session it serves: the relay's egress proxy then
|
|
2
|
+
loads that session's hosts and nothing else (deploy/world-browser/navigation.mjs)."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import json
|
|
7
|
+
import sys
|
|
8
|
+
import time
|
|
9
|
+
import urllib.error
|
|
10
|
+
import urllib.request
|
|
11
|
+
from collections.abc import Callable
|
|
12
|
+
from dataclasses import dataclass, field
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
from .host_requests import BrowserRelay
|
|
16
|
+
from .validation import fail
|
|
17
|
+
|
|
18
|
+
GUARD_REQUIRED_FROM = 1794787200.0
|
|
19
|
+
BIND_TIMEOUT_S = 5.0
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True)
|
|
23
|
+
class RelayNavigation:
|
|
24
|
+
opener: Callable[..., Any] = field(default=urllib.request.urlopen)
|
|
25
|
+
clock: Callable[[], float] = field(default=time.time)
|
|
26
|
+
|
|
27
|
+
def bind(self, relay: BrowserRelay, hosts: list[str]) -> str:
|
|
28
|
+
request = urllib.request.Request(
|
|
29
|
+
f"{relay.http_url}/navigation",
|
|
30
|
+
data=json.dumps({"hosts": hosts}).encode(),
|
|
31
|
+
method="PUT",
|
|
32
|
+
headers={"authorization": f"Bearer {relay.token}", "content-type": "application/json"},
|
|
33
|
+
)
|
|
34
|
+
try:
|
|
35
|
+
with self.opener(request, timeout=BIND_TIMEOUT_S) as answer:
|
|
36
|
+
return str(json.loads(answer.read() or b"{}").get("navigation", "bound"))
|
|
37
|
+
except urllib.error.HTTPError as error:
|
|
38
|
+
if error.code == 404 and self.clock() < GUARD_REQUIRED_FROM:
|
|
39
|
+
log_relay("browser relay predates the navigation guard", relay)
|
|
40
|
+
return "unguarded"
|
|
41
|
+
fail("host", f"the browser could not be bound to this session (HTTP {error.code})")
|
|
42
|
+
except (OSError, ValueError) as error:
|
|
43
|
+
log_relay("browser relay could not be bound", relay, error=str(error))
|
|
44
|
+
fail("host", "the browser could not be bound to this session: its relay did not answer as expected")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def log_relay(msg: str, relay: BrowserRelay, **extra: str) -> None:
|
|
48
|
+
print(json.dumps({"msg": msg, "relay": relay.url, **extra}), file=sys.stderr, flush=True)
|
|
@@ -15,6 +15,8 @@ from email.message import Message
|
|
|
15
15
|
from typing import TYPE_CHECKING, Any
|
|
16
16
|
|
|
17
17
|
from .handler_errors import carried
|
|
18
|
+
from .host_browser import RelayNavigation
|
|
19
|
+
from .host_requests import BrowserRelay, parse_memory_peak, parse_world_state
|
|
18
20
|
from .host_surfaces import public_host
|
|
19
21
|
from .host_workers import WORKER_EXITED, refusal
|
|
20
22
|
from .validation import ContractError, fail
|
|
@@ -35,10 +37,18 @@ class ControlPlane:
|
|
|
35
37
|
"""One host's control plane. A host with no secret has no control plane at all rather than an
|
|
36
38
|
open one, and one with no worlds domain could not tell its own address from a session's."""
|
|
37
39
|
|
|
38
|
-
def __init__(
|
|
40
|
+
def __init__(
|
|
41
|
+
self,
|
|
42
|
+
host: WorldHost,
|
|
43
|
+
secret: str,
|
|
44
|
+
operations: Mapping[str, Callable[..., dict]],
|
|
45
|
+
worlds_domain: str,
|
|
46
|
+
navigation: RelayNavigation | None = None,
|
|
47
|
+
):
|
|
39
48
|
if not secret:
|
|
40
49
|
raise ValueError("the world host needs a control-plane secret (WORLDS_HOST_SECRET)")
|
|
41
50
|
self.host = host
|
|
51
|
+
self.navigation = navigation or RelayNavigation()
|
|
42
52
|
self.operations = operations
|
|
43
53
|
self.domain = self.domain_of(worlds_domain)
|
|
44
54
|
self._secret = secret.encode()
|
|
@@ -109,6 +119,20 @@ class ControlPlane:
|
|
|
109
119
|
return 404, {"error": "unknown control route"}
|
|
110
120
|
|
|
111
121
|
def _open(self, body: dict) -> dict:
|
|
122
|
+
answer = self._open_session(body)
|
|
123
|
+
relay = BrowserRelay.parse(body.get("browser"))
|
|
124
|
+
if relay is not None:
|
|
125
|
+
self._bind_or_close(body["sessionId"], relay)
|
|
126
|
+
return answer
|
|
127
|
+
|
|
128
|
+
def _bind_or_close(self, session_id: str, relay: BrowserRelay) -> None:
|
|
129
|
+
try:
|
|
130
|
+
self.navigation.bind(relay, self.host.browser_hosts(session_id, self.domain))
|
|
131
|
+
except Exception:
|
|
132
|
+
self.host.close(session_id, "browser_unbound")
|
|
133
|
+
raise
|
|
134
|
+
|
|
135
|
+
def _open_session(self, body: dict) -> dict:
|
|
112
136
|
return self.host.open(
|
|
113
137
|
session_id=body["sessionId"],
|
|
114
138
|
version_id=body["versionId"],
|
|
@@ -123,14 +147,23 @@ class ControlPlane:
|
|
|
123
147
|
pending=body.get("pending") is True,
|
|
124
148
|
auth=body.get("auth"),
|
|
125
149
|
ledger_source=body.get("ledgerSource"),
|
|
150
|
+
parameters=body.get("parameters"),
|
|
151
|
+
world_state=parse_world_state(body.get("worldState")),
|
|
152
|
+
memory_peak_mb=parse_memory_peak(body.get("memoryPeakMb")),
|
|
126
153
|
)
|
|
127
154
|
|
|
155
|
+
def _swap_browser(self, session_id: str, browser: Any) -> dict:
|
|
156
|
+
relay = BrowserRelay.parse(browser)
|
|
157
|
+
if relay is not None:
|
|
158
|
+
self.navigation.bind(relay, self.host.browser_hosts(session_id, self.domain))
|
|
159
|
+
return self.host.set_browser(session_id, browser)
|
|
160
|
+
|
|
128
161
|
def _session(self, method: str, parts: list[str], body: dict) -> Answer:
|
|
129
162
|
sid = parts[2]
|
|
130
163
|
self.host.forwarding.refuse(sid)
|
|
131
164
|
verb = parts[3] if len(parts) > 3 else None
|
|
132
165
|
if method == "PUT" and verb == "browser" and len(parts) == 4:
|
|
133
|
-
return 200, self.
|
|
166
|
+
return 200, self._swap_browser(sid, body.get("browser"))
|
|
134
167
|
if method == "POST" and verb == "export" and len(parts) == 5:
|
|
135
168
|
return self._upload(sid, parts[4], body.get("url"))
|
|
136
169
|
row = session_verb(method, verb)
|
|
@@ -31,6 +31,7 @@ from .handler_errors import log_line
|
|
|
31
31
|
from .host_owners import PROXY_TIMEOUT_S
|
|
32
32
|
from .request_ledger import LEDGER_STOP_S
|
|
33
33
|
from .session_move import same_host
|
|
34
|
+
from .ui_replay import REPLAY_STOP_S
|
|
34
35
|
|
|
35
36
|
if TYPE_CHECKING:
|
|
36
37
|
from .host_session import Session
|
|
@@ -47,7 +48,9 @@ ENLIST_S = 1.0
|
|
|
47
48
|
LINGER_S = 5.0
|
|
48
49
|
IN_FLIGHT_S = PROXY_TIMEOUT_S
|
|
49
50
|
EXIT_S = 110.0
|
|
50
|
-
assert
|
|
51
|
+
assert (
|
|
52
|
+
LINGER_S + IN_FLIGHT_S + LEDGER_STOP_S + REPLAY_STOP_S + 5.0 <= EXIT_S
|
|
53
|
+
), "a stopping host's exit fits the grace past its drain"
|
|
51
54
|
|
|
52
55
|
MOVED = "moved"
|
|
53
56
|
STAYED = "stayed"
|
|
@@ -4,6 +4,7 @@ platform what the host is and holds. A step that fails is logged and the next on
|
|
|
4
4
|
|
|
5
5
|
from __future__ import annotations
|
|
6
6
|
|
|
7
|
+
import http.client
|
|
7
8
|
import json
|
|
8
9
|
import threading
|
|
9
10
|
import urllib.error
|
|
@@ -18,6 +19,8 @@ if TYPE_CHECKING:
|
|
|
18
19
|
|
|
19
20
|
LEASE_SWEEP_S = 30.0
|
|
20
21
|
HEARTBEAT_S = 10.0
|
|
22
|
+
#: The most a heartbeat reads of the platform's answer.
|
|
23
|
+
ANSWER_LIMIT_BYTES = 64 << 10
|
|
21
24
|
|
|
22
25
|
|
|
23
26
|
def swept(step: Callable[[], list[str]], done: str) -> list[str]:
|
|
@@ -49,7 +52,8 @@ def heartbeat_once(world_host: WorldHost, platform_url: str, secret: str, timeou
|
|
|
49
52
|
"""Tell the platform what this host is and holds (:meth:`WorldHost.stats`).
|
|
50
53
|
The platform's registry is what places new sessions: a host it has not
|
|
51
54
|
heard from lately gets none. Returns whether the platform took it."""
|
|
52
|
-
|
|
55
|
+
stats = world_host.stats()
|
|
56
|
+
body = json.dumps(stats).encode("utf-8")
|
|
53
57
|
req = urllib.request.Request(
|
|
54
58
|
f"{platform_url.rstrip('/')}/api/public/world-hosts",
|
|
55
59
|
data=body,
|
|
@@ -58,10 +62,23 @@ def heartbeat_once(world_host: WorldHost, platform_url: str, secret: str, timeou
|
|
|
58
62
|
)
|
|
59
63
|
try:
|
|
60
64
|
with urllib.request.urlopen(req, timeout=timeout_s) as r:
|
|
61
|
-
|
|
62
|
-
|
|
65
|
+
taken = 200 <= r.status < 300
|
|
66
|
+
peaks_taken = taken and _peaks_taken(r.read(ANSWER_LIMIT_BYTES))
|
|
67
|
+
except (urllib.error.URLError, OSError, ValueError, http.client.HTTPException) as error:
|
|
63
68
|
log_line(f"heartbeat failed: {error}")
|
|
64
69
|
return False
|
|
70
|
+
if peaks_taken:
|
|
71
|
+
world_host.reported(stats)
|
|
72
|
+
return taken
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _peaks_taken(answer: bytes) -> bool:
|
|
76
|
+
"""Whether the platform says it kept the beat's worker peaks: one that predates them takes the beat
|
|
77
|
+
and ignores them, so they stay pending until a platform that keeps them answers."""
|
|
78
|
+
try:
|
|
79
|
+
return json.loads(answer).get("peaksTaken") is True
|
|
80
|
+
except (ValueError, AttributeError):
|
|
81
|
+
return False
|
|
65
82
|
|
|
66
83
|
|
|
67
84
|
class Heartbeat:
|
|
@@ -90,8 +107,14 @@ class Heartbeat:
|
|
|
90
107
|
self._due.set()
|
|
91
108
|
|
|
92
109
|
def _beat(self) -> None:
|
|
93
|
-
|
|
110
|
+
self._once()
|
|
94
111
|
while not self._stopped.is_set():
|
|
95
112
|
self._due.wait(self.interval_s)
|
|
96
113
|
self._due.clear()
|
|
114
|
+
self._once()
|
|
115
|
+
|
|
116
|
+
def _once(self) -> None:
|
|
117
|
+
try:
|
|
97
118
|
heartbeat_once(self.world_host, self.platform_url, self.secret)
|
|
119
|
+
except Exception as error: # noqa: BLE001 - one failed beat must not stop the beats after it
|
|
120
|
+
log_line(f"heartbeat failed: {type(error).__name__}")
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""The prometheus exposition of a World Host's :meth:`WorldHost.stats`. ``worlds_host_sessions`` is the
|
|
2
|
+
number the fleet scales on (sessions per host, not cpu)."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
from .host_handoff import OUTCOMES
|
|
7
|
+
from .host_workers import WORKERS_POISONED
|
|
8
|
+
from .worker_watch import WORKERS_NEAR_CAP
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _largest_share(workers: list[dict]) -> list[str]:
|
|
12
|
+
"""The most of its own cap any worker holds (``VmSize`` over ``RLIMIT_AS``), no sample when none was
|
|
13
|
+
measured. A total: world code on the pod can read ``/metrics``, so no line here names a version."""
|
|
14
|
+
shares = [w["sizeMb"] / w["capMb"] for w in workers if w.get("sizeMb") is not None and w.get("capMb")]
|
|
15
|
+
sample = [f"worlds_host_worker_cap_share_max {max(shares):.3f}"] if shares else []
|
|
16
|
+
return ["# TYPE worlds_host_worker_cap_share_max gauge", *sample]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def exposition(stats: dict) -> str:
|
|
20
|
+
counters: dict[str, int] = stats["counters"]
|
|
21
|
+
lines = [
|
|
22
|
+
"# TYPE worlds_host_sessions gauge",
|
|
23
|
+
f"worlds_host_sessions {stats['sessions']}",
|
|
24
|
+
"# TYPE worlds_host_capacity gauge",
|
|
25
|
+
f"worlds_host_capacity {stats['capacity']}",
|
|
26
|
+
"# TYPE worlds_host_draining gauge",
|
|
27
|
+
f"worlds_host_draining {int(stats['draining'])}",
|
|
28
|
+
"# TYPE worlds_host_workers gauge",
|
|
29
|
+
f"worlds_host_workers {stats['workers']}",
|
|
30
|
+
"# TYPE worlds_host_worker_memory_mb gauge",
|
|
31
|
+
f"worlds_host_worker_memory_mb {stats['workerMemoryMb']}",
|
|
32
|
+
"# TYPE worlds_host_versions_cached gauge",
|
|
33
|
+
f"worlds_host_versions_cached {len(stats['versions'])}",
|
|
34
|
+
"# TYPE worlds_host_uptime_seconds gauge",
|
|
35
|
+
f"worlds_host_uptime_seconds {stats['uptimeSeconds']:.0f}",
|
|
36
|
+
"# TYPE worlds_host_sessions_opened_total counter",
|
|
37
|
+
f"worlds_host_sessions_opened_total {counters.get('opened', 0)}",
|
|
38
|
+
"# TYPE worlds_host_open_failures_total counter",
|
|
39
|
+
f"worlds_host_open_failures_total {counters.get('open_failures', 0)}",
|
|
40
|
+
"# TYPE worlds_host_session_seconds_total counter",
|
|
41
|
+
f"worlds_host_session_seconds_total {stats['sessionSeconds']:.0f}",
|
|
42
|
+
"# TYPE worlds_host_exports_total counter",
|
|
43
|
+
f"worlds_host_exports_total {counters.get('exports', 0)}",
|
|
44
|
+
"# TYPE worlds_host_exports_uploaded_total counter",
|
|
45
|
+
f"worlds_host_exports_uploaded_total {counters.get('exports_uploaded', 0)}",
|
|
46
|
+
"# TYPE worlds_host_sessions_closed_total counter",
|
|
47
|
+
]
|
|
48
|
+
for reason in ("closed", "expired", "lost", "replaced", "moved"):
|
|
49
|
+
lines.append(f'worlds_host_sessions_closed_total{{reason="{reason}"}} {counters.get(f"closed.{reason}", 0)}')
|
|
50
|
+
lines.append("# TYPE worlds_host_handoffs_total counter")
|
|
51
|
+
for outcome in OUTCOMES:
|
|
52
|
+
lines.append(f'worlds_host_handoffs_total{{outcome="{outcome}"}} {counters.get(f"handoffs.{outcome}", 0)}')
|
|
53
|
+
workers = stats.get("workerMemory") or []
|
|
54
|
+
lines += _largest_share(workers)
|
|
55
|
+
lines += [
|
|
56
|
+
"# TYPE worlds_host_workers_near_cap gauge",
|
|
57
|
+
f"worlds_host_workers_near_cap {sum(1 for w in workers if w.get('nearCap'))}",
|
|
58
|
+
]
|
|
59
|
+
recoveries = ("workers_replaced", "workers_recycled", WORKERS_POISONED, WORKERS_NEAR_CAP, "sessions_recovered")
|
|
60
|
+
for name in (*recoveries, "sessions_unrecovered"):
|
|
61
|
+
lines += [f"# TYPE worlds_host_{name}_total counter", f"worlds_host_{name}_total {counters.get(name, 0)}"]
|
|
62
|
+
lines.append("# TYPE worlds_host_requests_total counter")
|
|
63
|
+
for surface in ("api", "ui", "browser", "proxied"):
|
|
64
|
+
lines.append(f'worlds_host_requests_total{{surface="{surface}"}} {counters.get(f"requests.{surface}", 0)}')
|
|
65
|
+
lines += [
|
|
66
|
+
"# TYPE worlds_host_platform_bearer_requests_total counter",
|
|
67
|
+
f"worlds_host_platform_bearer_requests_total {counters.get('requests.platform_bearer', 0)}",
|
|
68
|
+
]
|
|
69
|
+
return "\n".join(lines) + "\n"
|