@withgateway/sdk 0.3.1 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -6
- package/dist/cli.js +4372 -2265
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +4362 -2256
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.d.mts +1 -1
- package/dist/files/index.d.ts +1 -1
- package/dist/files/index.js +259 -58
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +249 -50
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
- package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
- package/dist/index.d.mts +5 -5
- package/dist/index.d.ts +5 -5
- package/dist/index.js +2240 -789
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +2150 -739
- package/dist/index.mjs.map +1 -1
- package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
- package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
- package/dist/traces/index.d.mts +1 -1
- package/dist/traces/index.d.ts +1 -1
- package/dist/traces/index.js +253 -52
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +243 -44
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +2 -2
- package/dist/tracing/index.d.ts +2 -2
- package/dist/tracing/index.js +75 -13
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +75 -13
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +893 -482
- package/dist/worlds/index.d.ts +893 -482
- package/dist/worlds/index.js +1968 -669
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +1960 -662
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +2 -27
- package/runtime/gatewaysdk/environment/_materialize.py +48 -88
- package/runtime/gatewaysdk/environment/_world.py +155 -237
- package/runtime/gatewaysdk/environment/backing.py +133 -0
- package/runtime/gatewaysdk/environment/core.py +2 -17
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
- package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
- package/runtime/gatewaysdk/environment/runtime.py +8 -13
- package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
- package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +347 -892
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
- package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
- package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +60 -216
- package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
- package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
- package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
- package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
- package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
- package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
- package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
- package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
- package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
- package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
- package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
- package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
- package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
- package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +58 -170
- package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
- package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
- package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
- package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
- package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
- package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
- package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
- package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
- package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
- package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
- package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
- package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
- package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
- package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
- package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
- package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
- package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
- package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
- package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
- package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
- package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
- package/runtime/gatewaysdk/environment/score.py +57 -0
- package/runtime/gatewaysdk/environment/secret_values.py +36 -0
- package/runtime/gatewaysdk/environment/world.py +43 -4
- package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +10 -11
- package/skills/connector-schema-authoring/SKILL.md +49 -16
- package/skills/connector-schema-authoring/connector-worlds.md +142 -11
- package/skills/connector-schema-authoring/world-schema.md +103 -51
- package/skills/world-data-ingestion/ROWS.md +39 -9
- package/skills/world-data-ingestion/SKILL.md +27 -27
- package/skills/worlds-getting-started/SKILL.md +33 -33
- package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
- package/skills/worlds-getting-started/references/gotchas.md +49 -1
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/skills/worlds-getting-started/references/openapi-import.md +50 -21
- package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
|
@@ -9,7 +9,7 @@ it. The worker is the same :class:`host.WorldHost` in a smaller room:
|
|
|
9
9
|
* its own process, so a handler that loops, leaks or crashes takes down that
|
|
10
10
|
version's sessions on this replica and nothing else;
|
|
11
11
|
* rlimits on address space, open files and file size, set before any world
|
|
12
|
-
code is imported;
|
|
12
|
+
code is imported (:class:`AddressSpaceLimit` says how macOS differs);
|
|
13
13
|
* a scrubbed environment: the control-plane secret, cloud credentials and the
|
|
14
14
|
rest of the supervisor's environment are not inherited. The worker gets a
|
|
15
15
|
secret of its own, good for its socket alone;
|
|
@@ -17,10 +17,11 @@ it. The worker is the same :class:`host.WorldHost` in a smaller room:
|
|
|
17
17
|
a process even when they share a host;
|
|
18
18
|
* a socket only the supervisor is answered on. Workers run as the host's user,
|
|
19
19
|
so file modes cannot keep one worker's handler from opening a sibling's
|
|
20
|
-
socket path: the worker asks the kernel who connected (``SO_PEERCRED
|
|
21
|
-
serves the supervisor's pid alone. The listening
|
|
22
|
-
bound before the worker starts
|
|
23
|
-
|
|
20
|
+
socket path: the worker asks the kernel who connected (``SO_PEERCRED``, on
|
|
21
|
+
macOS ``LOCAL_PEERPID``) and serves the supervisor's pid alone. The listening
|
|
22
|
+
socket is the supervisor's, bound before the worker starts in a short private
|
|
23
|
+
directory and handed down as a file descriptor, so a worker never binds a
|
|
24
|
+
path of its own;
|
|
24
25
|
* a worker nobody can read: it marks itself non-dumpable, which closes
|
|
25
26
|
``/proc/<pid>/environ``, ``/proc/<pid>/mem`` and ``ptrace`` to every process
|
|
26
27
|
that is not root, siblings under the same uid included. Its secret and its
|
|
@@ -34,7 +35,9 @@ fence (a sandboxed runtime class, a read-only root) when it is wanted; see
|
|
|
34
35
|
|
|
35
36
|
from __future__ import annotations
|
|
36
37
|
|
|
38
|
+
import collections
|
|
37
39
|
import ctypes
|
|
40
|
+
import hashlib
|
|
38
41
|
import http.client
|
|
39
42
|
import os
|
|
40
43
|
import resource
|
|
@@ -44,14 +47,19 @@ import socket
|
|
|
44
47
|
import struct
|
|
45
48
|
import subprocess
|
|
46
49
|
import sys
|
|
50
|
+
import tempfile
|
|
47
51
|
import threading
|
|
48
52
|
import time
|
|
49
|
-
from collections.abc import Mapping
|
|
53
|
+
from collections.abc import Callable, Iterator, Mapping
|
|
54
|
+
from contextlib import contextmanager, suppress
|
|
50
55
|
from dataclasses import dataclass, field
|
|
51
56
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
52
57
|
from pathlib import Path
|
|
58
|
+
from typing import Protocol
|
|
53
59
|
|
|
60
|
+
from .cache_layout import WORKERS_DIR
|
|
54
61
|
from .grading import VerifierSandbox
|
|
62
|
+
from .handler_errors import log_line
|
|
55
63
|
|
|
56
64
|
#: Address space a worker may use, in MiB, unless ``WORLDS_WORKER_MEMORY_MB`` says otherwise.
|
|
57
65
|
DEFAULT_WORKER_MEMORY_MB = 2048
|
|
@@ -63,21 +71,26 @@ DEFAULT_WORKER_MEMORY_MB = 2048
|
|
|
63
71
|
DEFAULT_WORKER_BUDGET_MB = 4096
|
|
64
72
|
#: What one worker is counted at against the budget, in MiB, unless
|
|
65
73
|
#: ``WORLDS_WORKER_FOOTPRINT_MB`` says otherwise. Unset, a worker is counted at
|
|
66
|
-
#: its whole address-space cap
|
|
67
|
-
#:
|
|
68
|
-
#:
|
|
69
|
-
#:
|
|
70
|
-
#:
|
|
71
|
-
#: container for footprint x workers plus the supervisor's share.
|
|
74
|
+
#: its whole address-space cap: the safe reading, and a 50x over-reservation of
|
|
75
|
+
#: the ~40 MiB a worker holding sessions keeps resident. A fleet that must host
|
|
76
|
+
#: many worlds at once sets the footprint it measured, keeps the address-space
|
|
77
|
+
#: cap as the rail each worker cannot cross, and sizes the container for
|
|
78
|
+
#: footprint x workers plus the supervisor's share.
|
|
72
79
|
DEFAULT_WORKER_FOOTPRINT_MB: int | None = None
|
|
73
80
|
#: Largest file a worker may write (its session databases), in MiB.
|
|
74
81
|
DEFAULT_WORKER_FILE_MB = 4096
|
|
75
82
|
#: Open files per worker: one per live session plus the sockets serving them.
|
|
76
83
|
DEFAULT_WORKER_NOFILE = 4096
|
|
77
|
-
|
|
84
|
+
FLOOR_PRECISION_BYTES = 1 << 20
|
|
85
|
+
#: The largest limit a kernel reports, which ``RLIM_INFINITY`` is read as in arithmetic: macOS
|
|
86
|
+
#: spells it so, Linux spells it -1.
|
|
87
|
+
MAX_RLIMIT = 2**63 - 1
|
|
78
88
|
DEFAULT_WORKER_IDLE_S = 300.0
|
|
79
|
-
#: How long the supervisor waits for a fresh worker to answer on its socket.
|
|
80
89
|
WORKER_START_TIMEOUT_S = 30.0
|
|
90
|
+
#: How long one health probe waits: a worker that cannot answer it in this time is not serving.
|
|
91
|
+
HEALTH_TIMEOUT_S = 2.0
|
|
92
|
+
#: The most a probe reads of a worker's answer: the worker runs tenant code, so its size is not its to choose.
|
|
93
|
+
PROBE_LIMIT_BYTES = 64 << 10
|
|
81
94
|
#: A worker's sessions live as long as the supervisor's do; the supervisor
|
|
82
95
|
#: holds the leases, so the worker's own are effectively unbounded.
|
|
83
96
|
WORKER_LEASE_S = 365.0 * 86400.0
|
|
@@ -87,10 +100,20 @@ INHERITED_ENV = ("PATH", "PYTHONPATH", "LANG", "LC_ALL", "TZ", "VIRTUAL_ENV")
|
|
|
87
100
|
#: Headers the supervisor never forwards to a worker: the ones that describe
|
|
88
101
|
#: the supervisor's connection, and its own control-plane secret.
|
|
89
102
|
UNFORWARDED_HEADERS = frozenset({"connection", "content-length", "transfer-encoding", "x-host-secret"})
|
|
103
|
+
FINISH_THREADS = 8
|
|
104
|
+
FINISH_SETTLE_S = 10.0
|
|
90
105
|
|
|
91
106
|
|
|
92
107
|
class WorkerGone(Exception):
|
|
93
|
-
"""The worker did not answer: it exited,
|
|
108
|
+
"""The worker did not answer: it exited, never came up, or dropped the request."""
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class WorkerTimeout(WorkerGone):
|
|
112
|
+
"""The worker did not answer in time; it may be alive and still working on the request."""
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
class WorkerUnreached(WorkerGone):
|
|
116
|
+
"""The worker took no connection: the request never reached it."""
|
|
94
117
|
|
|
95
118
|
|
|
96
119
|
class WorkerCapacity(Exception):
|
|
@@ -98,11 +121,6 @@ class WorkerCapacity(Exception):
|
|
|
98
121
|
worker for another version can start here."""
|
|
99
122
|
|
|
100
123
|
|
|
101
|
-
# --------------------------------------------------------------------------- #
|
|
102
|
-
# HTTP over a unix socket: the only way into a worker.
|
|
103
|
-
# --------------------------------------------------------------------------- #
|
|
104
|
-
|
|
105
|
-
|
|
106
124
|
class UnixHTTPConnection(http.client.HTTPConnection):
|
|
107
125
|
def __init__(self, path: str | Path, timeout: float | None = None):
|
|
108
126
|
super().__init__("worker", timeout=timeout)
|
|
@@ -115,6 +133,24 @@ class UnixHTTPConnection(http.client.HTTPConnection):
|
|
|
115
133
|
self.sock = sock
|
|
116
134
|
|
|
117
135
|
|
|
136
|
+
def _hang_up(sock: socket.socket, cut: threading.Event) -> None:
|
|
137
|
+
cut.set()
|
|
138
|
+
with suppress(OSError):
|
|
139
|
+
sock.shutdown(socket.SHUT_RDWR)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
@contextmanager
|
|
143
|
+
def _deadline(sock: socket.socket, seconds: float) -> Iterator[threading.Event]:
|
|
144
|
+
"""Shut ``sock`` once ``seconds`` pass, so nothing read from it waits longer; the event says it was."""
|
|
145
|
+
cut = threading.Event()
|
|
146
|
+
timer = threading.Timer(seconds, _hang_up, (sock, cut))
|
|
147
|
+
timer.start()
|
|
148
|
+
try:
|
|
149
|
+
yield cut
|
|
150
|
+
finally:
|
|
151
|
+
timer.cancel()
|
|
152
|
+
|
|
153
|
+
|
|
118
154
|
def bind_worker_socket(path: str | Path) -> socket.socket:
|
|
119
155
|
"""The supervisor's end of a worker: a listening unix socket at ``path``,
|
|
120
156
|
``0600`` inside a ``0700`` directory, ready to hand to the worker."""
|
|
@@ -128,18 +164,40 @@ def bind_worker_socket(path: str | Path) -> socket.socket:
|
|
|
128
164
|
return sock
|
|
129
165
|
|
|
130
166
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
return None
|
|
138
|
-
raw = conn.getsockopt(socket.SOL_SOCKET, option, struct.calcsize("3i"))
|
|
167
|
+
#: macOS's ``SOL_LOCAL`` and ``LOCAL_PEERPID`` from <sys/un.h>; CPython exports neither.
|
|
168
|
+
_SOL_LOCAL, _LOCAL_PEERPID = 0, 0x002
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _linux_peer_pid(conn: socket.socket) -> int:
|
|
172
|
+
raw = conn.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, struct.calcsize("3i"))
|
|
139
173
|
pid, _uid, _gid = struct.unpack("3i", raw)
|
|
140
174
|
return pid
|
|
141
175
|
|
|
142
176
|
|
|
177
|
+
def _darwin_peer_pid(conn: socket.socket) -> int:
|
|
178
|
+
raw = conn.getsockopt(_SOL_LOCAL, _LOCAL_PEERPID, struct.calcsize("i"))
|
|
179
|
+
return struct.unpack("i", raw)[0]
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
_PEER_PID = {"linux": _linux_peer_pid, "darwin": _darwin_peer_pid}
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def peer_pid(conn: socket.socket) -> int | None:
|
|
186
|
+
"""The pid of whoever connected, as the kernel reports it: Linux's
|
|
187
|
+
``SO_PEERCRED`` (``struct ucred {pid, uid, gid}``), macOS's
|
|
188
|
+
``LOCAL_PEERPID``. None where the platform has no such thing, or where the
|
|
189
|
+
kernel cannot name the peer (macOS raises ``ENOTCONN`` once the client has
|
|
190
|
+
hung up): a worker serving its supervisor alone refuses both, and keeps
|
|
191
|
+
serving, since ``socketserver`` asks outside its own error handling."""
|
|
192
|
+
read = _PEER_PID.get(sys.platform)
|
|
193
|
+
if read is None:
|
|
194
|
+
return None
|
|
195
|
+
try:
|
|
196
|
+
return read(conn)
|
|
197
|
+
except OSError:
|
|
198
|
+
return None
|
|
199
|
+
|
|
200
|
+
|
|
143
201
|
class UnixHTTPServer(ThreadingHTTPServer):
|
|
144
202
|
"""The worker's server: a ``ThreadingHTTPServer`` over the listening unix
|
|
145
203
|
socket the supervisor handed it (an inherited descriptor), or one it binds
|
|
@@ -195,37 +253,100 @@ def seal_process() -> bool:
|
|
|
195
253
|
return False
|
|
196
254
|
|
|
197
255
|
|
|
256
|
+
class Kernel(Protocol):
|
|
257
|
+
"""Where a process's limits are read and set: the ``resource`` module."""
|
|
258
|
+
|
|
259
|
+
RLIM_INFINITY: int
|
|
260
|
+
|
|
261
|
+
def getrlimit(self, name: int, /) -> tuple[int, int]: ...
|
|
262
|
+
|
|
263
|
+
def setrlimit(self, name: int, limits: tuple[int, int], /) -> None: ...
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
@dataclass(frozen=True)
|
|
267
|
+
class ProcessLimit:
|
|
268
|
+
"""One rlimit on this process, soft and hard alike. A limit the kernel
|
|
269
|
+
holds lower than asked is lowered to its hard limit, never raised; one it
|
|
270
|
+
refuses fails the start."""
|
|
271
|
+
|
|
272
|
+
name: int
|
|
273
|
+
wanted: int
|
|
274
|
+
label: str
|
|
275
|
+
kernel: Kernel = resource
|
|
276
|
+
|
|
277
|
+
def apply(self) -> int:
|
|
278
|
+
value = self.within_hard()
|
|
279
|
+
self.kernel.setrlimit(self.name, (value, value))
|
|
280
|
+
return value
|
|
281
|
+
|
|
282
|
+
def hard(self) -> int:
|
|
283
|
+
return self.kernel.getrlimit(self.name)[1]
|
|
284
|
+
|
|
285
|
+
def ceiling(self) -> int:
|
|
286
|
+
"""The hard limit as a number: ``RLIM_INFINITY`` reads as :data:`MAX_RLIMIT`."""
|
|
287
|
+
hard = self.hard()
|
|
288
|
+
return MAX_RLIMIT if hard == self.kernel.RLIM_INFINITY else hard
|
|
289
|
+
|
|
290
|
+
def within_hard(self) -> int:
|
|
291
|
+
return min(self.wanted, self.ceiling())
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
class AddressSpaceLimit(ProcessLimit):
|
|
295
|
+
"""The memory cap. Linux takes any cap up to the hard limit, so there the
|
|
296
|
+
budget is the cap exactly. macOS refuses a cap below what the process
|
|
297
|
+
already maps (about 392 GiB on Apple Silicon, mostly the shared region's
|
|
298
|
+
reservation), so there the budget is granted on top of that floor: it still
|
|
299
|
+
bounds growth, and a kernel that takes no cap at all fails the start."""
|
|
300
|
+
|
|
301
|
+
def apply(self) -> int:
|
|
302
|
+
try:
|
|
303
|
+
return super().apply()
|
|
304
|
+
except ValueError:
|
|
305
|
+
return self._above_floor()
|
|
306
|
+
|
|
307
|
+
def _above_floor(self) -> int:
|
|
308
|
+
ceiling = self.ceiling()
|
|
309
|
+
floor = self._lowest_taken(ceiling)
|
|
310
|
+
if floor >= ceiling:
|
|
311
|
+
raise ValueError(f"the kernel refuses every {self.label} cap under its hard limit")
|
|
312
|
+
value = min(floor + self.wanted, ceiling)
|
|
313
|
+
self.kernel.setrlimit(self.name, (value, value))
|
|
314
|
+
return value
|
|
315
|
+
|
|
316
|
+
def _lowest_taken(self, ceiling: int) -> int:
|
|
317
|
+
"""The lowest cap the kernel takes, to within FLOOR_PRECISION_BYTES.
|
|
318
|
+
Each probe moves the soft limit alone and is undone at once: a lowered
|
|
319
|
+
hard limit cannot be raised again, and a process left near its floor
|
|
320
|
+
fails its next allocation."""
|
|
321
|
+
restore = self.kernel.getrlimit(self.name)
|
|
322
|
+
refused, taken = self.within_hard(), ceiling
|
|
323
|
+
while taken - refused > FLOOR_PRECISION_BYTES:
|
|
324
|
+
probe = (refused + taken) // 2
|
|
325
|
+
try:
|
|
326
|
+
self.kernel.setrlimit(self.name, (probe, restore[1]))
|
|
327
|
+
except ValueError:
|
|
328
|
+
refused = probe
|
|
329
|
+
continue
|
|
330
|
+
self.kernel.setrlimit(self.name, restore)
|
|
331
|
+
taken = probe
|
|
332
|
+
return taken
|
|
333
|
+
|
|
334
|
+
|
|
198
335
|
def apply_limits(
|
|
199
336
|
*,
|
|
200
337
|
memory_mb: int = DEFAULT_WORKER_MEMORY_MB,
|
|
201
338
|
file_mb: int = DEFAULT_WORKER_FILE_MB,
|
|
202
339
|
nofile: int = DEFAULT_WORKER_NOFILE,
|
|
203
340
|
) -> dict[str, int]:
|
|
204
|
-
"""Cap this process before it imports world code.
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
(resource.
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
(resource.RLIMIT_CORE, 0),
|
|
214
|
-
):
|
|
215
|
-
_, hard = resource.getrlimit(name)
|
|
216
|
-
value = wanted if hard == resource.RLIM_INFINITY else min(wanted, hard)
|
|
217
|
-
resource.setrlimit(name, (value, value))
|
|
218
|
-
applied[_limit_name(name)] = value
|
|
219
|
-
return applied
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
def _limit_name(limit: int) -> str:
|
|
223
|
-
return {
|
|
224
|
-
resource.RLIMIT_AS: "memory_bytes",
|
|
225
|
-
resource.RLIMIT_FSIZE: "file_bytes",
|
|
226
|
-
resource.RLIMIT_NOFILE: "open_files",
|
|
227
|
-
resource.RLIMIT_CORE: "core_bytes",
|
|
228
|
-
}[limit]
|
|
341
|
+
"""Cap this process before it imports world code. Returns each limit as
|
|
342
|
+
set, by label, so the worker can report them."""
|
|
343
|
+
limits = (
|
|
344
|
+
AddressSpaceLimit(resource.RLIMIT_AS, memory_mb << 20, "memory_bytes"),
|
|
345
|
+
ProcessLimit(resource.RLIMIT_FSIZE, file_mb << 20, "file_bytes"),
|
|
346
|
+
ProcessLimit(resource.RLIMIT_NOFILE, nofile, "open_files"),
|
|
347
|
+
ProcessLimit(resource.RLIMIT_CORE, 0, "core_bytes"),
|
|
348
|
+
)
|
|
349
|
+
return {limit.label: limit.apply() for limit in limits}
|
|
229
350
|
|
|
230
351
|
|
|
231
352
|
def worker_environment(
|
|
@@ -246,7 +367,6 @@ def worker_environment(
|
|
|
246
367
|
WORLDS_WORKER_SECRET=secret,
|
|
247
368
|
WORLDS_WORKER_MEMORY_MB=str(memory_mb),
|
|
248
369
|
WORLDS_INGRESS_DOMAIN=worlds_domain,
|
|
249
|
-
WORLDS_SUPERVISOR_PID=str(os.getpid()),
|
|
250
370
|
)
|
|
251
371
|
return env
|
|
252
372
|
|
|
@@ -295,20 +415,13 @@ class ConfinedVerifierSandbox(VerifierSandbox):
|
|
|
295
415
|
return verifier_environment(home=home)
|
|
296
416
|
|
|
297
417
|
|
|
298
|
-
# --------------------------------------------------------------------------- #
|
|
299
|
-
# The supervisor's side: start, reach and stop workers.
|
|
300
|
-
# --------------------------------------------------------------------------- #
|
|
301
|
-
|
|
302
|
-
|
|
303
418
|
@dataclass
|
|
304
419
|
class Worker:
|
|
305
420
|
version_id: str
|
|
306
421
|
process: subprocess.Popen
|
|
307
422
|
socket_path: Path
|
|
308
423
|
secret: str
|
|
309
|
-
#: Sessions the supervisor has placed on this worker.
|
|
310
424
|
sessions: set[str] = field(default_factory=set)
|
|
311
|
-
#: When the worker last had a session, for idle reaping.
|
|
312
425
|
idle_since: float = field(default_factory=time.time)
|
|
313
426
|
|
|
314
427
|
@property
|
|
@@ -327,25 +440,78 @@ class Worker:
|
|
|
327
440
|
*,
|
|
328
441
|
control: bool = False,
|
|
329
442
|
timeout: float = 60.0,
|
|
443
|
+
limit: int | None = None,
|
|
330
444
|
) -> tuple[int, list[tuple[str, str]], bytes]:
|
|
331
445
|
"""One request over the socket. ``control`` adds the worker's own
|
|
332
|
-
secret, so the worker's control plane answers. A worker that
|
|
333
|
-
|
|
446
|
+
secret, so the worker's control plane answers. A worker that takes no connection raises
|
|
447
|
+
:class:`WorkerUnreached`; one that drops the request or answers past ``limit`` bytes raises
|
|
448
|
+
:class:`WorkerGone`. The whole exchange gets ``timeout`` seconds, not each read: an answer
|
|
449
|
+
sent slowly is cut off when they pass (the connection is shut, not waited out) and raises
|
|
450
|
+
:class:`WorkerTimeout`."""
|
|
334
451
|
forward = {k: v for k, v in headers.items() if k.lower() not in UNFORWARDED_HEADERS}
|
|
335
452
|
if control:
|
|
336
453
|
forward["X-Host-Secret"] = self.secret
|
|
337
|
-
|
|
454
|
+
started = time.monotonic()
|
|
455
|
+
conn = self._connected(timeout)
|
|
456
|
+
cut = threading.Event()
|
|
338
457
|
try:
|
|
339
|
-
conn.
|
|
340
|
-
|
|
341
|
-
|
|
458
|
+
with _deadline(conn.sock, timeout - (time.monotonic() - started)) as cut:
|
|
459
|
+
conn.request(method, path, body=body or None, headers=forward)
|
|
460
|
+
answer = conn.getresponse()
|
|
461
|
+
payload = answer.read() if limit is None else answer.read(limit + 1)
|
|
462
|
+
if cut.is_set():
|
|
463
|
+
raise WorkerTimeout(
|
|
464
|
+
f"worker for version {self.version_id} (pid {self.pid}) did not answer in {timeout:g} s"
|
|
465
|
+
)
|
|
466
|
+
if limit is not None and len(payload) > limit:
|
|
467
|
+
raise WorkerGone(f"worker for version {self.version_id} (pid {self.pid}) answered past {limit} bytes")
|
|
342
468
|
return answer.status, answer.getheaders(), payload
|
|
343
469
|
except (OSError, http.client.HTTPException) as error:
|
|
344
|
-
|
|
345
|
-
|
|
470
|
+
gone = WorkerTimeout if cut.is_set() or isinstance(error, TimeoutError) else WorkerGone
|
|
471
|
+
raise gone(f"worker for version {self.version_id} (pid {self.pid}) did not answer: {error}") from error
|
|
472
|
+
finally:
|
|
473
|
+
conn.close()
|
|
474
|
+
|
|
475
|
+
def _connected(self, timeout: float) -> UnixHTTPConnection:
|
|
476
|
+
conn = UnixHTTPConnection(self.socket_path, timeout=timeout)
|
|
477
|
+
try:
|
|
478
|
+
conn.connect()
|
|
479
|
+
except OSError as error:
|
|
480
|
+
conn.close()
|
|
481
|
+
raise WorkerUnreached(
|
|
482
|
+
f"worker for version {self.version_id} (pid {self.pid}) took no connection: {error}"
|
|
346
483
|
) from error
|
|
484
|
+
return conn
|
|
485
|
+
|
|
486
|
+
def probe(
|
|
487
|
+
self, path: str, *, deadline_s: float = HEALTH_TIMEOUT_S, limit: int = PROBE_LIMIT_BYTES
|
|
488
|
+
) -> tuple[int, bytes]:
|
|
489
|
+
"""One control GET whose whole answer must arrive within ``deadline_s`` and ``limit`` bytes.
|
|
490
|
+
The worker runs tenant code: an answer sent slowly or without end is cut off when the
|
|
491
|
+
deadline passes (the connection is shut, not waited out), and raises :class:`WorkerGone`."""
|
|
492
|
+
unanswered = f"worker for version {self.version_id} (pid {self.pid}) did not answer {path}"
|
|
493
|
+
conn = UnixHTTPConnection(self.socket_path, timeout=deadline_s)
|
|
494
|
+
try:
|
|
495
|
+
conn.connect()
|
|
496
|
+
with _deadline(conn.sock, deadline_s) as cut:
|
|
497
|
+
conn.request("GET", path, headers={"X-Host-Secret": self.secret})
|
|
498
|
+
answer = conn.getresponse()
|
|
499
|
+
payload = answer.read(limit + 1)
|
|
500
|
+
except (OSError, http.client.HTTPException) as error:
|
|
501
|
+
raise WorkerGone(unanswered) from error
|
|
347
502
|
finally:
|
|
348
503
|
conn.close()
|
|
504
|
+
if cut.is_set() or len(payload) > limit:
|
|
505
|
+
raise WorkerGone(f"{unanswered} within {deadline_s:g} s and {limit} bytes")
|
|
506
|
+
return answer.status, payload
|
|
507
|
+
|
|
508
|
+
def answers_health(self, timeout_s: float = HEALTH_TIMEOUT_S) -> bool:
|
|
509
|
+
"""Whether the worker is alive and answers its health check, whole, in time."""
|
|
510
|
+
try:
|
|
511
|
+
status, _ = self.probe("/healthz", deadline_s=timeout_s)
|
|
512
|
+
except WorkerGone:
|
|
513
|
+
return False
|
|
514
|
+
return status == 200
|
|
349
515
|
|
|
350
516
|
def stop(self, grace_s: float = 5.0) -> None:
|
|
351
517
|
if self.alive():
|
|
@@ -358,6 +524,69 @@ class Worker:
|
|
|
358
524
|
self.socket_path.unlink(missing_ok=True)
|
|
359
525
|
|
|
360
526
|
|
|
527
|
+
class Finishing:
|
|
528
|
+
"""What leaves a session on a version worker, finished off the lease sweeper's thread: a few daemon
|
|
529
|
+
threads, one job at a time for a version in the order they came (so a worker that never answers
|
|
530
|
+
holds one thread, not the pool). A session with no worker is finished at once, where it is."""
|
|
531
|
+
|
|
532
|
+
def __init__(self, threads: int = FINISH_THREADS) -> None:
|
|
533
|
+
self.threads = threads
|
|
534
|
+
self._changed = threading.Condition()
|
|
535
|
+
self._waiting: dict[str, collections.deque[Callable[[], None]]] = {}
|
|
536
|
+
self._serving: set[str] = set()
|
|
537
|
+
self._started = 0
|
|
538
|
+
|
|
539
|
+
def leave(self, worker: Worker | None, job: Callable[[], None]) -> None:
|
|
540
|
+
if worker is None:
|
|
541
|
+
job()
|
|
542
|
+
return
|
|
543
|
+
with self._changed:
|
|
544
|
+
self._waiting.setdefault(worker.version_id, collections.deque()).append(job)
|
|
545
|
+
self._staff()
|
|
546
|
+
self._changed.notify()
|
|
547
|
+
|
|
548
|
+
def settle(self, deadline_s: float = FINISH_SETTLE_S) -> bool:
|
|
549
|
+
"""Wait up to ``deadline_s`` for every job handed off to be done; whether they are."""
|
|
550
|
+
until = time.monotonic() + deadline_s
|
|
551
|
+
with self._changed:
|
|
552
|
+
while self._waiting:
|
|
553
|
+
remaining = until - time.monotonic()
|
|
554
|
+
if remaining <= 0:
|
|
555
|
+
return False
|
|
556
|
+
self._changed.wait(remaining)
|
|
557
|
+
return True
|
|
558
|
+
|
|
559
|
+
def _staff(self) -> None:
|
|
560
|
+
if self._started < self.threads:
|
|
561
|
+
self._started += 1
|
|
562
|
+
threading.Thread(target=self._run, name=f"world-host-finish-{self._started}", daemon=True).start()
|
|
563
|
+
|
|
564
|
+
def _run(self) -> None:
|
|
565
|
+
while True:
|
|
566
|
+
with self._changed:
|
|
567
|
+
self._changed.wait_for(lambda: self._next() is not None)
|
|
568
|
+
version = self._next()
|
|
569
|
+
assert version is not None
|
|
570
|
+
job = self._waiting[version].popleft()
|
|
571
|
+
self._serving.add(version)
|
|
572
|
+
self._work(version, job)
|
|
573
|
+
|
|
574
|
+
def _next(self) -> str | None:
|
|
575
|
+
return next((version for version, jobs in self._waiting.items() if jobs and version not in self._serving), None)
|
|
576
|
+
|
|
577
|
+
def _work(self, version: str, job: Callable[[], None]) -> None:
|
|
578
|
+
try:
|
|
579
|
+
job()
|
|
580
|
+
except Exception as error: # noqa: BLE001 - one session's finish never stops the pool
|
|
581
|
+
log_line(f"world host: a session did not finish: {type(error).__name__}")
|
|
582
|
+
finally:
|
|
583
|
+
with self._changed:
|
|
584
|
+
self._serving.discard(version)
|
|
585
|
+
if not self._waiting[version]:
|
|
586
|
+
del self._waiting[version]
|
|
587
|
+
self._changed.notify_all()
|
|
588
|
+
|
|
589
|
+
|
|
361
590
|
class WorkerPool:
|
|
362
591
|
"""Every version worker on this replica, started on the first session of a
|
|
363
592
|
version and stopped once it has had none for a while."""
|
|
@@ -387,16 +616,20 @@ class WorkerPool:
|
|
|
387
616
|
"(WORLDS_WORKER_BUDGET_MB, WORLDS_WORKER_FOOTPRINT_MB)"
|
|
388
617
|
)
|
|
389
618
|
self.root = Path(cache_root)
|
|
390
|
-
self.workers_root = self.root /
|
|
619
|
+
self.workers_root = self.root / WORKERS_DIR
|
|
391
620
|
# Workers live and die with the supervisor; whatever a previous one
|
|
392
621
|
# left (sockets, session files) is unreachable.
|
|
393
622
|
shutil.rmtree(self.workers_root, ignore_errors=True)
|
|
394
623
|
self.workers_root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
624
|
+
# AF_UNIX caps a socket path at 103 bytes on macOS (107 on Linux): the
|
|
625
|
+
# sockets live in a short private (0700) directory of their own, so no
|
|
626
|
+
# cache root or version id is too long to serve.
|
|
627
|
+
self._sockets = tempfile.TemporaryDirectory(prefix="gw-")
|
|
628
|
+
self.sockets = Path(self._sockets.name)
|
|
395
629
|
self.worlds_domain = worlds_domain
|
|
396
630
|
self.memory_mb = memory_mb
|
|
397
631
|
self.budget_mb = budget_mb
|
|
398
632
|
self.footprint_mb = footprint_mb
|
|
399
|
-
#: Workers live at once: the budget's worth of footprints.
|
|
400
633
|
self.max_workers = budget_mb // footprint_mb
|
|
401
634
|
self.idle_s = idle_s
|
|
402
635
|
self.python = python
|
|
@@ -406,24 +639,54 @@ class WorkerPool:
|
|
|
406
639
|
def get(self, version_id: str) -> Worker:
|
|
407
640
|
"""The live worker for a version, started if there is none. Starting one
|
|
408
641
|
past the budget first stops workers holding no sessions (whatever their
|
|
409
|
-
idle time); when every one is in use, raises :class:`WorkerCapacity`.
|
|
642
|
+
idle time); when every one is in use, raises :class:`WorkerCapacity`. A
|
|
643
|
+
dead worker that still holds sessions is never replaced here: its
|
|
644
|
+
sessions' files are on disk, and the supervisor recovers them through
|
|
645
|
+
:meth:`replace` (raises :class:`WorkerGone`)."""
|
|
410
646
|
with self._lock:
|
|
411
647
|
worker = self._workers.get(version_id)
|
|
412
648
|
if worker is not None and worker.alive():
|
|
413
649
|
return worker
|
|
650
|
+
if worker is not None and worker.sessions:
|
|
651
|
+
raise WorkerGone(f"worker for version {version_id} (pid {worker.pid}) exited holding sessions")
|
|
414
652
|
if worker is not None:
|
|
415
653
|
worker.stop()
|
|
416
654
|
del self._workers[version_id]
|
|
417
|
-
self.
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
655
|
+
return self._start(version_id)
|
|
656
|
+
|
|
657
|
+
def replace(self, dead: Worker) -> Worker:
|
|
658
|
+
"""A fresh worker in ``dead``'s place, its sessions left for the caller to
|
|
659
|
+
reopen; ``dead`` is stopped if it still runs. Refused (:class:`WorkerGone`)
|
|
660
|
+
when the pool no longer holds ``dead``."""
|
|
661
|
+
with self._lock:
|
|
662
|
+
if self._workers.get(dead.version_id) is not dead:
|
|
663
|
+
raise WorkerGone(f"worker for version {dead.version_id} (pid {dead.pid}) was replaced already")
|
|
664
|
+
del self._workers[dead.version_id]
|
|
665
|
+
dead.stop()
|
|
666
|
+
return self._start(dead.version_id)
|
|
667
|
+
|
|
668
|
+
def _start(self, version_id: str) -> Worker:
|
|
669
|
+
self._make_room()
|
|
670
|
+
worker = self._spawn(version_id)
|
|
671
|
+
self._workers[version_id] = worker
|
|
672
|
+
return worker
|
|
673
|
+
|
|
674
|
+
def dead(self) -> list[Worker]:
|
|
675
|
+
"""The workers that exited still holding sessions: theirs to recover."""
|
|
676
|
+
with self._lock:
|
|
677
|
+
return [w for w in self._workers.values() if w.sessions and not w.alive()]
|
|
678
|
+
|
|
679
|
+
def live(self) -> list[Worker]:
|
|
680
|
+
"""The workers still running, holding sessions or not."""
|
|
681
|
+
with self._lock:
|
|
682
|
+
return [w for w in self._workers.values() if w.alive()]
|
|
421
683
|
|
|
422
684
|
def _make_room(self) -> None:
|
|
423
685
|
"""Under ``_lock``: leave one worker's worth of the budget free, by
|
|
424
|
-
forgetting exited workers first, then stopping
|
|
425
|
-
longest idle first, and refusing when every live
|
|
426
|
-
|
|
686
|
+
forgetting exited workers that hold no sessions first, then stopping
|
|
687
|
+
session-less ones longest idle first, and refusing when every live
|
|
688
|
+
worker is in use."""
|
|
689
|
+
exited = [v for v, w in self._workers.items() if not w.alive() and not w.sessions]
|
|
427
690
|
for version_id in exited:
|
|
428
691
|
self._workers.pop(version_id).stop()
|
|
429
692
|
spare = sorted(
|
|
@@ -442,15 +705,6 @@ class WorkerPool:
|
|
|
442
705
|
with self._lock:
|
|
443
706
|
return self._workers.get(version_id)
|
|
444
707
|
|
|
445
|
-
def drop(self, version_id: str) -> Worker | None:
|
|
446
|
-
"""Stop and forget a version's worker; returns it (with the sessions
|
|
447
|
-
it held) so the caller can forget those too."""
|
|
448
|
-
with self._lock:
|
|
449
|
-
worker = self._workers.pop(version_id, None)
|
|
450
|
-
if worker is not None:
|
|
451
|
-
worker.stop()
|
|
452
|
-
return worker
|
|
453
|
-
|
|
454
708
|
def reap_idle(self, now: float | None = None) -> list[str]:
|
|
455
709
|
"""Stop workers that have had no session for ``idle_s``; returns their versions."""
|
|
456
710
|
now = time.time() if now is None else now
|
|
@@ -475,7 +729,7 @@ class WorkerPool:
|
|
|
475
729
|
def _spawn(self, version_id: str) -> Worker:
|
|
476
730
|
home = self.workers_root / version_id
|
|
477
731
|
home.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
478
|
-
socket_path =
|
|
732
|
+
socket_path = self.sockets / f"{hashlib.sha256(version_id.encode()).hexdigest()[:16]}.sock"
|
|
479
733
|
secret = secrets.token_urlsafe(24)
|
|
480
734
|
# The supervisor binds and listens; the worker inherits the descriptor
|
|
481
735
|
# and serves on it, so the socket is the supervisor's from the start.
|
|
@@ -519,13 +773,8 @@ class WorkerPool:
|
|
|
519
773
|
raise WorkerGone(
|
|
520
774
|
f"worker for version {worker.version_id} exited with {worker.process.returncode} on start"
|
|
521
775
|
)
|
|
522
|
-
if worker.socket_path.exists():
|
|
523
|
-
|
|
524
|
-
status, _, _ = worker.request("GET", "/healthz", {}, timeout=2.0)
|
|
525
|
-
except WorkerGone:
|
|
526
|
-
status = 0
|
|
527
|
-
if status == 200:
|
|
528
|
-
return
|
|
776
|
+
if worker.socket_path.exists() and worker.answers_health():
|
|
777
|
+
return
|
|
529
778
|
time.sleep(0.05)
|
|
530
779
|
worker.stop()
|
|
531
780
|
raise WorkerGone(f"worker for version {worker.version_id} did not come up in {WORKER_START_TIMEOUT_S:.0f}s")
|