@withgateway/sdk 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. package/README.md +8 -6
  2. package/dist/cli.js +4372 -2265
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +4362 -2256
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +259 -58
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +249 -50
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/hooks/index.d.mts +1 -1
  13. package/dist/hooks/index.d.ts +1 -1
  14. package/dist/hooks/index.js +124 -11
  15. package/dist/hooks/index.js.map +1 -1
  16. package/dist/hooks/index.mjs +124 -11
  17. package/dist/hooks/index.mjs.map +1 -1
  18. package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
  19. package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
  20. package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
  21. package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
  22. package/dist/index.d.mts +5 -5
  23. package/dist/index.d.ts +5 -5
  24. package/dist/index.js +2240 -789
  25. package/dist/index.js.map +1 -1
  26. package/dist/index.mjs +2150 -739
  27. package/dist/index.mjs.map +1 -1
  28. package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
  29. package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
  30. package/dist/traces/index.d.mts +1 -1
  31. package/dist/traces/index.d.ts +1 -1
  32. package/dist/traces/index.js +253 -52
  33. package/dist/traces/index.js.map +1 -1
  34. package/dist/traces/index.mjs +243 -44
  35. package/dist/traces/index.mjs.map +1 -1
  36. package/dist/tracing/index.d.mts +2 -2
  37. package/dist/tracing/index.d.ts +2 -2
  38. package/dist/tracing/index.js +75 -13
  39. package/dist/tracing/index.js.map +1 -1
  40. package/dist/tracing/index.mjs +75 -13
  41. package/dist/tracing/index.mjs.map +1 -1
  42. package/dist/worlds/index.d.mts +893 -482
  43. package/dist/worlds/index.d.ts +893 -482
  44. package/dist/worlds/index.js +1968 -669
  45. package/dist/worlds/index.js.map +1 -1
  46. package/dist/worlds/index.mjs +1960 -662
  47. package/dist/worlds/index.mjs.map +1 -1
  48. package/package.json +1 -1
  49. package/runtime/gatewaysdk/environment/_bundle.py +2 -27
  50. package/runtime/gatewaysdk/environment/_materialize.py +48 -88
  51. package/runtime/gatewaysdk/environment/_world.py +155 -237
  52. package/runtime/gatewaysdk/environment/backing.py +133 -0
  53. package/runtime/gatewaysdk/environment/core.py +2 -17
  54. package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
  55. package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
  56. package/runtime/gatewaysdk/environment/runtime.py +8 -13
  57. package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
  58. package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
  59. package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
  60. package/runtime/gatewaysdk/environment/schema/api.py +347 -892
  61. package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
  62. package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
  63. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
  64. package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
  65. package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
  66. package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
  67. package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
  68. package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
  69. package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
  70. package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
  71. package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
  72. package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
  73. package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
  74. package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
  75. package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
  76. package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
  77. package/runtime/gatewaysdk/environment/schema/data.py +60 -216
  78. package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
  79. package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
  80. package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
  81. package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
  82. package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
  83. package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
  84. package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
  85. package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
  86. package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
  87. package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
  88. package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
  89. package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
  90. package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
  91. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
  92. package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
  93. package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
  94. package/runtime/gatewaysdk/environment/schema/door.py +899 -0
  95. package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
  96. package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
  97. package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
  98. package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
  99. package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
  100. package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
  101. package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
  102. package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
  103. package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
  104. package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
  105. package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
  106. package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
  107. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
  108. package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
  109. package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
  110. package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
  111. package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
  112. package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
  113. package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
  114. package/runtime/gatewaysdk/environment/schema/local.py +58 -170
  115. package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
  116. package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
  117. package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
  118. package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
  119. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
  120. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
  121. package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
  122. package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
  123. package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
  124. package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
  125. package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
  126. package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
  127. package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
  128. package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
  129. package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
  130. package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
  131. package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
  132. package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
  133. package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
  134. package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
  135. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
  136. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
  137. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
  138. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
  139. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
  140. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
  141. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
  142. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
  143. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
  144. package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
  145. package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
  146. package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
  147. package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
  148. package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
  149. package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
  150. package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
  151. package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
  152. package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
  153. package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
  154. package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
  155. package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
  156. package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
  157. package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
  158. package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
  159. package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
  160. package/runtime/gatewaysdk/environment/score.py +57 -0
  161. package/runtime/gatewaysdk/environment/secret_values.py +36 -0
  162. package/runtime/gatewaysdk/environment/world.py +43 -4
  163. package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
  164. package/skills/agents/connector-template-author.md +5 -1
  165. package/skills/benchmarks-getting-started/SKILL.md +10 -11
  166. package/skills/connector-schema-authoring/SKILL.md +49 -16
  167. package/skills/connector-schema-authoring/connector-worlds.md +142 -11
  168. package/skills/connector-schema-authoring/world-schema.md +103 -51
  169. package/skills/world-data-ingestion/ROWS.md +39 -9
  170. package/skills/world-data-ingestion/SKILL.md +27 -27
  171. package/skills/worlds-getting-started/SKILL.md +33 -33
  172. package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
  173. package/skills/worlds-getting-started/references/gotchas.md +49 -1
  174. package/skills/worlds-getting-started/references/journal-checks.md +182 -0
  175. package/skills/worlds-getting-started/references/openapi-import.md +50 -21
  176. package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
@@ -9,7 +9,7 @@ it. The worker is the same :class:`host.WorldHost` in a smaller room:
9
9
  * its own process, so a handler that loops, leaks or crashes takes down that
10
10
  version's sessions on this replica and nothing else;
11
11
  * rlimits on address space, open files and file size, set before any world
12
- code is imported;
12
+ code is imported (:class:`AddressSpaceLimit` says how macOS differs);
13
13
  * a scrubbed environment: the control-plane secret, cloud credentials and the
14
14
  rest of the supervisor's environment are not inherited. The worker gets a
15
15
  secret of its own, good for its socket alone;
@@ -17,10 +17,11 @@ it. The worker is the same :class:`host.WorldHost` in a smaller room:
17
17
  a process even when they share a host;
18
18
  * a socket only the supervisor is answered on. Workers run as the host's user,
19
19
  so file modes cannot keep one worker's handler from opening a sibling's
20
- socket path: the worker asks the kernel who connected (``SO_PEERCRED``) and
21
- serves the supervisor's pid alone. The listening socket is the supervisor's,
22
- bound before the worker starts and handed down as a file descriptor, so a
23
- worker never binds a path of its own;
20
+ socket path: the worker asks the kernel who connected (``SO_PEERCRED``, on
21
+ macOS ``LOCAL_PEERPID``) and serves the supervisor's pid alone. The listening
22
+ socket is the supervisor's, bound before the worker starts in a short private
23
+ directory and handed down as a file descriptor, so a worker never binds a
24
+ path of its own;
24
25
  * a worker nobody can read: it marks itself non-dumpable, which closes
25
26
  ``/proc/<pid>/environ``, ``/proc/<pid>/mem`` and ``ptrace`` to every process
26
27
  that is not root, siblings under the same uid included. Its secret and its
@@ -34,7 +35,9 @@ fence (a sandboxed runtime class, a read-only root) when it is wanted; see
34
35
 
35
36
  from __future__ import annotations
36
37
 
38
+ import collections
37
39
  import ctypes
40
+ import hashlib
38
41
  import http.client
39
42
  import os
40
43
  import resource
@@ -44,14 +47,19 @@ import socket
44
47
  import struct
45
48
  import subprocess
46
49
  import sys
50
+ import tempfile
47
51
  import threading
48
52
  import time
49
- from collections.abc import Mapping
53
+ from collections.abc import Callable, Iterator, Mapping
54
+ from contextlib import contextmanager, suppress
50
55
  from dataclasses import dataclass, field
51
56
  from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
52
57
  from pathlib import Path
58
+ from typing import Protocol
53
59
 
60
+ from .cache_layout import WORKERS_DIR
54
61
  from .grading import VerifierSandbox
62
+ from .handler_errors import log_line
55
63
 
56
64
  #: Address space a worker may use, in MiB, unless ``WORLDS_WORKER_MEMORY_MB`` says otherwise.
57
65
  DEFAULT_WORKER_MEMORY_MB = 2048
@@ -63,21 +71,26 @@ DEFAULT_WORKER_MEMORY_MB = 2048
63
71
  DEFAULT_WORKER_BUDGET_MB = 4096
64
72
  #: What one worker is counted at against the budget, in MiB, unless
65
73
  #: ``WORLDS_WORKER_FOOTPRINT_MB`` says otherwise. Unset, a worker is counted at
66
- #: its whole address-space cap — two of 2 GiB inside a 6 GiB container — which
67
- #: is the safe reading and a 50x over-reservation: a measured worker holding
68
- #: sessions is ~40 MiB resident (prod, 2026-09-17). A fleet that must host many
69
- #: worlds at once sets this to the footprint it has measured, keeps the
70
- #: address-space cap as the rail each worker cannot cross, and sizes the
71
- #: container for footprint x workers plus the supervisor's share.
74
+ #: its whole address-space cap: the safe reading, and a 50x over-reservation of
75
+ #: the ~40 MiB a worker holding sessions keeps resident. A fleet that must host
76
+ #: many worlds at once sets the footprint it measured, keeps the address-space
77
+ #: cap as the rail each worker cannot cross, and sizes the container for
78
+ #: footprint x workers plus the supervisor's share.
72
79
  DEFAULT_WORKER_FOOTPRINT_MB: int | None = None
73
80
  #: Largest file a worker may write (its session databases), in MiB.
74
81
  DEFAULT_WORKER_FILE_MB = 4096
75
82
  #: Open files per worker: one per live session plus the sockets serving them.
76
83
  DEFAULT_WORKER_NOFILE = 4096
77
- #: How long a worker with no sessions is kept before it is stopped.
84
+ FLOOR_PRECISION_BYTES = 1 << 20
85
+ #: The largest limit a kernel reports, which ``RLIM_INFINITY`` is read as in arithmetic: macOS
86
+ #: spells it so, Linux spells it -1.
87
+ MAX_RLIMIT = 2**63 - 1
78
88
  DEFAULT_WORKER_IDLE_S = 300.0
79
- #: How long the supervisor waits for a fresh worker to answer on its socket.
80
89
  WORKER_START_TIMEOUT_S = 30.0
90
+ #: How long one health probe waits: a worker that cannot answer it in this time is not serving.
91
+ HEALTH_TIMEOUT_S = 2.0
92
+ #: The most a probe reads of a worker's answer: the worker runs tenant code, so its size is not its to choose.
93
+ PROBE_LIMIT_BYTES = 64 << 10
81
94
  #: A worker's sessions live as long as the supervisor's do; the supervisor
82
95
  #: holds the leases, so the worker's own are effectively unbounded.
83
96
  WORKER_LEASE_S = 365.0 * 86400.0
@@ -87,10 +100,20 @@ INHERITED_ENV = ("PATH", "PYTHONPATH", "LANG", "LC_ALL", "TZ", "VIRTUAL_ENV")
87
100
  #: Headers the supervisor never forwards to a worker: the ones that describe
88
101
  #: the supervisor's connection, and its own control-plane secret.
89
102
  UNFORWARDED_HEADERS = frozenset({"connection", "content-length", "transfer-encoding", "x-host-secret"})
103
+ FINISH_THREADS = 8
104
+ FINISH_SETTLE_S = 10.0
90
105
 
91
106
 
92
107
  class WorkerGone(Exception):
93
- """The worker did not answer: it exited, or never came up."""
108
+ """The worker did not answer: it exited, never came up, or dropped the request."""
109
+
110
+
111
+ class WorkerTimeout(WorkerGone):
112
+ """The worker did not answer in time; it may be alive and still working on the request."""
113
+
114
+
115
+ class WorkerUnreached(WorkerGone):
116
+ """The worker took no connection: the request never reached it."""
94
117
 
95
118
 
96
119
  class WorkerCapacity(Exception):
@@ -98,11 +121,6 @@ class WorkerCapacity(Exception):
98
121
  worker for another version can start here."""
99
122
 
100
123
 
101
- # --------------------------------------------------------------------------- #
102
- # HTTP over a unix socket: the only way into a worker.
103
- # --------------------------------------------------------------------------- #
104
-
105
-
106
124
  class UnixHTTPConnection(http.client.HTTPConnection):
107
125
  def __init__(self, path: str | Path, timeout: float | None = None):
108
126
  super().__init__("worker", timeout=timeout)
@@ -115,6 +133,24 @@ class UnixHTTPConnection(http.client.HTTPConnection):
115
133
  self.sock = sock
116
134
 
117
135
 
136
+ def _hang_up(sock: socket.socket, cut: threading.Event) -> None:
137
+ cut.set()
138
+ with suppress(OSError):
139
+ sock.shutdown(socket.SHUT_RDWR)
140
+
141
+
142
+ @contextmanager
143
+ def _deadline(sock: socket.socket, seconds: float) -> Iterator[threading.Event]:
144
+ """Shut ``sock`` once ``seconds`` pass, so nothing read from it waits longer; the event says it was."""
145
+ cut = threading.Event()
146
+ timer = threading.Timer(seconds, _hang_up, (sock, cut))
147
+ timer.start()
148
+ try:
149
+ yield cut
150
+ finally:
151
+ timer.cancel()
152
+
153
+
118
154
  def bind_worker_socket(path: str | Path) -> socket.socket:
119
155
  """The supervisor's end of a worker: a listening unix socket at ``path``,
120
156
  ``0600`` inside a ``0700`` directory, ready to hand to the worker."""
@@ -128,18 +164,40 @@ def bind_worker_socket(path: str | Path) -> socket.socket:
128
164
  return sock
129
165
 
130
166
 
131
- def peer_pid(conn: socket.socket) -> int | None:
132
- """The pid of whoever connected, as the kernel reports it (``SO_PEERCRED``:
133
- ``struct ucred {pid, uid, gid}``); None where the platform has no such
134
- thing."""
135
- option = getattr(socket, "SO_PEERCRED", None)
136
- if option is None:
137
- return None
138
- raw = conn.getsockopt(socket.SOL_SOCKET, option, struct.calcsize("3i"))
167
+ #: macOS's ``SOL_LOCAL`` and ``LOCAL_PEERPID`` from <sys/un.h>; CPython exports neither.
168
+ _SOL_LOCAL, _LOCAL_PEERPID = 0, 0x002
169
+
170
+
171
+ def _linux_peer_pid(conn: socket.socket) -> int:
172
+ raw = conn.getsockopt(socket.SOL_SOCKET, socket.SO_PEERCRED, struct.calcsize("3i"))
139
173
  pid, _uid, _gid = struct.unpack("3i", raw)
140
174
  return pid
141
175
 
142
176
 
177
+ def _darwin_peer_pid(conn: socket.socket) -> int:
178
+ raw = conn.getsockopt(_SOL_LOCAL, _LOCAL_PEERPID, struct.calcsize("i"))
179
+ return struct.unpack("i", raw)[0]
180
+
181
+
182
+ _PEER_PID = {"linux": _linux_peer_pid, "darwin": _darwin_peer_pid}
183
+
184
+
185
+ def peer_pid(conn: socket.socket) -> int | None:
186
+ """The pid of whoever connected, as the kernel reports it: Linux's
187
+ ``SO_PEERCRED`` (``struct ucred {pid, uid, gid}``), macOS's
188
+ ``LOCAL_PEERPID``. None where the platform has no such thing, or where the
189
+ kernel cannot name the peer (macOS raises ``ENOTCONN`` once the client has
190
+ hung up): a worker serving its supervisor alone refuses both, and keeps
191
+ serving, since ``socketserver`` asks outside its own error handling."""
192
+ read = _PEER_PID.get(sys.platform)
193
+ if read is None:
194
+ return None
195
+ try:
196
+ return read(conn)
197
+ except OSError:
198
+ return None
199
+
200
+
143
201
  class UnixHTTPServer(ThreadingHTTPServer):
144
202
  """The worker's server: a ``ThreadingHTTPServer`` over the listening unix
145
203
  socket the supervisor handed it (an inherited descriptor), or one it binds
@@ -195,37 +253,100 @@ def seal_process() -> bool:
195
253
  return False
196
254
 
197
255
 
256
+ class Kernel(Protocol):
257
+ """Where a process's limits are read and set: the ``resource`` module."""
258
+
259
+ RLIM_INFINITY: int
260
+
261
+ def getrlimit(self, name: int, /) -> tuple[int, int]: ...
262
+
263
+ def setrlimit(self, name: int, limits: tuple[int, int], /) -> None: ...
264
+
265
+
266
+ @dataclass(frozen=True)
267
+ class ProcessLimit:
268
+ """One rlimit on this process, soft and hard alike. A limit the kernel
269
+ holds lower than asked is lowered to its hard limit, never raised; one it
270
+ refuses fails the start."""
271
+
272
+ name: int
273
+ wanted: int
274
+ label: str
275
+ kernel: Kernel = resource
276
+
277
+ def apply(self) -> int:
278
+ value = self.within_hard()
279
+ self.kernel.setrlimit(self.name, (value, value))
280
+ return value
281
+
282
+ def hard(self) -> int:
283
+ return self.kernel.getrlimit(self.name)[1]
284
+
285
+ def ceiling(self) -> int:
286
+ """The hard limit as a number: ``RLIM_INFINITY`` reads as :data:`MAX_RLIMIT`."""
287
+ hard = self.hard()
288
+ return MAX_RLIMIT if hard == self.kernel.RLIM_INFINITY else hard
289
+
290
+ def within_hard(self) -> int:
291
+ return min(self.wanted, self.ceiling())
292
+
293
+
294
+ class AddressSpaceLimit(ProcessLimit):
295
+ """The memory cap. Linux takes any cap up to the hard limit, so there the
296
+ budget is the cap exactly. macOS refuses a cap below what the process
297
+ already maps (about 392 GiB on Apple Silicon, mostly the shared region's
298
+ reservation), so there the budget is granted on top of that floor: it still
299
+ bounds growth, and a kernel that takes no cap at all fails the start."""
300
+
301
+ def apply(self) -> int:
302
+ try:
303
+ return super().apply()
304
+ except ValueError:
305
+ return self._above_floor()
306
+
307
+ def _above_floor(self) -> int:
308
+ ceiling = self.ceiling()
309
+ floor = self._lowest_taken(ceiling)
310
+ if floor >= ceiling:
311
+ raise ValueError(f"the kernel refuses every {self.label} cap under its hard limit")
312
+ value = min(floor + self.wanted, ceiling)
313
+ self.kernel.setrlimit(self.name, (value, value))
314
+ return value
315
+
316
+ def _lowest_taken(self, ceiling: int) -> int:
317
+ """The lowest cap the kernel takes, to within FLOOR_PRECISION_BYTES.
318
+ Each probe moves the soft limit alone and is undone at once: a lowered
319
+ hard limit cannot be raised again, and a process left near its floor
320
+ fails its next allocation."""
321
+ restore = self.kernel.getrlimit(self.name)
322
+ refused, taken = self.within_hard(), ceiling
323
+ while taken - refused > FLOOR_PRECISION_BYTES:
324
+ probe = (refused + taken) // 2
325
+ try:
326
+ self.kernel.setrlimit(self.name, (probe, restore[1]))
327
+ except ValueError:
328
+ refused = probe
329
+ continue
330
+ self.kernel.setrlimit(self.name, restore)
331
+ taken = probe
332
+ return taken
333
+
334
+
198
335
  def apply_limits(
199
336
  *,
200
337
  memory_mb: int = DEFAULT_WORKER_MEMORY_MB,
201
338
  file_mb: int = DEFAULT_WORKER_FILE_MB,
202
339
  nofile: int = DEFAULT_WORKER_NOFILE,
203
340
  ) -> dict[str, int]:
204
- """Cap this process before it imports world code. A limit the kernel will
205
- not grant (a hard limit lower than asked) is lowered to what it grants
206
- rather than refused: the worker still runs, just tighter. Returns the
207
- limits as set, so the worker can report them."""
208
- applied: dict[str, int] = {}
209
- for name, wanted in (
210
- (resource.RLIMIT_AS, memory_mb << 20),
211
- (resource.RLIMIT_FSIZE, file_mb << 20),
212
- (resource.RLIMIT_NOFILE, nofile),
213
- (resource.RLIMIT_CORE, 0),
214
- ):
215
- _, hard = resource.getrlimit(name)
216
- value = wanted if hard == resource.RLIM_INFINITY else min(wanted, hard)
217
- resource.setrlimit(name, (value, value))
218
- applied[_limit_name(name)] = value
219
- return applied
220
-
221
-
222
- def _limit_name(limit: int) -> str:
223
- return {
224
- resource.RLIMIT_AS: "memory_bytes",
225
- resource.RLIMIT_FSIZE: "file_bytes",
226
- resource.RLIMIT_NOFILE: "open_files",
227
- resource.RLIMIT_CORE: "core_bytes",
228
- }[limit]
341
+ """Cap this process before it imports world code. Returns each limit as
342
+ set, by label, so the worker can report them."""
343
+ limits = (
344
+ AddressSpaceLimit(resource.RLIMIT_AS, memory_mb << 20, "memory_bytes"),
345
+ ProcessLimit(resource.RLIMIT_FSIZE, file_mb << 20, "file_bytes"),
346
+ ProcessLimit(resource.RLIMIT_NOFILE, nofile, "open_files"),
347
+ ProcessLimit(resource.RLIMIT_CORE, 0, "core_bytes"),
348
+ )
349
+ return {limit.label: limit.apply() for limit in limits}
229
350
 
230
351
 
231
352
  def worker_environment(
@@ -246,7 +367,6 @@ def worker_environment(
246
367
  WORLDS_WORKER_SECRET=secret,
247
368
  WORLDS_WORKER_MEMORY_MB=str(memory_mb),
248
369
  WORLDS_INGRESS_DOMAIN=worlds_domain,
249
- WORLDS_SUPERVISOR_PID=str(os.getpid()),
250
370
  )
251
371
  return env
252
372
 
@@ -295,20 +415,13 @@ class ConfinedVerifierSandbox(VerifierSandbox):
295
415
  return verifier_environment(home=home)
296
416
 
297
417
 
298
- # --------------------------------------------------------------------------- #
299
- # The supervisor's side: start, reach and stop workers.
300
- # --------------------------------------------------------------------------- #
301
-
302
-
303
418
  @dataclass
304
419
  class Worker:
305
420
  version_id: str
306
421
  process: subprocess.Popen
307
422
  socket_path: Path
308
423
  secret: str
309
- #: Sessions the supervisor has placed on this worker.
310
424
  sessions: set[str] = field(default_factory=set)
311
- #: When the worker last had a session, for idle reaping.
312
425
  idle_since: float = field(default_factory=time.time)
313
426
 
314
427
  @property
@@ -327,25 +440,78 @@ class Worker:
327
440
  *,
328
441
  control: bool = False,
329
442
  timeout: float = 60.0,
443
+ limit: int | None = None,
330
444
  ) -> tuple[int, list[tuple[str, str]], bytes]:
331
445
  """One request over the socket. ``control`` adds the worker's own
332
- secret, so the worker's control plane answers. A worker that cannot be
333
- reached raises :class:`WorkerGone`."""
446
+ secret, so the worker's control plane answers. A worker that takes no connection raises
447
+ :class:`WorkerUnreached`; one that drops the request or answers past ``limit`` bytes raises
448
+ :class:`WorkerGone`. The whole exchange gets ``timeout`` seconds, not each read: an answer
449
+ sent slowly is cut off when they pass (the connection is shut, not waited out) and raises
450
+ :class:`WorkerTimeout`."""
334
451
  forward = {k: v for k, v in headers.items() if k.lower() not in UNFORWARDED_HEADERS}
335
452
  if control:
336
453
  forward["X-Host-Secret"] = self.secret
337
- conn = UnixHTTPConnection(self.socket_path, timeout=timeout)
454
+ started = time.monotonic()
455
+ conn = self._connected(timeout)
456
+ cut = threading.Event()
338
457
  try:
339
- conn.request(method, path, body=body or None, headers=forward)
340
- answer = conn.getresponse()
341
- payload = answer.read()
458
+ with _deadline(conn.sock, timeout - (time.monotonic() - started)) as cut:
459
+ conn.request(method, path, body=body or None, headers=forward)
460
+ answer = conn.getresponse()
461
+ payload = answer.read() if limit is None else answer.read(limit + 1)
462
+ if cut.is_set():
463
+ raise WorkerTimeout(
464
+ f"worker for version {self.version_id} (pid {self.pid}) did not answer in {timeout:g} s"
465
+ )
466
+ if limit is not None and len(payload) > limit:
467
+ raise WorkerGone(f"worker for version {self.version_id} (pid {self.pid}) answered past {limit} bytes")
342
468
  return answer.status, answer.getheaders(), payload
343
469
  except (OSError, http.client.HTTPException) as error:
344
- raise WorkerGone(
345
- f"worker for version {self.version_id} (pid {self.pid}) did not answer: {error}"
470
+ gone = WorkerTimeout if cut.is_set() or isinstance(error, TimeoutError) else WorkerGone
471
+ raise gone(f"worker for version {self.version_id} (pid {self.pid}) did not answer: {error}") from error
472
+ finally:
473
+ conn.close()
474
+
475
+ def _connected(self, timeout: float) -> UnixHTTPConnection:
476
+ conn = UnixHTTPConnection(self.socket_path, timeout=timeout)
477
+ try:
478
+ conn.connect()
479
+ except OSError as error:
480
+ conn.close()
481
+ raise WorkerUnreached(
482
+ f"worker for version {self.version_id} (pid {self.pid}) took no connection: {error}"
346
483
  ) from error
484
+ return conn
485
+
486
+ def probe(
487
+ self, path: str, *, deadline_s: float = HEALTH_TIMEOUT_S, limit: int = PROBE_LIMIT_BYTES
488
+ ) -> tuple[int, bytes]:
489
+ """One control GET whose whole answer must arrive within ``deadline_s`` and ``limit`` bytes.
490
+ The worker runs tenant code: an answer sent slowly or without end is cut off when the
491
+ deadline passes (the connection is shut, not waited out), and raises :class:`WorkerGone`."""
492
+ unanswered = f"worker for version {self.version_id} (pid {self.pid}) did not answer {path}"
493
+ conn = UnixHTTPConnection(self.socket_path, timeout=deadline_s)
494
+ try:
495
+ conn.connect()
496
+ with _deadline(conn.sock, deadline_s) as cut:
497
+ conn.request("GET", path, headers={"X-Host-Secret": self.secret})
498
+ answer = conn.getresponse()
499
+ payload = answer.read(limit + 1)
500
+ except (OSError, http.client.HTTPException) as error:
501
+ raise WorkerGone(unanswered) from error
347
502
  finally:
348
503
  conn.close()
504
+ if cut.is_set() or len(payload) > limit:
505
+ raise WorkerGone(f"{unanswered} within {deadline_s:g} s and {limit} bytes")
506
+ return answer.status, payload
507
+
508
+ def answers_health(self, timeout_s: float = HEALTH_TIMEOUT_S) -> bool:
509
+ """Whether the worker is alive and answers its health check, whole, in time."""
510
+ try:
511
+ status, _ = self.probe("/healthz", deadline_s=timeout_s)
512
+ except WorkerGone:
513
+ return False
514
+ return status == 200
349
515
 
350
516
  def stop(self, grace_s: float = 5.0) -> None:
351
517
  if self.alive():
@@ -358,6 +524,69 @@ class Worker:
358
524
  self.socket_path.unlink(missing_ok=True)
359
525
 
360
526
 
527
+ class Finishing:
528
+ """What leaves a session on a version worker, finished off the lease sweeper's thread: a few daemon
529
+ threads, one job at a time for a version in the order they came (so a worker that never answers
530
+ holds one thread, not the pool). A session with no worker is finished at once, where it is."""
531
+
532
+ def __init__(self, threads: int = FINISH_THREADS) -> None:
533
+ self.threads = threads
534
+ self._changed = threading.Condition()
535
+ self._waiting: dict[str, collections.deque[Callable[[], None]]] = {}
536
+ self._serving: set[str] = set()
537
+ self._started = 0
538
+
539
+ def leave(self, worker: Worker | None, job: Callable[[], None]) -> None:
540
+ if worker is None:
541
+ job()
542
+ return
543
+ with self._changed:
544
+ self._waiting.setdefault(worker.version_id, collections.deque()).append(job)
545
+ self._staff()
546
+ self._changed.notify()
547
+
548
+ def settle(self, deadline_s: float = FINISH_SETTLE_S) -> bool:
549
+ """Wait up to ``deadline_s`` for every job handed off to be done; whether they are."""
550
+ until = time.monotonic() + deadline_s
551
+ with self._changed:
552
+ while self._waiting:
553
+ remaining = until - time.monotonic()
554
+ if remaining <= 0:
555
+ return False
556
+ self._changed.wait(remaining)
557
+ return True
558
+
559
+ def _staff(self) -> None:
560
+ if self._started < self.threads:
561
+ self._started += 1
562
+ threading.Thread(target=self._run, name=f"world-host-finish-{self._started}", daemon=True).start()
563
+
564
+ def _run(self) -> None:
565
+ while True:
566
+ with self._changed:
567
+ self._changed.wait_for(lambda: self._next() is not None)
568
+ version = self._next()
569
+ assert version is not None
570
+ job = self._waiting[version].popleft()
571
+ self._serving.add(version)
572
+ self._work(version, job)
573
+
574
+ def _next(self) -> str | None:
575
+ return next((version for version, jobs in self._waiting.items() if jobs and version not in self._serving), None)
576
+
577
+ def _work(self, version: str, job: Callable[[], None]) -> None:
578
+ try:
579
+ job()
580
+ except Exception as error: # noqa: BLE001 - one session's finish never stops the pool
581
+ log_line(f"world host: a session did not finish: {type(error).__name__}")
582
+ finally:
583
+ with self._changed:
584
+ self._serving.discard(version)
585
+ if not self._waiting[version]:
586
+ del self._waiting[version]
587
+ self._changed.notify_all()
588
+
589
+
361
590
  class WorkerPool:
362
591
  """Every version worker on this replica, started on the first session of a
363
592
  version and stopped once it has had none for a while."""
@@ -387,16 +616,20 @@ class WorkerPool:
387
616
  "(WORLDS_WORKER_BUDGET_MB, WORLDS_WORKER_FOOTPRINT_MB)"
388
617
  )
389
618
  self.root = Path(cache_root)
390
- self.workers_root = self.root / "workers"
619
+ self.workers_root = self.root / WORKERS_DIR
391
620
  # Workers live and die with the supervisor; whatever a previous one
392
621
  # left (sockets, session files) is unreachable.
393
622
  shutil.rmtree(self.workers_root, ignore_errors=True)
394
623
  self.workers_root.mkdir(parents=True, exist_ok=True, mode=0o700)
624
+ # AF_UNIX caps a socket path at 103 bytes on macOS (107 on Linux): the
625
+ # sockets live in a short private (0700) directory of their own, so no
626
+ # cache root or version id is too long to serve.
627
+ self._sockets = tempfile.TemporaryDirectory(prefix="gw-")
628
+ self.sockets = Path(self._sockets.name)
395
629
  self.worlds_domain = worlds_domain
396
630
  self.memory_mb = memory_mb
397
631
  self.budget_mb = budget_mb
398
632
  self.footprint_mb = footprint_mb
399
- #: Workers live at once: the budget's worth of footprints.
400
633
  self.max_workers = budget_mb // footprint_mb
401
634
  self.idle_s = idle_s
402
635
  self.python = python
@@ -406,24 +639,54 @@ class WorkerPool:
406
639
  def get(self, version_id: str) -> Worker:
407
640
  """The live worker for a version, started if there is none. Starting one
408
641
  past the budget first stops workers holding no sessions (whatever their
409
- idle time); when every one is in use, raises :class:`WorkerCapacity`."""
642
+ idle time); when every one is in use, raises :class:`WorkerCapacity`. A
643
+ dead worker that still holds sessions is never replaced here: its
644
+ sessions' files are on disk, and the supervisor recovers them through
645
+ :meth:`replace` (raises :class:`WorkerGone`)."""
410
646
  with self._lock:
411
647
  worker = self._workers.get(version_id)
412
648
  if worker is not None and worker.alive():
413
649
  return worker
650
+ if worker is not None and worker.sessions:
651
+ raise WorkerGone(f"worker for version {version_id} (pid {worker.pid}) exited holding sessions")
414
652
  if worker is not None:
415
653
  worker.stop()
416
654
  del self._workers[version_id]
417
- self._make_room()
418
- worker = self._spawn(version_id)
419
- self._workers[version_id] = worker
420
- return worker
655
+ return self._start(version_id)
656
+
657
+ def replace(self, dead: Worker) -> Worker:
658
+ """A fresh worker in ``dead``'s place, its sessions left for the caller to
659
+ reopen; ``dead`` is stopped if it still runs. Refused (:class:`WorkerGone`)
660
+ when the pool no longer holds ``dead``."""
661
+ with self._lock:
662
+ if self._workers.get(dead.version_id) is not dead:
663
+ raise WorkerGone(f"worker for version {dead.version_id} (pid {dead.pid}) was replaced already")
664
+ del self._workers[dead.version_id]
665
+ dead.stop()
666
+ return self._start(dead.version_id)
667
+
668
+ def _start(self, version_id: str) -> Worker:
669
+ self._make_room()
670
+ worker = self._spawn(version_id)
671
+ self._workers[version_id] = worker
672
+ return worker
673
+
674
+ def dead(self) -> list[Worker]:
675
+ """The workers that exited still holding sessions: theirs to recover."""
676
+ with self._lock:
677
+ return [w for w in self._workers.values() if w.sessions and not w.alive()]
678
+
679
+ def live(self) -> list[Worker]:
680
+ """The workers still running, holding sessions or not."""
681
+ with self._lock:
682
+ return [w for w in self._workers.values() if w.alive()]
421
683
 
422
684
  def _make_room(self) -> None:
423
685
  """Under ``_lock``: leave one worker's worth of the budget free, by
424
- forgetting exited workers first, then stopping session-less ones
425
- longest idle first, and refusing when every live worker is in use."""
426
- exited = [v for v, w in self._workers.items() if not w.alive()]
686
+ forgetting exited workers that hold no sessions first, then stopping
687
+ session-less ones longest idle first, and refusing when every live
688
+ worker is in use."""
689
+ exited = [v for v, w in self._workers.items() if not w.alive() and not w.sessions]
427
690
  for version_id in exited:
428
691
  self._workers.pop(version_id).stop()
429
692
  spare = sorted(
@@ -442,15 +705,6 @@ class WorkerPool:
442
705
  with self._lock:
443
706
  return self._workers.get(version_id)
444
707
 
445
- def drop(self, version_id: str) -> Worker | None:
446
- """Stop and forget a version's worker; returns it (with the sessions
447
- it held) so the caller can forget those too."""
448
- with self._lock:
449
- worker = self._workers.pop(version_id, None)
450
- if worker is not None:
451
- worker.stop()
452
- return worker
453
-
454
708
  def reap_idle(self, now: float | None = None) -> list[str]:
455
709
  """Stop workers that have had no session for ``idle_s``; returns their versions."""
456
710
  now = time.time() if now is None else now
@@ -475,7 +729,7 @@ class WorkerPool:
475
729
  def _spawn(self, version_id: str) -> Worker:
476
730
  home = self.workers_root / version_id
477
731
  home.mkdir(parents=True, exist_ok=True, mode=0o700)
478
- socket_path = home / "worker.sock"
732
+ socket_path = self.sockets / f"{hashlib.sha256(version_id.encode()).hexdigest()[:16]}.sock"
479
733
  secret = secrets.token_urlsafe(24)
480
734
  # The supervisor binds and listens; the worker inherits the descriptor
481
735
  # and serves on it, so the socket is the supervisor's from the start.
@@ -519,13 +773,8 @@ class WorkerPool:
519
773
  raise WorkerGone(
520
774
  f"worker for version {worker.version_id} exited with {worker.process.returncode} on start"
521
775
  )
522
- if worker.socket_path.exists():
523
- try:
524
- status, _, _ = worker.request("GET", "/healthz", {}, timeout=2.0)
525
- except WorkerGone:
526
- status = 0
527
- if status == 200:
528
- return
776
+ if worker.socket_path.exists() and worker.answers_health():
777
+ return
529
778
  time.sleep(0.05)
530
779
  worker.stop()
531
780
  raise WorkerGone(f"worker for version {worker.version_id} did not come up in {WORKER_START_TIMEOUT_S:.0f}s")