@withgateway/sdk 0.1.19 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (130) hide show
  1. package/README.md +141 -2
  2. package/dist/cli.js +8677 -1320
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +8683 -1327
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/index.d.mts +1 -1
  7. package/dist/index.d.ts +1 -1
  8. package/dist/index.js +1005 -55
  9. package/dist/index.js.map +1 -1
  10. package/dist/index.mjs +969 -47
  11. package/dist/index.mjs.map +1 -1
  12. package/dist/tracing/index.js +9 -2
  13. package/dist/tracing/index.js.map +1 -1
  14. package/dist/tracing/index.mjs +9 -2
  15. package/dist/tracing/index.mjs.map +1 -1
  16. package/dist/worlds/index.d.mts +738 -28
  17. package/dist/worlds/index.d.ts +738 -28
  18. package/dist/worlds/index.js +966 -51
  19. package/dist/worlds/index.js.map +1 -1
  20. package/dist/worlds/index.mjs +960 -45
  21. package/dist/worlds/index.mjs.map +1 -1
  22. package/package.json +9 -2
  23. package/runtime/gatewaysdk/__init__.py +0 -0
  24. package/runtime/gatewaysdk/environment/__init__.py +108 -0
  25. package/runtime/gatewaysdk/environment/_bundle.py +281 -0
  26. package/runtime/gatewaysdk/environment/_harbor.py +276 -0
  27. package/runtime/gatewaysdk/environment/_materialize.py +97 -0
  28. package/runtime/gatewaysdk/environment/_tar.py +33 -0
  29. package/runtime/gatewaysdk/environment/_world.py +346 -0
  30. package/runtime/gatewaysdk/environment/core.py +527 -0
  31. package/runtime/gatewaysdk/environment/errors.py +58 -0
  32. package/runtime/gatewaysdk/environment/runtime.py +150 -0
  33. package/runtime/gatewaysdk/environment/schema/__init__.py +35 -0
  34. package/runtime/gatewaysdk/environment/schema/__main__.py +225 -0
  35. package/runtime/gatewaysdk/environment/schema/_slack_fidelity.py +116 -0
  36. package/runtime/gatewaysdk/environment/schema/_slack_scenario.py +183 -0
  37. package/runtime/gatewaysdk/environment/schema/api.py +1628 -0
  38. package/runtime/gatewaysdk/environment/schema/batch.py +755 -0
  39. package/runtime/gatewaysdk/environment/schema/compiler.py +615 -0
  40. package/runtime/gatewaysdk/environment/schema/conform.py +428 -0
  41. package/runtime/gatewaysdk/environment/schema/connector.py +777 -0
  42. package/runtime/gatewaysdk/environment/schema/connectors/apple-business-manager/handlers.py +17 -0
  43. package/runtime/gatewaysdk/environment/schema/connectors/apple-business-manager/parity.json +28 -0
  44. package/runtime/gatewaysdk/environment/schema/connectors/apple-business-manager/provenance.json +233 -0
  45. package/runtime/gatewaysdk/environment/schema/connectors/apple-business-manager/scope.toml +112 -0
  46. package/runtime/gatewaysdk/environment/schema/connectors/apple-business-manager/world.json +1380 -0
  47. package/runtime/gatewaysdk/environment/schema/connectors/base.json +28 -0
  48. package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +18 -0
  49. package/runtime/gatewaysdk/environment/schema/connectors/custom/world.json +123 -0
  50. package/runtime/gatewaysdk/environment/schema/connectors/github/handlers.py +81 -0
  51. package/runtime/gatewaysdk/environment/schema/connectors/github/parity.json +37 -0
  52. package/runtime/gatewaysdk/environment/schema/connectors/github/provenance.json +134 -0
  53. package/runtime/gatewaysdk/environment/schema/connectors/github/scope.toml +158 -0
  54. package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +5869 -0
  55. package/runtime/gatewaysdk/environment/schema/connectors/google-calendar/handlers.py +23 -0
  56. package/runtime/gatewaysdk/environment/schema/connectors/google-calendar/world.json +209 -0
  57. package/runtime/gatewaysdk/environment/schema/connectors/google-drive/handlers.py +18 -0
  58. package/runtime/gatewaysdk/environment/schema/connectors/google-drive/world.json +286 -0
  59. package/runtime/gatewaysdk/environment/schema/connectors/jamf/handlers.py +34 -0
  60. package/runtime/gatewaysdk/environment/schema/connectors/jamf/parity.json +49 -0
  61. package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +181 -0
  62. package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +128 -0
  63. package/runtime/gatewaysdk/environment/schema/connectors/jamf/world.json +2218 -0
  64. package/runtime/gatewaysdk/environment/schema/connectors/jira/handlers.py +28 -0
  65. package/runtime/gatewaysdk/environment/schema/connectors/jira/world.json +389 -0
  66. package/runtime/gatewaysdk/environment/schema/connectors/kandji/handlers.py +38 -0
  67. package/runtime/gatewaysdk/environment/schema/connectors/kandji/parity.json +29 -0
  68. package/runtime/gatewaysdk/environment/schema/connectors/kandji/provenance.json +152 -0
  69. package/runtime/gatewaysdk/environment/schema/connectors/kandji/scope.toml +120 -0
  70. package/runtime/gatewaysdk/environment/schema/connectors/kandji/world.json +765 -0
  71. package/runtime/gatewaysdk/environment/schema/connectors/linear/handlers.py +20 -0
  72. package/runtime/gatewaysdk/environment/schema/connectors/linear/world.json +363 -0
  73. package/runtime/gatewaysdk/environment/schema/connectors/microsoft-teams/fidelity.json +45 -0
  74. package/runtime/gatewaysdk/environment/schema/connectors/microsoft-teams/handlers.py +1 -0
  75. package/runtime/gatewaysdk/environment/schema/connectors/microsoft-teams/provenance.json +851 -0
  76. package/runtime/gatewaysdk/environment/schema/connectors/microsoft-teams/scope.md +231 -0
  77. package/runtime/gatewaysdk/environment/schema/connectors/microsoft-teams/teams-types.json +1710 -0
  78. package/runtime/gatewaysdk/environment/schema/connectors/microsoft-teams/world.json +858 -0
  79. package/runtime/gatewaysdk/environment/schema/connectors/netsuite/handlers.py +32 -0
  80. package/runtime/gatewaysdk/environment/schema/connectors/netsuite/parity.json +50 -0
  81. package/runtime/gatewaysdk/environment/schema/connectors/netsuite/provenance.json +137 -0
  82. package/runtime/gatewaysdk/environment/schema/connectors/netsuite/scope.toml +78 -0
  83. package/runtime/gatewaysdk/environment/schema/connectors/netsuite/world.json +16243 -0
  84. package/runtime/gatewaysdk/environment/schema/connectors/salesforce/handlers.py +806 -0
  85. package/runtime/gatewaysdk/environment/schema/connectors/salesforce/parity.json +46 -0
  86. package/runtime/gatewaysdk/environment/schema/connectors/salesforce/provenance.json +143 -0
  87. package/runtime/gatewaysdk/environment/schema/connectors/salesforce/scope.toml +174 -0
  88. package/runtime/gatewaysdk/environment/schema/connectors/salesforce/world.json +4695 -0
  89. package/runtime/gatewaysdk/environment/schema/connectors/slack/MODEL.md +236 -0
  90. package/runtime/gatewaysdk/environment/schema/connectors/slack/capabilities.json +1039 -0
  91. package/runtime/gatewaysdk/environment/schema/connectors/slack/handlers.py +762 -0
  92. package/runtime/gatewaysdk/environment/schema/connectors/slack/parity.json +65 -0
  93. package/runtime/gatewaysdk/environment/schema/connectors/slack/provenance.json +294 -0
  94. package/runtime/gatewaysdk/environment/schema/connectors/slack/scope.toml +44 -0
  95. package/runtime/gatewaysdk/environment/schema/connectors/slack/world.json +4993 -0
  96. package/runtime/gatewaysdk/environment/schema/connectors/workday/handlers.py +75 -0
  97. package/runtime/gatewaysdk/environment/schema/connectors/workday/parity.json +36 -0
  98. package/runtime/gatewaysdk/environment/schema/connectors/workday/provenance.json +179 -0
  99. package/runtime/gatewaysdk/environment/schema/connectors/workday/scope.toml +180 -0
  100. package/runtime/gatewaysdk/environment/schema/connectors/workday/world.json +806 -0
  101. package/runtime/gatewaysdk/environment/schema/data.py +593 -0
  102. package/runtime/gatewaysdk/environment/schema/extract.py +263 -0
  103. package/runtime/gatewaysdk/environment/schema/host.py +2407 -0
  104. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +275 -0
  105. package/runtime/gatewaysdk/environment/schema/platform.py +1272 -0
  106. package/runtime/gatewaysdk/environment/schema/scaffold.py +250 -0
  107. package/runtime/gatewaysdk/environment/schema/skill.py +118 -0
  108. package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +202 -0
  109. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +331 -0
  110. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONVENTIONS.md +100 -0
  111. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +313 -0
  112. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +223 -0
  113. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +293 -0
  114. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +390 -0
  115. package/runtime/gatewaysdk/environment/schema/slack.py +757 -0
  116. package/runtime/gatewaysdk/environment/schema/snapshot.py +479 -0
  117. package/runtime/gatewaysdk/environment/schema/store.py +904 -0
  118. package/runtime/gatewaysdk/environment/schema/tasks.py +525 -0
  119. package/runtime/gatewaysdk/environment/schema/tools_world.py +322 -0
  120. package/runtime/gatewaysdk/environment/schema/validation.py +577 -0
  121. package/runtime/gatewaysdk/environment/schema/worker.py +78 -0
  122. package/runtime/gatewaysdk/environment/schema/workers.py +500 -0
  123. package/runtime/gatewaysdk/environment/schema/world_tests.py +336 -0
  124. package/skills/agents/connector-template-author.md +202 -0
  125. package/skills/connector-schema-authoring/CONNECTORS.md +331 -0
  126. package/skills/connector-schema-authoring/CONVENTIONS.md +100 -0
  127. package/skills/connector-schema-authoring/SKILL.md +313 -0
  128. package/skills/world-data-ingestion/ROWS.md +223 -0
  129. package/skills/world-data-ingestion/SKILL.md +293 -0
  130. package/skills/worlds-getting-started/SKILL.md +390 -0
@@ -0,0 +1,500 @@
1
+ """Version workers: one subprocess per world version that runs world-authored code.
2
+
3
+ A schema world with an ``api/handlers.py`` brings its own Python. The host
4
+ process is shared by every tenant on the replica, so that code never runs in
5
+ it: the supervisor (:mod:`host`) starts one worker per version, over a unix
6
+ socket only the supervisor can reach, and forwards that version's sessions to
7
+ it. The worker is the same :class:`host.WorldHost` in a smaller room:
8
+
9
+ * its own process, so a handler that loops, leaks or crashes takes down that
10
+ version's sessions on this replica and nothing else;
11
+ * rlimits on address space, open files and file size, set before any world
12
+ code is imported;
13
+ * a scrubbed environment: the control-plane secret, cloud credentials and the
14
+ rest of the supervisor's environment are not inherited. The worker gets a
15
+ secret of its own, good for its socket alone;
16
+ * one version: a worker refuses to open any other, so two tenants never share
17
+ a process even when they share a host;
18
+ * a socket only the supervisor is answered on. Workers run as the host's user,
19
+ so file modes cannot keep one worker's handler from opening a sibling's
20
+ socket path: the worker asks the kernel who connected (``SO_PEERCRED``) and
21
+ serves the supervisor's pid alone. The listening socket is the supervisor's,
22
+ bound before the worker starts and handed down as a file descriptor, so a
23
+ worker never binds a path of its own;
24
+ * a worker nobody can read: it marks itself non-dumpable, which closes
25
+ ``/proc/<pid>/environ``, ``/proc/<pid>/mem`` and ``ptrace`` to every process
26
+ that is not root, siblings under the same uid included. Its secret and its
27
+ sessions' memory stay its own.
28
+
29
+ What this is not: a uid boundary on the filesystem. A worker can read the
30
+ version cache and the files under a sibling's home. The deploy provides that
31
+ fence (a sandboxed runtime class, a read-only root) when it is wanted; see
32
+ ``deploy/world-host/README.md``.
33
+ """
34
+
35
+ from __future__ import annotations
36
+
37
+ import ctypes
38
+ import http.client
39
+ import os
40
+ import resource
41
+ import secrets
42
+ import shutil
43
+ import socket
44
+ import struct
45
+ import subprocess
46
+ import sys
47
+ import threading
48
+ import time
49
+ from collections.abc import Mapping
50
+ from dataclasses import dataclass, field
51
+ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
52
+ from pathlib import Path
53
+
54
+ #: Address space a worker may use, in MiB, unless ``WORLDS_WORKER_MEMORY_MB`` says otherwise.
55
+ DEFAULT_WORKER_MEMORY_MB = 2048
56
+ #: Address space every worker together may use, in MiB, unless
57
+ #: ``WORLDS_WORKER_BUDGET_MB`` says otherwise. The number of workers a replica
58
+ #: runs at once is this over the per-worker cap: two of 2 GiB inside the 6 GiB
59
+ #: host container, the rest the supervisor's (its caches, the sessions it
60
+ #: serves in-process, exports in flight). A version past that is refused
61
+ #: ``at_capacity`` and placed on another replica instead of the kernel
62
+ #: choosing which process to kill.
63
+ DEFAULT_WORKER_BUDGET_MB = 4096
64
+ #: Largest file a worker may write (its session databases), in MiB.
65
+ DEFAULT_WORKER_FILE_MB = 4096
66
+ #: Open files per worker: one per live session plus the sockets serving them.
67
+ DEFAULT_WORKER_NOFILE = 4096
68
+ #: How long a worker with no sessions is kept before it is stopped.
69
+ DEFAULT_WORKER_IDLE_S = 300.0
70
+ #: How long the supervisor waits for a fresh worker to answer on its socket.
71
+ WORKER_START_TIMEOUT_S = 30.0
72
+ #: A worker's sessions live as long as the supervisor's do; the supervisor
73
+ #: holds the leases, so the worker's own are effectively unbounded.
74
+ WORKER_LEASE_S = 365.0 * 86400.0
75
+ #: Environment the worker inherits. Nothing else crosses: no host secret, no
76
+ #: cloud credentials, no platform URL.
77
+ INHERITED_ENV = ("PATH", "PYTHONPATH", "LANG", "LC_ALL", "TZ", "VIRTUAL_ENV")
78
+ #: Headers the supervisor never forwards to a worker: the ones that describe
79
+ #: the supervisor's connection, and its own control-plane secret.
80
+ UNFORWARDED_HEADERS = frozenset({"connection", "content-length", "transfer-encoding", "x-host-secret"})
81
+
82
+
83
+ class WorkerGone(Exception):
84
+ """The worker did not answer: it exited, or never came up."""
85
+
86
+
87
+ class WorkerCapacity(Exception):
88
+ """Every worker the memory budget allows is live and holds sessions, so no
89
+ worker for another version can start here."""
90
+
91
+
92
+ # --------------------------------------------------------------------------- #
93
+ # HTTP over a unix socket: the only way into a worker.
94
+ # --------------------------------------------------------------------------- #
95
+
96
+
97
+ class UnixHTTPConnection(http.client.HTTPConnection):
98
+ def __init__(self, path: str | Path, timeout: float | None = None):
99
+ super().__init__("worker", timeout=timeout)
100
+ self._unix_path = str(path)
101
+
102
+ def connect(self) -> None:
103
+ sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
104
+ sock.settimeout(self.timeout)
105
+ sock.connect(self._unix_path)
106
+ self.sock = sock
107
+
108
+
109
+ def bind_worker_socket(path: str | Path) -> socket.socket:
110
+ """The supervisor's end of a worker: a listening unix socket at ``path``,
111
+ ``0600`` inside a ``0700`` directory, ready to hand to the worker."""
112
+ path = Path(path)
113
+ path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
114
+ path.unlink(missing_ok=True)
115
+ sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
116
+ sock.bind(str(path))
117
+ os.chmod(path, 0o600)
118
+ sock.listen(128)
119
+ return sock
120
+
121
+
122
+ def peer_pid(conn: socket.socket) -> int | None:
123
+ """The pid of whoever connected, as the kernel reports it (``SO_PEERCRED``:
124
+ ``struct ucred {pid, uid, gid}``); None where the platform has no such
125
+ thing."""
126
+ option = getattr(socket, "SO_PEERCRED", None)
127
+ if option is None:
128
+ return None
129
+ raw = conn.getsockopt(socket.SOL_SOCKET, option, struct.calcsize("3i"))
130
+ pid, _uid, _gid = struct.unpack("3i", raw)
131
+ return pid
132
+
133
+
134
+ class UnixHTTPServer(ThreadingHTTPServer):
135
+ """The worker's server: a ``ThreadingHTTPServer`` over the listening unix
136
+ socket the supervisor handed it (an inherited descriptor), or one it binds
137
+ itself at a path (tests, a hand-run worker). With ``supervisor_pid`` set,
138
+ only connections from that pid are served: the kernel names the peer, so a
139
+ sibling worker that can open the path (same uid) is still refused."""
140
+
141
+ address_family = socket.AF_UNIX
142
+ allow_reuse_address = False
143
+ allow_reuse_port = False
144
+
145
+ def __init__(
146
+ self,
147
+ listening: str | Path | socket.socket,
148
+ handler: type[BaseHTTPRequestHandler],
149
+ *,
150
+ supervisor_pid: int | None = None,
151
+ ):
152
+ self.supervisor_pid = supervisor_pid
153
+ self.refused = 0
154
+ # The base class wants a (host, port) pair; it is only what
155
+ # ``server_address`` reports. The socket is bound here, not by it.
156
+ super().__init__(("worker", 0), handler, bind_and_activate=False)
157
+ self.socket.close()
158
+ self.socket = listening if isinstance(listening, socket.socket) else bind_worker_socket(listening)
159
+ self.server_name = "worker"
160
+ self.server_port = 0
161
+
162
+ def verify_request(self, request, client_address) -> bool:
163
+ if self.supervisor_pid is None:
164
+ return True
165
+ if peer_pid(request) == self.supervisor_pid:
166
+ return True
167
+ self.refused += 1
168
+ return False
169
+
170
+
171
+ _PR_SET_DUMPABLE = 4
172
+
173
+
174
+ def seal_process() -> bool:
175
+ """Make this process unreadable to its peers: not dumpable, so its
176
+ ``/proc/<pid>/environ`` and memory are root's to read and ``ptrace`` from
177
+ another process under the same uid is refused; and a ``077`` umask, so what
178
+ it writes is its own. True when the kernel took the setting."""
179
+ os.umask(0o077)
180
+ if not sys.platform.startswith("linux"):
181
+ return False
182
+ try:
183
+ libc = ctypes.CDLL(None, use_errno=True)
184
+ return libc.prctl(_PR_SET_DUMPABLE, 0, 0, 0, 0) == 0
185
+ except (OSError, AttributeError):
186
+ return False
187
+
188
+
189
+ def apply_limits(
190
+ *,
191
+ memory_mb: int = DEFAULT_WORKER_MEMORY_MB,
192
+ file_mb: int = DEFAULT_WORKER_FILE_MB,
193
+ nofile: int = DEFAULT_WORKER_NOFILE,
194
+ ) -> dict[str, int]:
195
+ """Cap this process before it imports world code. A limit the kernel will
196
+ not grant (a hard limit lower than asked) is lowered to what it grants
197
+ rather than refused: the worker still runs, just tighter. Returns the
198
+ limits as set, so the worker can report them."""
199
+ applied: dict[str, int] = {}
200
+ for name, wanted in (
201
+ (resource.RLIMIT_AS, memory_mb << 20),
202
+ (resource.RLIMIT_FSIZE, file_mb << 20),
203
+ (resource.RLIMIT_NOFILE, nofile),
204
+ (resource.RLIMIT_CORE, 0),
205
+ ):
206
+ _, hard = resource.getrlimit(name)
207
+ value = wanted if hard == resource.RLIM_INFINITY else min(wanted, hard)
208
+ resource.setrlimit(name, (value, value))
209
+ applied[_limit_name(name)] = value
210
+ return applied
211
+
212
+
213
+ def _limit_name(limit: int) -> str:
214
+ return {
215
+ resource.RLIMIT_AS: "memory_bytes",
216
+ resource.RLIMIT_FSIZE: "file_bytes",
217
+ resource.RLIMIT_NOFILE: "open_files",
218
+ resource.RLIMIT_CORE: "core_bytes",
219
+ }[limit]
220
+
221
+
222
+ def worker_environment(
223
+ *,
224
+ secret: str,
225
+ home: Path,
226
+ worlds_domain: str,
227
+ memory_mb: int,
228
+ source: Mapping[str, str] | None = None,
229
+ ) -> dict[str, str]:
230
+ """The environment a worker is started with: the interpreter's own needs,
231
+ a home of its own, and the worker's settings. Built from an allow-list,
232
+ so whatever else the supervisor was given stays with the supervisor."""
233
+ inherited: Mapping[str, str] = os.environ if source is None else source
234
+ env = {name: inherited[name] for name in INHERITED_ENV if name in inherited}
235
+ env.update(
236
+ HOME=str(home),
237
+ WORLDS_WORKER_SECRET=secret,
238
+ WORLDS_WORKER_MEMORY_MB=str(memory_mb),
239
+ WORLDS_INGRESS_DOMAIN=worlds_domain,
240
+ WORLDS_SUPERVISOR_PID=str(os.getpid()),
241
+ )
242
+ return env
243
+
244
+
245
+ def verifier_environment(*, home: Path, source: Mapping[str, str] | None = None) -> dict[str, str]:
246
+ """The environment a scenario's ``verify.py`` runs with: the interpreter's
247
+ needs and a home under the session, nothing of the process that started
248
+ it. A verifier is world-authored code too; it gets no host or worker
249
+ secret, no cloud credentials, no platform URL."""
250
+ inherited: Mapping[str, str] = os.environ if source is None else source
251
+ env = {name: inherited[name] for name in INHERITED_ENV if name in inherited}
252
+ env["HOME"] = str(home)
253
+ return env
254
+
255
+
256
+ def confine_child() -> None:
257
+ """The worker's rlimits and seal, applied by a child that is about to run
258
+ world code (a verifier), before that code is loaded."""
259
+ apply_limits()
260
+ seal_process()
261
+
262
+
263
+ #: ``python -c`` prelude that confines the interpreter, then runs the script
264
+ #: named next on the command line as ``__main__``. Done in the child, not via
265
+ #: ``preexec_fn``, which is not safe under the supervisor's request threads.
266
+ _CONFINED_PRELUDE = (
267
+ "from gatewaysdk.environment.schema.workers import confine_child; confine_child(); "
268
+ "import runpy, sys; sys.argv = sys.argv[1:]; runpy.run_path(sys.argv[0], run_name='__main__')"
269
+ )
270
+
271
+
272
+ def confined_command(script: Path, python: str = sys.executable) -> list[str]:
273
+ """The argv that runs ``script`` as a confined child (see :func:`confine_child`)."""
274
+ return [python, "-c", _CONFINED_PRELUDE, str(script)]
275
+
276
+
277
+ # --------------------------------------------------------------------------- #
278
+ # The supervisor's side: start, reach and stop workers.
279
+ # --------------------------------------------------------------------------- #
280
+
281
+
282
+ @dataclass
283
+ class Worker:
284
+ version_id: str
285
+ process: subprocess.Popen
286
+ socket_path: Path
287
+ secret: str
288
+ #: Sessions the supervisor has placed on this worker.
289
+ sessions: set[str] = field(default_factory=set)
290
+ #: When the worker last had a session, for idle reaping.
291
+ idle_since: float = field(default_factory=time.time)
292
+
293
+ @property
294
+ def pid(self) -> int:
295
+ return self.process.pid
296
+
297
+ def alive(self) -> bool:
298
+ return self.process.poll() is None
299
+
300
+ def request(
301
+ self,
302
+ method: str,
303
+ path: str,
304
+ headers: dict[str, str],
305
+ body: bytes = b"",
306
+ *,
307
+ control: bool = False,
308
+ timeout: float = 60.0,
309
+ ) -> tuple[int, list[tuple[str, str]], bytes]:
310
+ """One request over the socket. ``control`` adds the worker's own
311
+ secret, so the worker's control plane answers. A worker that cannot be
312
+ reached raises :class:`WorkerGone`."""
313
+ forward = {k: v for k, v in headers.items() if k.lower() not in UNFORWARDED_HEADERS}
314
+ if control:
315
+ forward["X-Host-Secret"] = self.secret
316
+ conn = UnixHTTPConnection(self.socket_path, timeout=timeout)
317
+ try:
318
+ conn.request(method, path, body=body or None, headers=forward)
319
+ answer = conn.getresponse()
320
+ payload = answer.read()
321
+ return answer.status, answer.getheaders(), payload
322
+ except (OSError, http.client.HTTPException) as error:
323
+ raise WorkerGone(
324
+ f"worker for version {self.version_id} (pid {self.pid}) did not answer: {error}"
325
+ ) from error
326
+ finally:
327
+ conn.close()
328
+
329
+ def stop(self, grace_s: float = 5.0) -> None:
330
+ if self.alive():
331
+ self.process.terminate()
332
+ try:
333
+ self.process.wait(timeout=grace_s)
334
+ except subprocess.TimeoutExpired:
335
+ self.process.kill()
336
+ self.process.wait(timeout=grace_s)
337
+ self.socket_path.unlink(missing_ok=True)
338
+
339
+
340
+ class WorkerPool:
341
+ """Every version worker on this replica, started on the first session of a
342
+ version and stopped once it has had none for a while."""
343
+
344
+ def __init__(
345
+ self,
346
+ cache_root: Path,
347
+ *,
348
+ worlds_domain: str,
349
+ memory_mb: int = DEFAULT_WORKER_MEMORY_MB,
350
+ budget_mb: int = DEFAULT_WORKER_BUDGET_MB,
351
+ idle_s: float = DEFAULT_WORKER_IDLE_S,
352
+ python: str = sys.executable,
353
+ ):
354
+ if budget_mb < memory_mb:
355
+ raise ValueError(
356
+ f"the worker budget ({budget_mb} MiB) holds no worker of {memory_mb} MiB "
357
+ "(WORLDS_WORKER_BUDGET_MB, WORLDS_WORKER_MEMORY_MB)"
358
+ )
359
+ self.root = Path(cache_root)
360
+ self.workers_root = self.root / "workers"
361
+ # Workers live and die with the supervisor; whatever a previous one
362
+ # left (sockets, session files) is unreachable.
363
+ shutil.rmtree(self.workers_root, ignore_errors=True)
364
+ self.workers_root.mkdir(parents=True, exist_ok=True, mode=0o700)
365
+ self.worlds_domain = worlds_domain
366
+ self.memory_mb = memory_mb
367
+ self.budget_mb = budget_mb
368
+ #: Workers live at once: the budget's worth of per-worker address space.
369
+ self.max_workers = budget_mb // memory_mb
370
+ self.idle_s = idle_s
371
+ self.python = python
372
+ self._lock = threading.Lock()
373
+ self._workers: dict[str, Worker] = {}
374
+
375
+ def get(self, version_id: str) -> Worker:
376
+ """The live worker for a version, started if there is none. Starting one
377
+ past the budget first stops workers holding no sessions (whatever their
378
+ idle time); when every one is in use, raises :class:`WorkerCapacity`."""
379
+ with self._lock:
380
+ worker = self._workers.get(version_id)
381
+ if worker is not None and worker.alive():
382
+ return worker
383
+ if worker is not None:
384
+ worker.stop()
385
+ del self._workers[version_id]
386
+ self._make_room()
387
+ worker = self._spawn(version_id)
388
+ self._workers[version_id] = worker
389
+ return worker
390
+
391
+ def _make_room(self) -> None:
392
+ """Under ``_lock``: leave one worker's worth of the budget free, by
393
+ forgetting exited workers first, then stopping session-less ones
394
+ longest idle first, and refusing when every live worker is in use."""
395
+ exited = [v for v, w in self._workers.items() if not w.alive()]
396
+ for version_id in exited:
397
+ self._workers.pop(version_id).stop()
398
+ spare = sorted(
399
+ (w for w in self._workers.values() if not w.sessions),
400
+ key=lambda w: w.idle_since,
401
+ )
402
+ while len(self._workers) >= self.max_workers and spare:
403
+ self._workers.pop(spare.pop(0).version_id).stop()
404
+ if len(self._workers) >= self.max_workers:
405
+ raise WorkerCapacity(
406
+ f"this host's {self.max_workers} version workers ({self.budget_mb} MiB of "
407
+ f"{self.memory_mb} MiB each) all hold sessions; place elsewhere"
408
+ )
409
+
410
+ def peek(self, version_id: str) -> Worker | None:
411
+ with self._lock:
412
+ return self._workers.get(version_id)
413
+
414
+ def drop(self, version_id: str) -> Worker | None:
415
+ """Stop and forget a version's worker; returns it (with the sessions
416
+ it held) so the caller can forget those too."""
417
+ with self._lock:
418
+ worker = self._workers.pop(version_id, None)
419
+ if worker is not None:
420
+ worker.stop()
421
+ return worker
422
+
423
+ def reap_idle(self, now: float | None = None) -> list[str]:
424
+ """Stop workers that have had no session for ``idle_s``; returns their versions."""
425
+ now = time.time() if now is None else now
426
+ with self._lock:
427
+ idle = [
428
+ w.version_id
429
+ for w in self._workers.values()
430
+ if not w.sessions and (now - w.idle_since >= self.idle_s or not w.alive())
431
+ ]
432
+ stopped = [self._workers.pop(v) for v in idle]
433
+ for worker in stopped:
434
+ worker.stop()
435
+ return idle
436
+
437
+ def stop_all(self) -> None:
438
+ with self._lock:
439
+ workers = list(self._workers.values())
440
+ self._workers.clear()
441
+ for worker in workers:
442
+ worker.stop()
443
+
444
+ def _spawn(self, version_id: str) -> Worker:
445
+ home = self.workers_root / version_id
446
+ home.mkdir(parents=True, exist_ok=True, mode=0o700)
447
+ socket_path = home / "worker.sock"
448
+ secret = secrets.token_urlsafe(24)
449
+ # The supervisor binds and listens; the worker inherits the descriptor
450
+ # and serves on it, so the socket is the supervisor's from the start.
451
+ listening = bind_worker_socket(socket_path)
452
+ try:
453
+ process = subprocess.Popen(
454
+ [
455
+ self.python,
456
+ "-m",
457
+ "gatewaysdk.environment.schema.worker",
458
+ "--cache-root",
459
+ str(self.root),
460
+ "--version",
461
+ version_id,
462
+ "--socket-fd",
463
+ str(listening.fileno()),
464
+ ],
465
+ env=worker_environment(
466
+ secret=secret,
467
+ home=home,
468
+ worlds_domain=self.worlds_domain,
469
+ memory_mb=self.memory_mb,
470
+ ),
471
+ cwd=str(home),
472
+ stdin=subprocess.DEVNULL,
473
+ stdout=sys.stderr,
474
+ stderr=sys.stderr,
475
+ start_new_session=True,
476
+ pass_fds=(listening.fileno(),),
477
+ )
478
+ finally:
479
+ listening.close()
480
+ worker = Worker(version_id, process, socket_path, secret)
481
+ self._await_ready(worker)
482
+ return worker
483
+
484
+ def _await_ready(self, worker: Worker) -> None:
485
+ deadline = time.time() + WORKER_START_TIMEOUT_S
486
+ while time.time() < deadline:
487
+ if not worker.alive():
488
+ raise WorkerGone(
489
+ f"worker for version {worker.version_id} exited with {worker.process.returncode} on start"
490
+ )
491
+ if worker.socket_path.exists():
492
+ try:
493
+ status, _, _ = worker.request("GET", "/healthz", {}, timeout=2.0)
494
+ except WorkerGone:
495
+ status = 0
496
+ if status == 200:
497
+ return
498
+ time.sleep(0.05)
499
+ worker.stop()
500
+ raise WorkerGone(f"worker for version {worker.version_id} did not come up in {WORKER_START_TIMEOUT_S:.0f}s")