@withgateway/sdk 0.1.20 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +141 -2
- package/dist/cli.js +8683 -1343
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +8683 -1344
- package/dist/cli.mjs.map +1 -1
- package/dist/index.d.mts +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +979 -46
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +943 -38
- package/dist/index.mjs.map +1 -1
- package/dist/tracing/index.js +9 -2
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +9 -2
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +738 -28
- package/dist/worlds/index.d.ts +738 -28
- package/dist/worlds/index.js +940 -42
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +934 -36
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +9 -2
- package/runtime/gatewaysdk/__init__.py +0 -0
- package/runtime/gatewaysdk/environment/__init__.py +108 -0
- package/runtime/gatewaysdk/environment/_bundle.py +281 -0
- package/runtime/gatewaysdk/environment/_harbor.py +276 -0
- package/runtime/gatewaysdk/environment/_materialize.py +97 -0
- package/runtime/gatewaysdk/environment/_tar.py +33 -0
- package/runtime/gatewaysdk/environment/_world.py +346 -0
- package/runtime/gatewaysdk/environment/core.py +527 -0
- package/runtime/gatewaysdk/environment/errors.py +58 -0
- package/runtime/gatewaysdk/environment/runtime.py +150 -0
- package/runtime/gatewaysdk/environment/schema/__init__.py +35 -0
- package/runtime/gatewaysdk/environment/schema/__main__.py +225 -0
- package/runtime/gatewaysdk/environment/schema/_slack_fidelity.py +116 -0
- package/runtime/gatewaysdk/environment/schema/_slack_scenario.py +183 -0
- package/runtime/gatewaysdk/environment/schema/api.py +1628 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +755 -0
- package/runtime/gatewaysdk/environment/schema/compiler.py +615 -0
- package/runtime/gatewaysdk/environment/schema/conform.py +428 -0
- package/runtime/gatewaysdk/environment/schema/connector.py +777 -0
- package/runtime/gatewaysdk/environment/schema/connectors/apple-business-manager/handlers.py +17 -0
- package/runtime/gatewaysdk/environment/schema/connectors/apple-business-manager/parity.json +28 -0
- package/runtime/gatewaysdk/environment/schema/connectors/apple-business-manager/provenance.json +233 -0
- package/runtime/gatewaysdk/environment/schema/connectors/apple-business-manager/scope.toml +112 -0
- package/runtime/gatewaysdk/environment/schema/connectors/apple-business-manager/world.json +1380 -0
- package/runtime/gatewaysdk/environment/schema/connectors/base.json +28 -0
- package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +18 -0
- package/runtime/gatewaysdk/environment/schema/connectors/custom/world.json +123 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/handlers.py +81 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/parity.json +37 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/provenance.json +134 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/scope.toml +158 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +5869 -0
- package/runtime/gatewaysdk/environment/schema/connectors/google-calendar/handlers.py +23 -0
- package/runtime/gatewaysdk/environment/schema/connectors/google-calendar/world.json +209 -0
- package/runtime/gatewaysdk/environment/schema/connectors/google-drive/handlers.py +18 -0
- package/runtime/gatewaysdk/environment/schema/connectors/google-drive/world.json +286 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/handlers.py +34 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/parity.json +49 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +181 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +128 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/world.json +2218 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jira/handlers.py +28 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jira/world.json +389 -0
- package/runtime/gatewaysdk/environment/schema/connectors/kandji/handlers.py +38 -0
- package/runtime/gatewaysdk/environment/schema/connectors/kandji/parity.json +29 -0
- package/runtime/gatewaysdk/environment/schema/connectors/kandji/provenance.json +152 -0
- package/runtime/gatewaysdk/environment/schema/connectors/kandji/scope.toml +120 -0
- package/runtime/gatewaysdk/environment/schema/connectors/kandji/world.json +765 -0
- package/runtime/gatewaysdk/environment/schema/connectors/linear/handlers.py +20 -0
- package/runtime/gatewaysdk/environment/schema/connectors/linear/world.json +363 -0
- package/runtime/gatewaysdk/environment/schema/connectors/microsoft-teams/fidelity.json +45 -0
- package/runtime/gatewaysdk/environment/schema/connectors/microsoft-teams/handlers.py +1 -0
- package/runtime/gatewaysdk/environment/schema/connectors/microsoft-teams/provenance.json +851 -0
- package/runtime/gatewaysdk/environment/schema/connectors/microsoft-teams/scope.md +231 -0
- package/runtime/gatewaysdk/environment/schema/connectors/microsoft-teams/teams-types.json +1710 -0
- package/runtime/gatewaysdk/environment/schema/connectors/microsoft-teams/world.json +858 -0
- package/runtime/gatewaysdk/environment/schema/connectors/netsuite/handlers.py +32 -0
- package/runtime/gatewaysdk/environment/schema/connectors/netsuite/parity.json +50 -0
- package/runtime/gatewaysdk/environment/schema/connectors/netsuite/provenance.json +137 -0
- package/runtime/gatewaysdk/environment/schema/connectors/netsuite/scope.toml +78 -0
- package/runtime/gatewaysdk/environment/schema/connectors/netsuite/world.json +16243 -0
- package/runtime/gatewaysdk/environment/schema/connectors/salesforce/handlers.py +806 -0
- package/runtime/gatewaysdk/environment/schema/connectors/salesforce/parity.json +46 -0
- package/runtime/gatewaysdk/environment/schema/connectors/salesforce/provenance.json +143 -0
- package/runtime/gatewaysdk/environment/schema/connectors/salesforce/scope.toml +174 -0
- package/runtime/gatewaysdk/environment/schema/connectors/salesforce/world.json +4695 -0
- package/runtime/gatewaysdk/environment/schema/connectors/slack/MODEL.md +236 -0
- package/runtime/gatewaysdk/environment/schema/connectors/slack/capabilities.json +1039 -0
- package/runtime/gatewaysdk/environment/schema/connectors/slack/handlers.py +762 -0
- package/runtime/gatewaysdk/environment/schema/connectors/slack/parity.json +65 -0
- package/runtime/gatewaysdk/environment/schema/connectors/slack/provenance.json +294 -0
- package/runtime/gatewaysdk/environment/schema/connectors/slack/scope.toml +44 -0
- package/runtime/gatewaysdk/environment/schema/connectors/slack/world.json +4993 -0
- package/runtime/gatewaysdk/environment/schema/connectors/workday/handlers.py +75 -0
- package/runtime/gatewaysdk/environment/schema/connectors/workday/parity.json +36 -0
- package/runtime/gatewaysdk/environment/schema/connectors/workday/provenance.json +179 -0
- package/runtime/gatewaysdk/environment/schema/connectors/workday/scope.toml +180 -0
- package/runtime/gatewaysdk/environment/schema/connectors/workday/world.json +806 -0
- package/runtime/gatewaysdk/environment/schema/data.py +593 -0
- package/runtime/gatewaysdk/environment/schema/extract.py +263 -0
- package/runtime/gatewaysdk/environment/schema/host.py +2407 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +275 -0
- package/runtime/gatewaysdk/environment/schema/platform.py +1272 -0
- package/runtime/gatewaysdk/environment/schema/scaffold.py +250 -0
- package/runtime/gatewaysdk/environment/schema/skill.py +118 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +202 -0
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +331 -0
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONVENTIONS.md +100 -0
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +313 -0
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +223 -0
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +293 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +390 -0
- package/runtime/gatewaysdk/environment/schema/slack.py +757 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +479 -0
- package/runtime/gatewaysdk/environment/schema/store.py +904 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +525 -0
- package/runtime/gatewaysdk/environment/schema/tools_world.py +322 -0
- package/runtime/gatewaysdk/environment/schema/validation.py +577 -0
- package/runtime/gatewaysdk/environment/schema/worker.py +78 -0
- package/runtime/gatewaysdk/environment/schema/workers.py +500 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +336 -0
- package/skills/agents/connector-template-author.md +202 -0
- package/skills/connector-schema-authoring/CONNECTORS.md +331 -0
- package/skills/connector-schema-authoring/CONVENTIONS.md +100 -0
- package/skills/connector-schema-authoring/SKILL.md +313 -0
- package/skills/world-data-ingestion/ROWS.md +223 -0
- package/skills/world-data-ingestion/SKILL.md +293 -0
- package/skills/worlds-getting-started/SKILL.md +390 -0
|
@@ -0,0 +1,500 @@
|
|
|
1
|
+
"""Version workers: one subprocess per world version that runs world-authored code.
|
|
2
|
+
|
|
3
|
+
A schema world with an ``api/handlers.py`` brings its own Python. The host
|
|
4
|
+
process is shared by every tenant on the replica, so that code never runs in
|
|
5
|
+
it: the supervisor (:mod:`host`) starts one worker per version, over a unix
|
|
6
|
+
socket only the supervisor can reach, and forwards that version's sessions to
|
|
7
|
+
it. The worker is the same :class:`host.WorldHost` in a smaller room:
|
|
8
|
+
|
|
9
|
+
* its own process, so a handler that loops, leaks or crashes takes down that
|
|
10
|
+
version's sessions on this replica and nothing else;
|
|
11
|
+
* rlimits on address space, open files and file size, set before any world
|
|
12
|
+
code is imported;
|
|
13
|
+
* a scrubbed environment: the control-plane secret, cloud credentials and the
|
|
14
|
+
rest of the supervisor's environment are not inherited. The worker gets a
|
|
15
|
+
secret of its own, good for its socket alone;
|
|
16
|
+
* one version: a worker refuses to open any other, so two tenants never share
|
|
17
|
+
a process even when they share a host;
|
|
18
|
+
* a socket only the supervisor is answered on. Workers run as the host's user,
|
|
19
|
+
so file modes cannot keep one worker's handler from opening a sibling's
|
|
20
|
+
socket path: the worker asks the kernel who connected (``SO_PEERCRED``) and
|
|
21
|
+
serves the supervisor's pid alone. The listening socket is the supervisor's,
|
|
22
|
+
bound before the worker starts and handed down as a file descriptor, so a
|
|
23
|
+
worker never binds a path of its own;
|
|
24
|
+
* a worker nobody can read: it marks itself non-dumpable, which closes
|
|
25
|
+
``/proc/<pid>/environ``, ``/proc/<pid>/mem`` and ``ptrace`` to every process
|
|
26
|
+
that is not root, siblings under the same uid included. Its secret and its
|
|
27
|
+
sessions' memory stay its own.
|
|
28
|
+
|
|
29
|
+
What this is not: a uid boundary on the filesystem. A worker can read the
|
|
30
|
+
version cache and the files under a sibling's home. The deploy provides that
|
|
31
|
+
fence (a sandboxed runtime class, a read-only root) when it is wanted; see
|
|
32
|
+
``deploy/world-host/README.md``.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
from __future__ import annotations
|
|
36
|
+
|
|
37
|
+
import ctypes
|
|
38
|
+
import http.client
|
|
39
|
+
import os
|
|
40
|
+
import resource
|
|
41
|
+
import secrets
|
|
42
|
+
import shutil
|
|
43
|
+
import socket
|
|
44
|
+
import struct
|
|
45
|
+
import subprocess
|
|
46
|
+
import sys
|
|
47
|
+
import threading
|
|
48
|
+
import time
|
|
49
|
+
from collections.abc import Mapping
|
|
50
|
+
from dataclasses import dataclass, field
|
|
51
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
52
|
+
from pathlib import Path
|
|
53
|
+
|
|
54
|
+
#: Address space a worker may use, in MiB, unless ``WORLDS_WORKER_MEMORY_MB`` says otherwise.
|
|
55
|
+
DEFAULT_WORKER_MEMORY_MB = 2048
|
|
56
|
+
#: Address space every worker together may use, in MiB, unless
|
|
57
|
+
#: ``WORLDS_WORKER_BUDGET_MB`` says otherwise. The number of workers a replica
|
|
58
|
+
#: runs at once is this over the per-worker cap: two of 2 GiB inside the 6 GiB
|
|
59
|
+
#: host container, the rest the supervisor's (its caches, the sessions it
|
|
60
|
+
#: serves in-process, exports in flight). A version past that is refused
|
|
61
|
+
#: ``at_capacity`` and placed on another replica instead of the kernel
|
|
62
|
+
#: choosing which process to kill.
|
|
63
|
+
DEFAULT_WORKER_BUDGET_MB = 4096
|
|
64
|
+
#: Largest file a worker may write (its session databases), in MiB.
|
|
65
|
+
DEFAULT_WORKER_FILE_MB = 4096
|
|
66
|
+
#: Open files per worker: one per live session plus the sockets serving them.
|
|
67
|
+
DEFAULT_WORKER_NOFILE = 4096
|
|
68
|
+
#: How long a worker with no sessions is kept before it is stopped.
|
|
69
|
+
DEFAULT_WORKER_IDLE_S = 300.0
|
|
70
|
+
#: How long the supervisor waits for a fresh worker to answer on its socket.
|
|
71
|
+
WORKER_START_TIMEOUT_S = 30.0
|
|
72
|
+
#: A worker's sessions live as long as the supervisor's do; the supervisor
|
|
73
|
+
#: holds the leases, so the worker's own are effectively unbounded.
|
|
74
|
+
WORKER_LEASE_S = 365.0 * 86400.0
|
|
75
|
+
#: Environment the worker inherits. Nothing else crosses: no host secret, no
|
|
76
|
+
#: cloud credentials, no platform URL.
|
|
77
|
+
INHERITED_ENV = ("PATH", "PYTHONPATH", "LANG", "LC_ALL", "TZ", "VIRTUAL_ENV")
|
|
78
|
+
#: Headers the supervisor never forwards to a worker: the ones that describe
|
|
79
|
+
#: the supervisor's connection, and its own control-plane secret.
|
|
80
|
+
UNFORWARDED_HEADERS = frozenset({"connection", "content-length", "transfer-encoding", "x-host-secret"})
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class WorkerGone(Exception):
|
|
84
|
+
"""The worker did not answer: it exited, or never came up."""
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class WorkerCapacity(Exception):
|
|
88
|
+
"""Every worker the memory budget allows is live and holds sessions, so no
|
|
89
|
+
worker for another version can start here."""
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
# --------------------------------------------------------------------------- #
|
|
93
|
+
# HTTP over a unix socket: the only way into a worker.
|
|
94
|
+
# --------------------------------------------------------------------------- #
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
class UnixHTTPConnection(http.client.HTTPConnection):
|
|
98
|
+
def __init__(self, path: str | Path, timeout: float | None = None):
|
|
99
|
+
super().__init__("worker", timeout=timeout)
|
|
100
|
+
self._unix_path = str(path)
|
|
101
|
+
|
|
102
|
+
def connect(self) -> None:
|
|
103
|
+
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
|
104
|
+
sock.settimeout(self.timeout)
|
|
105
|
+
sock.connect(self._unix_path)
|
|
106
|
+
self.sock = sock
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def bind_worker_socket(path: str | Path) -> socket.socket:
|
|
110
|
+
"""The supervisor's end of a worker: a listening unix socket at ``path``,
|
|
111
|
+
``0600`` inside a ``0700`` directory, ready to hand to the worker."""
|
|
112
|
+
path = Path(path)
|
|
113
|
+
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
114
|
+
path.unlink(missing_ok=True)
|
|
115
|
+
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
|
116
|
+
sock.bind(str(path))
|
|
117
|
+
os.chmod(path, 0o600)
|
|
118
|
+
sock.listen(128)
|
|
119
|
+
return sock
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def peer_pid(conn: socket.socket) -> int | None:
|
|
123
|
+
"""The pid of whoever connected, as the kernel reports it (``SO_PEERCRED``:
|
|
124
|
+
``struct ucred {pid, uid, gid}``); None where the platform has no such
|
|
125
|
+
thing."""
|
|
126
|
+
option = getattr(socket, "SO_PEERCRED", None)
|
|
127
|
+
if option is None:
|
|
128
|
+
return None
|
|
129
|
+
raw = conn.getsockopt(socket.SOL_SOCKET, option, struct.calcsize("3i"))
|
|
130
|
+
pid, _uid, _gid = struct.unpack("3i", raw)
|
|
131
|
+
return pid
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
class UnixHTTPServer(ThreadingHTTPServer):
|
|
135
|
+
"""The worker's server: a ``ThreadingHTTPServer`` over the listening unix
|
|
136
|
+
socket the supervisor handed it (an inherited descriptor), or one it binds
|
|
137
|
+
itself at a path (tests, a hand-run worker). With ``supervisor_pid`` set,
|
|
138
|
+
only connections from that pid are served: the kernel names the peer, so a
|
|
139
|
+
sibling worker that can open the path (same uid) is still refused."""
|
|
140
|
+
|
|
141
|
+
address_family = socket.AF_UNIX
|
|
142
|
+
allow_reuse_address = False
|
|
143
|
+
allow_reuse_port = False
|
|
144
|
+
|
|
145
|
+
def __init__(
|
|
146
|
+
self,
|
|
147
|
+
listening: str | Path | socket.socket,
|
|
148
|
+
handler: type[BaseHTTPRequestHandler],
|
|
149
|
+
*,
|
|
150
|
+
supervisor_pid: int | None = None,
|
|
151
|
+
):
|
|
152
|
+
self.supervisor_pid = supervisor_pid
|
|
153
|
+
self.refused = 0
|
|
154
|
+
# The base class wants a (host, port) pair; it is only what
|
|
155
|
+
# ``server_address`` reports. The socket is bound here, not by it.
|
|
156
|
+
super().__init__(("worker", 0), handler, bind_and_activate=False)
|
|
157
|
+
self.socket.close()
|
|
158
|
+
self.socket = listening if isinstance(listening, socket.socket) else bind_worker_socket(listening)
|
|
159
|
+
self.server_name = "worker"
|
|
160
|
+
self.server_port = 0
|
|
161
|
+
|
|
162
|
+
def verify_request(self, request, client_address) -> bool:
|
|
163
|
+
if self.supervisor_pid is None:
|
|
164
|
+
return True
|
|
165
|
+
if peer_pid(request) == self.supervisor_pid:
|
|
166
|
+
return True
|
|
167
|
+
self.refused += 1
|
|
168
|
+
return False
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
_PR_SET_DUMPABLE = 4
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def seal_process() -> bool:
|
|
175
|
+
"""Make this process unreadable to its peers: not dumpable, so its
|
|
176
|
+
``/proc/<pid>/environ`` and memory are root's to read and ``ptrace`` from
|
|
177
|
+
another process under the same uid is refused; and a ``077`` umask, so what
|
|
178
|
+
it writes is its own. True when the kernel took the setting."""
|
|
179
|
+
os.umask(0o077)
|
|
180
|
+
if not sys.platform.startswith("linux"):
|
|
181
|
+
return False
|
|
182
|
+
try:
|
|
183
|
+
libc = ctypes.CDLL(None, use_errno=True)
|
|
184
|
+
return libc.prctl(_PR_SET_DUMPABLE, 0, 0, 0, 0) == 0
|
|
185
|
+
except (OSError, AttributeError):
|
|
186
|
+
return False
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def apply_limits(
|
|
190
|
+
*,
|
|
191
|
+
memory_mb: int = DEFAULT_WORKER_MEMORY_MB,
|
|
192
|
+
file_mb: int = DEFAULT_WORKER_FILE_MB,
|
|
193
|
+
nofile: int = DEFAULT_WORKER_NOFILE,
|
|
194
|
+
) -> dict[str, int]:
|
|
195
|
+
"""Cap this process before it imports world code. A limit the kernel will
|
|
196
|
+
not grant (a hard limit lower than asked) is lowered to what it grants
|
|
197
|
+
rather than refused: the worker still runs, just tighter. Returns the
|
|
198
|
+
limits as set, so the worker can report them."""
|
|
199
|
+
applied: dict[str, int] = {}
|
|
200
|
+
for name, wanted in (
|
|
201
|
+
(resource.RLIMIT_AS, memory_mb << 20),
|
|
202
|
+
(resource.RLIMIT_FSIZE, file_mb << 20),
|
|
203
|
+
(resource.RLIMIT_NOFILE, nofile),
|
|
204
|
+
(resource.RLIMIT_CORE, 0),
|
|
205
|
+
):
|
|
206
|
+
_, hard = resource.getrlimit(name)
|
|
207
|
+
value = wanted if hard == resource.RLIM_INFINITY else min(wanted, hard)
|
|
208
|
+
resource.setrlimit(name, (value, value))
|
|
209
|
+
applied[_limit_name(name)] = value
|
|
210
|
+
return applied
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _limit_name(limit: int) -> str:
|
|
214
|
+
return {
|
|
215
|
+
resource.RLIMIT_AS: "memory_bytes",
|
|
216
|
+
resource.RLIMIT_FSIZE: "file_bytes",
|
|
217
|
+
resource.RLIMIT_NOFILE: "open_files",
|
|
218
|
+
resource.RLIMIT_CORE: "core_bytes",
|
|
219
|
+
}[limit]
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def worker_environment(
|
|
223
|
+
*,
|
|
224
|
+
secret: str,
|
|
225
|
+
home: Path,
|
|
226
|
+
worlds_domain: str,
|
|
227
|
+
memory_mb: int,
|
|
228
|
+
source: Mapping[str, str] | None = None,
|
|
229
|
+
) -> dict[str, str]:
|
|
230
|
+
"""The environment a worker is started with: the interpreter's own needs,
|
|
231
|
+
a home of its own, and the worker's settings. Built from an allow-list,
|
|
232
|
+
so whatever else the supervisor was given stays with the supervisor."""
|
|
233
|
+
inherited: Mapping[str, str] = os.environ if source is None else source
|
|
234
|
+
env = {name: inherited[name] for name in INHERITED_ENV if name in inherited}
|
|
235
|
+
env.update(
|
|
236
|
+
HOME=str(home),
|
|
237
|
+
WORLDS_WORKER_SECRET=secret,
|
|
238
|
+
WORLDS_WORKER_MEMORY_MB=str(memory_mb),
|
|
239
|
+
WORLDS_INGRESS_DOMAIN=worlds_domain,
|
|
240
|
+
WORLDS_SUPERVISOR_PID=str(os.getpid()),
|
|
241
|
+
)
|
|
242
|
+
return env
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def verifier_environment(*, home: Path, source: Mapping[str, str] | None = None) -> dict[str, str]:
|
|
246
|
+
"""The environment a scenario's ``verify.py`` runs with: the interpreter's
|
|
247
|
+
needs and a home under the session, nothing of the process that started
|
|
248
|
+
it. A verifier is world-authored code too; it gets no host or worker
|
|
249
|
+
secret, no cloud credentials, no platform URL."""
|
|
250
|
+
inherited: Mapping[str, str] = os.environ if source is None else source
|
|
251
|
+
env = {name: inherited[name] for name in INHERITED_ENV if name in inherited}
|
|
252
|
+
env["HOME"] = str(home)
|
|
253
|
+
return env
|
|
254
|
+
|
|
255
|
+
|
|
256
|
+
def confine_child() -> None:
|
|
257
|
+
"""The worker's rlimits and seal, applied by a child that is about to run
|
|
258
|
+
world code (a verifier), before that code is loaded."""
|
|
259
|
+
apply_limits()
|
|
260
|
+
seal_process()
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
#: ``python -c`` prelude that confines the interpreter, then runs the script
|
|
264
|
+
#: named next on the command line as ``__main__``. Done in the child, not via
|
|
265
|
+
#: ``preexec_fn``, which is not safe under the supervisor's request threads.
|
|
266
|
+
_CONFINED_PRELUDE = (
|
|
267
|
+
"from gatewaysdk.environment.schema.workers import confine_child; confine_child(); "
|
|
268
|
+
"import runpy, sys; sys.argv = sys.argv[1:]; runpy.run_path(sys.argv[0], run_name='__main__')"
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def confined_command(script: Path, python: str = sys.executable) -> list[str]:
|
|
273
|
+
"""The argv that runs ``script`` as a confined child (see :func:`confine_child`)."""
|
|
274
|
+
return [python, "-c", _CONFINED_PRELUDE, str(script)]
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
# --------------------------------------------------------------------------- #
|
|
278
|
+
# The supervisor's side: start, reach and stop workers.
|
|
279
|
+
# --------------------------------------------------------------------------- #
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
@dataclass
|
|
283
|
+
class Worker:
|
|
284
|
+
version_id: str
|
|
285
|
+
process: subprocess.Popen
|
|
286
|
+
socket_path: Path
|
|
287
|
+
secret: str
|
|
288
|
+
#: Sessions the supervisor has placed on this worker.
|
|
289
|
+
sessions: set[str] = field(default_factory=set)
|
|
290
|
+
#: When the worker last had a session, for idle reaping.
|
|
291
|
+
idle_since: float = field(default_factory=time.time)
|
|
292
|
+
|
|
293
|
+
@property
|
|
294
|
+
def pid(self) -> int:
|
|
295
|
+
return self.process.pid
|
|
296
|
+
|
|
297
|
+
def alive(self) -> bool:
|
|
298
|
+
return self.process.poll() is None
|
|
299
|
+
|
|
300
|
+
def request(
|
|
301
|
+
self,
|
|
302
|
+
method: str,
|
|
303
|
+
path: str,
|
|
304
|
+
headers: dict[str, str],
|
|
305
|
+
body: bytes = b"",
|
|
306
|
+
*,
|
|
307
|
+
control: bool = False,
|
|
308
|
+
timeout: float = 60.0,
|
|
309
|
+
) -> tuple[int, list[tuple[str, str]], bytes]:
|
|
310
|
+
"""One request over the socket. ``control`` adds the worker's own
|
|
311
|
+
secret, so the worker's control plane answers. A worker that cannot be
|
|
312
|
+
reached raises :class:`WorkerGone`."""
|
|
313
|
+
forward = {k: v for k, v in headers.items() if k.lower() not in UNFORWARDED_HEADERS}
|
|
314
|
+
if control:
|
|
315
|
+
forward["X-Host-Secret"] = self.secret
|
|
316
|
+
conn = UnixHTTPConnection(self.socket_path, timeout=timeout)
|
|
317
|
+
try:
|
|
318
|
+
conn.request(method, path, body=body or None, headers=forward)
|
|
319
|
+
answer = conn.getresponse()
|
|
320
|
+
payload = answer.read()
|
|
321
|
+
return answer.status, answer.getheaders(), payload
|
|
322
|
+
except (OSError, http.client.HTTPException) as error:
|
|
323
|
+
raise WorkerGone(
|
|
324
|
+
f"worker for version {self.version_id} (pid {self.pid}) did not answer: {error}"
|
|
325
|
+
) from error
|
|
326
|
+
finally:
|
|
327
|
+
conn.close()
|
|
328
|
+
|
|
329
|
+
def stop(self, grace_s: float = 5.0) -> None:
|
|
330
|
+
if self.alive():
|
|
331
|
+
self.process.terminate()
|
|
332
|
+
try:
|
|
333
|
+
self.process.wait(timeout=grace_s)
|
|
334
|
+
except subprocess.TimeoutExpired:
|
|
335
|
+
self.process.kill()
|
|
336
|
+
self.process.wait(timeout=grace_s)
|
|
337
|
+
self.socket_path.unlink(missing_ok=True)
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
class WorkerPool:
|
|
341
|
+
"""Every version worker on this replica, started on the first session of a
|
|
342
|
+
version and stopped once it has had none for a while."""
|
|
343
|
+
|
|
344
|
+
def __init__(
|
|
345
|
+
self,
|
|
346
|
+
cache_root: Path,
|
|
347
|
+
*,
|
|
348
|
+
worlds_domain: str,
|
|
349
|
+
memory_mb: int = DEFAULT_WORKER_MEMORY_MB,
|
|
350
|
+
budget_mb: int = DEFAULT_WORKER_BUDGET_MB,
|
|
351
|
+
idle_s: float = DEFAULT_WORKER_IDLE_S,
|
|
352
|
+
python: str = sys.executable,
|
|
353
|
+
):
|
|
354
|
+
if budget_mb < memory_mb:
|
|
355
|
+
raise ValueError(
|
|
356
|
+
f"the worker budget ({budget_mb} MiB) holds no worker of {memory_mb} MiB "
|
|
357
|
+
"(WORLDS_WORKER_BUDGET_MB, WORLDS_WORKER_MEMORY_MB)"
|
|
358
|
+
)
|
|
359
|
+
self.root = Path(cache_root)
|
|
360
|
+
self.workers_root = self.root / "workers"
|
|
361
|
+
# Workers live and die with the supervisor; whatever a previous one
|
|
362
|
+
# left (sockets, session files) is unreachable.
|
|
363
|
+
shutil.rmtree(self.workers_root, ignore_errors=True)
|
|
364
|
+
self.workers_root.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
365
|
+
self.worlds_domain = worlds_domain
|
|
366
|
+
self.memory_mb = memory_mb
|
|
367
|
+
self.budget_mb = budget_mb
|
|
368
|
+
#: Workers live at once: the budget's worth of per-worker address space.
|
|
369
|
+
self.max_workers = budget_mb // memory_mb
|
|
370
|
+
self.idle_s = idle_s
|
|
371
|
+
self.python = python
|
|
372
|
+
self._lock = threading.Lock()
|
|
373
|
+
self._workers: dict[str, Worker] = {}
|
|
374
|
+
|
|
375
|
+
def get(self, version_id: str) -> Worker:
|
|
376
|
+
"""The live worker for a version, started if there is none. Starting one
|
|
377
|
+
past the budget first stops workers holding no sessions (whatever their
|
|
378
|
+
idle time); when every one is in use, raises :class:`WorkerCapacity`."""
|
|
379
|
+
with self._lock:
|
|
380
|
+
worker = self._workers.get(version_id)
|
|
381
|
+
if worker is not None and worker.alive():
|
|
382
|
+
return worker
|
|
383
|
+
if worker is not None:
|
|
384
|
+
worker.stop()
|
|
385
|
+
del self._workers[version_id]
|
|
386
|
+
self._make_room()
|
|
387
|
+
worker = self._spawn(version_id)
|
|
388
|
+
self._workers[version_id] = worker
|
|
389
|
+
return worker
|
|
390
|
+
|
|
391
|
+
def _make_room(self) -> None:
|
|
392
|
+
"""Under ``_lock``: leave one worker's worth of the budget free, by
|
|
393
|
+
forgetting exited workers first, then stopping session-less ones
|
|
394
|
+
longest idle first, and refusing when every live worker is in use."""
|
|
395
|
+
exited = [v for v, w in self._workers.items() if not w.alive()]
|
|
396
|
+
for version_id in exited:
|
|
397
|
+
self._workers.pop(version_id).stop()
|
|
398
|
+
spare = sorted(
|
|
399
|
+
(w for w in self._workers.values() if not w.sessions),
|
|
400
|
+
key=lambda w: w.idle_since,
|
|
401
|
+
)
|
|
402
|
+
while len(self._workers) >= self.max_workers and spare:
|
|
403
|
+
self._workers.pop(spare.pop(0).version_id).stop()
|
|
404
|
+
if len(self._workers) >= self.max_workers:
|
|
405
|
+
raise WorkerCapacity(
|
|
406
|
+
f"this host's {self.max_workers} version workers ({self.budget_mb} MiB of "
|
|
407
|
+
f"{self.memory_mb} MiB each) all hold sessions; place elsewhere"
|
|
408
|
+
)
|
|
409
|
+
|
|
410
|
+
def peek(self, version_id: str) -> Worker | None:
|
|
411
|
+
with self._lock:
|
|
412
|
+
return self._workers.get(version_id)
|
|
413
|
+
|
|
414
|
+
def drop(self, version_id: str) -> Worker | None:
|
|
415
|
+
"""Stop and forget a version's worker; returns it (with the sessions
|
|
416
|
+
it held) so the caller can forget those too."""
|
|
417
|
+
with self._lock:
|
|
418
|
+
worker = self._workers.pop(version_id, None)
|
|
419
|
+
if worker is not None:
|
|
420
|
+
worker.stop()
|
|
421
|
+
return worker
|
|
422
|
+
|
|
423
|
+
def reap_idle(self, now: float | None = None) -> list[str]:
|
|
424
|
+
"""Stop workers that have had no session for ``idle_s``; returns their versions."""
|
|
425
|
+
now = time.time() if now is None else now
|
|
426
|
+
with self._lock:
|
|
427
|
+
idle = [
|
|
428
|
+
w.version_id
|
|
429
|
+
for w in self._workers.values()
|
|
430
|
+
if not w.sessions and (now - w.idle_since >= self.idle_s or not w.alive())
|
|
431
|
+
]
|
|
432
|
+
stopped = [self._workers.pop(v) for v in idle]
|
|
433
|
+
for worker in stopped:
|
|
434
|
+
worker.stop()
|
|
435
|
+
return idle
|
|
436
|
+
|
|
437
|
+
def stop_all(self) -> None:
|
|
438
|
+
with self._lock:
|
|
439
|
+
workers = list(self._workers.values())
|
|
440
|
+
self._workers.clear()
|
|
441
|
+
for worker in workers:
|
|
442
|
+
worker.stop()
|
|
443
|
+
|
|
444
|
+
def _spawn(self, version_id: str) -> Worker:
|
|
445
|
+
home = self.workers_root / version_id
|
|
446
|
+
home.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
447
|
+
socket_path = home / "worker.sock"
|
|
448
|
+
secret = secrets.token_urlsafe(24)
|
|
449
|
+
# The supervisor binds and listens; the worker inherits the descriptor
|
|
450
|
+
# and serves on it, so the socket is the supervisor's from the start.
|
|
451
|
+
listening = bind_worker_socket(socket_path)
|
|
452
|
+
try:
|
|
453
|
+
process = subprocess.Popen(
|
|
454
|
+
[
|
|
455
|
+
self.python,
|
|
456
|
+
"-m",
|
|
457
|
+
"gatewaysdk.environment.schema.worker",
|
|
458
|
+
"--cache-root",
|
|
459
|
+
str(self.root),
|
|
460
|
+
"--version",
|
|
461
|
+
version_id,
|
|
462
|
+
"--socket-fd",
|
|
463
|
+
str(listening.fileno()),
|
|
464
|
+
],
|
|
465
|
+
env=worker_environment(
|
|
466
|
+
secret=secret,
|
|
467
|
+
home=home,
|
|
468
|
+
worlds_domain=self.worlds_domain,
|
|
469
|
+
memory_mb=self.memory_mb,
|
|
470
|
+
),
|
|
471
|
+
cwd=str(home),
|
|
472
|
+
stdin=subprocess.DEVNULL,
|
|
473
|
+
stdout=sys.stderr,
|
|
474
|
+
stderr=sys.stderr,
|
|
475
|
+
start_new_session=True,
|
|
476
|
+
pass_fds=(listening.fileno(),),
|
|
477
|
+
)
|
|
478
|
+
finally:
|
|
479
|
+
listening.close()
|
|
480
|
+
worker = Worker(version_id, process, socket_path, secret)
|
|
481
|
+
self._await_ready(worker)
|
|
482
|
+
return worker
|
|
483
|
+
|
|
484
|
+
def _await_ready(self, worker: Worker) -> None:
|
|
485
|
+
deadline = time.time() + WORKER_START_TIMEOUT_S
|
|
486
|
+
while time.time() < deadline:
|
|
487
|
+
if not worker.alive():
|
|
488
|
+
raise WorkerGone(
|
|
489
|
+
f"worker for version {worker.version_id} exited with {worker.process.returncode} on start"
|
|
490
|
+
)
|
|
491
|
+
if worker.socket_path.exists():
|
|
492
|
+
try:
|
|
493
|
+
status, _, _ = worker.request("GET", "/healthz", {}, timeout=2.0)
|
|
494
|
+
except WorkerGone:
|
|
495
|
+
status = 0
|
|
496
|
+
if status == 200:
|
|
497
|
+
return
|
|
498
|
+
time.sleep(0.05)
|
|
499
|
+
worker.stop()
|
|
500
|
+
raise WorkerGone(f"worker for version {worker.version_id} did not come up in {WORKER_START_TIMEOUT_S:.0f}s")
|