@withgateway/sdk 0.3.3 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -3
- package/dist/cli.js +3005 -1401
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2996 -1392
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js +128 -37
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs +126 -35
- package/dist/files/index.mjs.map +1 -1
- package/dist/hooks/index.d.mts +1 -1
- package/dist/hooks/index.d.ts +1 -1
- package/dist/hooks/index.js +124 -11
- package/dist/hooks/index.js.map +1 -1
- package/dist/hooks/index.mjs +124 -11
- package/dist/hooks/index.mjs.map +1 -1
- package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
- package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
- package/dist/{index-mJhG3vZU.d.mts → index-Di9nNqOY.d.mts} +26 -3
- package/dist/{index-DUHmS74p.d.ts → index-LA-PMRJT.d.ts} +26 -3
- package/dist/index.d.mts +4 -4
- package/dist/index.d.ts +4 -4
- package/dist/index.js +1078 -260
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1040 -251
- package/dist/index.mjs.map +1 -1
- package/dist/replay/index.d.mts +1 -1
- package/dist/replay/index.d.ts +1 -1
- package/dist/replay/index.js.map +1 -1
- package/dist/replay/index.mjs.map +1 -1
- package/dist/traces/index.js +125 -34
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs +123 -32
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.d.mts +1 -1
- package/dist/tracing/index.d.ts +1 -1
- package/dist/tracing/index.js +7 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +7 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +819 -438
- package/dist/worlds/index.d.ts +819 -438
- package/dist/worlds/index.js +912 -236
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +908 -232
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/_bundle.py +4 -6
- package/runtime/gatewaysdk/environment/_materialize.py +5 -25
- package/runtime/gatewaysdk/environment/_world.py +82 -66
- package/runtime/gatewaysdk/environment/backing.py +20 -24
- package/runtime/gatewaysdk/environment/core.py +1 -16
- package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
- package/runtime/gatewaysdk/environment/runtime.py +45 -2
- package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
- package/runtime/gatewaysdk/environment/schema/api.py +198 -474
- package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
- package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
- package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
- package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
- package/runtime/gatewaysdk/environment/schema/call_capture.py +321 -0
- package/runtime/gatewaysdk/environment/schema/call_log.py +276 -18
- package/runtime/gatewaysdk/environment/schema/call_redaction.py +207 -19
- package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
- package/runtime/gatewaysdk/environment/schema/compiler.py +4 -1
- package/runtime/gatewaysdk/environment/schema/conform.py +38 -35
- package/runtime/gatewaysdk/environment/schema/connector.py +94 -152
- package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
- package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +2 -1
- package/runtime/gatewaysdk/environment/schema/connectors/github/api/handlers.py +217 -58
- package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +10 -19
- package/runtime/gatewaysdk/environment/schema/connectors/github/scenarios/default/seed/github_webhooks.json +10 -2
- package/runtime/gatewaysdk/environment/schema/connectors/github/tests/webhooks.json +47 -0
- package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +21 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/api/handlers.py +185 -18
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/connector.toml +29 -14
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +43 -12
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scenarios/default/seed/jamf_webhooks.json +2 -2
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +4 -17
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/tests/webhooks.json +107 -0
- package/runtime/gatewaysdk/environment/schema/connectors/jamf/vendor-openapi.json +1 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/api/handlers.py +85 -1
- package/runtime/gatewaysdk/environment/schema/connectors/linear/connector.toml +19 -31
- package/runtime/gatewaysdk/environment/schema/connectors/linear/provenance.json +18 -8
- package/runtime/gatewaysdk/environment/schema/connectors/linear/scenarios/default/seed/webhooks.json +19 -3
- package/runtime/gatewaysdk/environment/schema/connectors/linear/tests/test_webhooks.py +71 -0
- package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
- package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
- package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
- package/runtime/gatewaysdk/environment/schema/data.py +14 -13
- package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
- package/runtime/gatewaysdk/environment/schema/database/guard.py +45 -37
- package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
- package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
- package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
- package/runtime/gatewaysdk/environment/schema/door.py +899 -0
- package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
- package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +52 -5
- package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
- package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
- package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
- package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
- package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
- package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
- package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
- package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
- package/runtime/gatewaysdk/environment/schema/journal.py +438 -0
- package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
- package/runtime/gatewaysdk/environment/schema/local.py +32 -116
- package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +20 -5
- package/runtime/gatewaysdk/environment/schema/platform.py +29 -234
- package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
- package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
- package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
- package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +136 -176
- package/runtime/gatewaysdk/environment/schema/scaffold.py +2 -1
- package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
- package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
- package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +50 -28
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +20 -22
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +25 -25
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
- package/runtime/gatewaysdk/environment/schema/snapshot.py +150 -30
- package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
- package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
- package/runtime/gatewaysdk/environment/schema/tools_world.py +103 -203
- package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
- package/runtime/gatewaysdk/environment/schema/verbs.py +38 -36
- package/runtime/gatewaysdk/environment/schema/webhooks.py +687 -0
- package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
- package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
- package/runtime/gatewaysdk/environment/schema/world_host.py +970 -0
- package/runtime/gatewaysdk/environment/schema/world_tests.py +83 -11
- package/runtime/gatewaysdk/environment/world.py +1 -1
- package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
- package/skills/agents/connector-template-author.md +5 -1
- package/skills/benchmarks-getting-started/SKILL.md +8 -9
- package/skills/connector-schema-authoring/CONNECTORS.md +3 -3
- package/skills/connector-schema-authoring/SKILL.md +50 -28
- package/skills/connector-schema-authoring/connector-worlds.md +282 -18
- package/skills/connector-schema-authoring/world-schema.md +7 -6
- package/skills/world-data-ingestion/ROWS.md +12 -8
- package/skills/world-data-ingestion/SKILL.md +20 -22
- package/skills/worlds-getting-started/SKILL.md +25 -25
- package/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
- package/skills/worlds-getting-started/references/gotchas.md +71 -7
- package/skills/worlds-getting-started/references/journal-checks.md +182 -0
|
@@ -15,7 +15,8 @@ import socket
|
|
|
15
15
|
import sys
|
|
16
16
|
from pathlib import Path
|
|
17
17
|
|
|
18
|
-
from .
|
|
18
|
+
from .cache_layout import VERSIONS_DIR, WORKERS_DIR
|
|
19
|
+
from .host import make_host_handler
|
|
19
20
|
from .workers import (
|
|
20
21
|
DEFAULT_WORKER_MEMORY_MB,
|
|
21
22
|
WORKER_LEASE_S,
|
|
@@ -23,6 +24,7 @@ from .workers import (
|
|
|
23
24
|
apply_limits,
|
|
24
25
|
seal_process,
|
|
25
26
|
)
|
|
27
|
+
from .world_host import WorldHost
|
|
26
28
|
|
|
27
29
|
|
|
28
30
|
def main(argv: list[str] | None = None) -> int:
|
|
@@ -39,7 +41,7 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
39
41
|
raise SystemExit("world worker: WORLDS_WORKER_SECRET is unset; a worker only answers its supervisor")
|
|
40
42
|
supervisor_pid: int | None = None
|
|
41
43
|
if args.socket_fd is not None:
|
|
42
|
-
supervisor_pid =
|
|
44
|
+
supervisor_pid = os.getppid()
|
|
43
45
|
listening: socket.socket | str = socket.socket(fileno=args.socket_fd)
|
|
44
46
|
else:
|
|
45
47
|
listening = args.socket
|
|
@@ -48,8 +50,8 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
48
50
|
|
|
49
51
|
cache_root = Path(args.cache_root)
|
|
50
52
|
world_host = WorldHost(
|
|
51
|
-
cache_root /
|
|
52
|
-
versions_root=cache_root /
|
|
53
|
+
cache_root / WORKERS_DIR / args.version,
|
|
54
|
+
versions_root=cache_root / VERSIONS_DIR,
|
|
53
55
|
lease_s=WORKER_LEASE_S,
|
|
54
56
|
run_handlers=True,
|
|
55
57
|
pinned_version=args.version,
|
|
@@ -35,6 +35,7 @@ fence (a sandboxed runtime class, a read-only root) when it is wanted; see
|
|
|
35
35
|
|
|
36
36
|
from __future__ import annotations
|
|
37
37
|
|
|
38
|
+
import collections
|
|
38
39
|
import ctypes
|
|
39
40
|
import hashlib
|
|
40
41
|
import http.client
|
|
@@ -49,13 +50,16 @@ import sys
|
|
|
49
50
|
import tempfile
|
|
50
51
|
import threading
|
|
51
52
|
import time
|
|
52
|
-
from collections.abc import Mapping
|
|
53
|
+
from collections.abc import Callable, Iterator, Mapping
|
|
54
|
+
from contextlib import contextmanager, suppress
|
|
53
55
|
from dataclasses import dataclass, field
|
|
54
56
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
55
57
|
from pathlib import Path
|
|
56
58
|
from typing import Protocol
|
|
57
59
|
|
|
60
|
+
from .cache_layout import WORKERS_DIR
|
|
58
61
|
from .grading import VerifierSandbox
|
|
62
|
+
from .handler_errors import log_line
|
|
59
63
|
|
|
60
64
|
#: Address space a worker may use, in MiB, unless ``WORLDS_WORKER_MEMORY_MB`` says otherwise.
|
|
61
65
|
DEFAULT_WORKER_MEMORY_MB = 2048
|
|
@@ -81,10 +85,12 @@ FLOOR_PRECISION_BYTES = 1 << 20
|
|
|
81
85
|
#: The largest limit a kernel reports, which ``RLIM_INFINITY`` is read as in arithmetic: macOS
|
|
82
86
|
#: spells it so, Linux spells it -1.
|
|
83
87
|
MAX_RLIMIT = 2**63 - 1
|
|
84
|
-
#: How long a worker with no sessions is kept before it is stopped.
|
|
85
88
|
DEFAULT_WORKER_IDLE_S = 300.0
|
|
86
|
-
#: How long the supervisor waits for a fresh worker to answer on its socket.
|
|
87
89
|
WORKER_START_TIMEOUT_S = 30.0
|
|
90
|
+
#: How long one health probe waits: a worker that cannot answer it in this time is not serving.
|
|
91
|
+
HEALTH_TIMEOUT_S = 2.0
|
|
92
|
+
#: The most a probe reads of a worker's answer: the worker runs tenant code, so its size is not its to choose.
|
|
93
|
+
PROBE_LIMIT_BYTES = 64 << 10
|
|
88
94
|
#: A worker's sessions live as long as the supervisor's do; the supervisor
|
|
89
95
|
#: holds the leases, so the worker's own are effectively unbounded.
|
|
90
96
|
WORKER_LEASE_S = 365.0 * 86400.0
|
|
@@ -94,10 +100,20 @@ INHERITED_ENV = ("PATH", "PYTHONPATH", "LANG", "LC_ALL", "TZ", "VIRTUAL_ENV")
|
|
|
94
100
|
#: Headers the supervisor never forwards to a worker: the ones that describe
|
|
95
101
|
#: the supervisor's connection, and its own control-plane secret.
|
|
96
102
|
UNFORWARDED_HEADERS = frozenset({"connection", "content-length", "transfer-encoding", "x-host-secret"})
|
|
103
|
+
FINISH_THREADS = 8
|
|
104
|
+
FINISH_SETTLE_S = 10.0
|
|
97
105
|
|
|
98
106
|
|
|
99
107
|
class WorkerGone(Exception):
|
|
100
|
-
"""The worker did not answer: it exited,
|
|
108
|
+
"""The worker did not answer: it exited, never came up, or dropped the request."""
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class WorkerTimeout(WorkerGone):
|
|
112
|
+
"""The worker did not answer in time; it may be alive and still working on the request."""
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
class WorkerUnreached(WorkerGone):
|
|
116
|
+
"""The worker took no connection: the request never reached it."""
|
|
101
117
|
|
|
102
118
|
|
|
103
119
|
class WorkerCapacity(Exception):
|
|
@@ -117,6 +133,24 @@ class UnixHTTPConnection(http.client.HTTPConnection):
|
|
|
117
133
|
self.sock = sock
|
|
118
134
|
|
|
119
135
|
|
|
136
|
+
def _hang_up(sock: socket.socket, cut: threading.Event) -> None:
|
|
137
|
+
cut.set()
|
|
138
|
+
with suppress(OSError):
|
|
139
|
+
sock.shutdown(socket.SHUT_RDWR)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
@contextmanager
|
|
143
|
+
def _deadline(sock: socket.socket, seconds: float) -> Iterator[threading.Event]:
|
|
144
|
+
"""Shut ``sock`` once ``seconds`` pass, so nothing read from it waits longer; the event says it was."""
|
|
145
|
+
cut = threading.Event()
|
|
146
|
+
timer = threading.Timer(seconds, _hang_up, (sock, cut))
|
|
147
|
+
timer.start()
|
|
148
|
+
try:
|
|
149
|
+
yield cut
|
|
150
|
+
finally:
|
|
151
|
+
timer.cancel()
|
|
152
|
+
|
|
153
|
+
|
|
120
154
|
def bind_worker_socket(path: str | Path) -> socket.socket:
|
|
121
155
|
"""The supervisor's end of a worker: a listening unix socket at ``path``,
|
|
122
156
|
``0600`` inside a ``0700`` directory, ready to hand to the worker."""
|
|
@@ -333,7 +367,6 @@ def worker_environment(
|
|
|
333
367
|
WORLDS_WORKER_SECRET=secret,
|
|
334
368
|
WORLDS_WORKER_MEMORY_MB=str(memory_mb),
|
|
335
369
|
WORLDS_INGRESS_DOMAIN=worlds_domain,
|
|
336
|
-
WORLDS_SUPERVISOR_PID=str(os.getpid()),
|
|
337
370
|
)
|
|
338
371
|
return env
|
|
339
372
|
|
|
@@ -389,7 +422,6 @@ class Worker:
|
|
|
389
422
|
socket_path: Path
|
|
390
423
|
secret: str
|
|
391
424
|
sessions: set[str] = field(default_factory=set)
|
|
392
|
-
#: When the worker last had a session, for idle reaping.
|
|
393
425
|
idle_since: float = field(default_factory=time.time)
|
|
394
426
|
|
|
395
427
|
@property
|
|
@@ -408,25 +440,78 @@ class Worker:
|
|
|
408
440
|
*,
|
|
409
441
|
control: bool = False,
|
|
410
442
|
timeout: float = 60.0,
|
|
443
|
+
limit: int | None = None,
|
|
411
444
|
) -> tuple[int, list[tuple[str, str]], bytes]:
|
|
412
445
|
"""One request over the socket. ``control`` adds the worker's own
|
|
413
|
-
secret, so the worker's control plane answers. A worker that
|
|
414
|
-
|
|
446
|
+
secret, so the worker's control plane answers. A worker that takes no connection raises
|
|
447
|
+
:class:`WorkerUnreached`; one that drops the request or answers past ``limit`` bytes raises
|
|
448
|
+
:class:`WorkerGone`. The whole exchange gets ``timeout`` seconds, not each read: an answer
|
|
449
|
+
sent slowly is cut off when they pass (the connection is shut, not waited out) and raises
|
|
450
|
+
:class:`WorkerTimeout`."""
|
|
415
451
|
forward = {k: v for k, v in headers.items() if k.lower() not in UNFORWARDED_HEADERS}
|
|
416
452
|
if control:
|
|
417
453
|
forward["X-Host-Secret"] = self.secret
|
|
418
|
-
|
|
454
|
+
started = time.monotonic()
|
|
455
|
+
conn = self._connected(timeout)
|
|
456
|
+
cut = threading.Event()
|
|
419
457
|
try:
|
|
420
|
-
conn.
|
|
421
|
-
|
|
422
|
-
|
|
458
|
+
with _deadline(conn.sock, timeout - (time.monotonic() - started)) as cut:
|
|
459
|
+
conn.request(method, path, body=body or None, headers=forward)
|
|
460
|
+
answer = conn.getresponse()
|
|
461
|
+
payload = answer.read() if limit is None else answer.read(limit + 1)
|
|
462
|
+
if cut.is_set():
|
|
463
|
+
raise WorkerTimeout(
|
|
464
|
+
f"worker for version {self.version_id} (pid {self.pid}) did not answer in {timeout:g} s"
|
|
465
|
+
)
|
|
466
|
+
if limit is not None and len(payload) > limit:
|
|
467
|
+
raise WorkerGone(f"worker for version {self.version_id} (pid {self.pid}) answered past {limit} bytes")
|
|
423
468
|
return answer.status, answer.getheaders(), payload
|
|
424
469
|
except (OSError, http.client.HTTPException) as error:
|
|
425
|
-
|
|
426
|
-
|
|
470
|
+
gone = WorkerTimeout if cut.is_set() or isinstance(error, TimeoutError) else WorkerGone
|
|
471
|
+
raise gone(f"worker for version {self.version_id} (pid {self.pid}) did not answer: {error}") from error
|
|
472
|
+
finally:
|
|
473
|
+
conn.close()
|
|
474
|
+
|
|
475
|
+
def _connected(self, timeout: float) -> UnixHTTPConnection:
|
|
476
|
+
conn = UnixHTTPConnection(self.socket_path, timeout=timeout)
|
|
477
|
+
try:
|
|
478
|
+
conn.connect()
|
|
479
|
+
except OSError as error:
|
|
480
|
+
conn.close()
|
|
481
|
+
raise WorkerUnreached(
|
|
482
|
+
f"worker for version {self.version_id} (pid {self.pid}) took no connection: {error}"
|
|
427
483
|
) from error
|
|
484
|
+
return conn
|
|
485
|
+
|
|
486
|
+
def probe(
|
|
487
|
+
self, path: str, *, deadline_s: float = HEALTH_TIMEOUT_S, limit: int = PROBE_LIMIT_BYTES
|
|
488
|
+
) -> tuple[int, bytes]:
|
|
489
|
+
"""One control GET whose whole answer must arrive within ``deadline_s`` and ``limit`` bytes.
|
|
490
|
+
The worker runs tenant code: an answer sent slowly or without end is cut off when the
|
|
491
|
+
deadline passes (the connection is shut, not waited out), and raises :class:`WorkerGone`."""
|
|
492
|
+
unanswered = f"worker for version {self.version_id} (pid {self.pid}) did not answer {path}"
|
|
493
|
+
conn = UnixHTTPConnection(self.socket_path, timeout=deadline_s)
|
|
494
|
+
try:
|
|
495
|
+
conn.connect()
|
|
496
|
+
with _deadline(conn.sock, deadline_s) as cut:
|
|
497
|
+
conn.request("GET", path, headers={"X-Host-Secret": self.secret})
|
|
498
|
+
answer = conn.getresponse()
|
|
499
|
+
payload = answer.read(limit + 1)
|
|
500
|
+
except (OSError, http.client.HTTPException) as error:
|
|
501
|
+
raise WorkerGone(unanswered) from error
|
|
428
502
|
finally:
|
|
429
503
|
conn.close()
|
|
504
|
+
if cut.is_set() or len(payload) > limit:
|
|
505
|
+
raise WorkerGone(f"{unanswered} within {deadline_s:g} s and {limit} bytes")
|
|
506
|
+
return answer.status, payload
|
|
507
|
+
|
|
508
|
+
def answers_health(self, timeout_s: float = HEALTH_TIMEOUT_S) -> bool:
|
|
509
|
+
"""Whether the worker is alive and answers its health check, whole, in time."""
|
|
510
|
+
try:
|
|
511
|
+
status, _ = self.probe("/healthz", deadline_s=timeout_s)
|
|
512
|
+
except WorkerGone:
|
|
513
|
+
return False
|
|
514
|
+
return status == 200
|
|
430
515
|
|
|
431
516
|
def stop(self, grace_s: float = 5.0) -> None:
|
|
432
517
|
if self.alive():
|
|
@@ -439,6 +524,69 @@ class Worker:
|
|
|
439
524
|
self.socket_path.unlink(missing_ok=True)
|
|
440
525
|
|
|
441
526
|
|
|
527
|
+
class Finishing:
|
|
528
|
+
"""What leaves a session on a version worker, finished off the lease sweeper's thread: a few daemon
|
|
529
|
+
threads, one job at a time for a version in the order they came (so a worker that never answers
|
|
530
|
+
holds one thread, not the pool). A session with no worker is finished at once, where it is."""
|
|
531
|
+
|
|
532
|
+
def __init__(self, threads: int = FINISH_THREADS) -> None:
|
|
533
|
+
self.threads = threads
|
|
534
|
+
self._changed = threading.Condition()
|
|
535
|
+
self._waiting: dict[str, collections.deque[Callable[[], None]]] = {}
|
|
536
|
+
self._serving: set[str] = set()
|
|
537
|
+
self._started = 0
|
|
538
|
+
|
|
539
|
+
def leave(self, worker: Worker | None, job: Callable[[], None]) -> None:
|
|
540
|
+
if worker is None:
|
|
541
|
+
job()
|
|
542
|
+
return
|
|
543
|
+
with self._changed:
|
|
544
|
+
self._waiting.setdefault(worker.version_id, collections.deque()).append(job)
|
|
545
|
+
self._staff()
|
|
546
|
+
self._changed.notify()
|
|
547
|
+
|
|
548
|
+
def settle(self, deadline_s: float = FINISH_SETTLE_S) -> bool:
|
|
549
|
+
"""Wait up to ``deadline_s`` for every job handed off to be done; whether they are."""
|
|
550
|
+
until = time.monotonic() + deadline_s
|
|
551
|
+
with self._changed:
|
|
552
|
+
while self._waiting:
|
|
553
|
+
remaining = until - time.monotonic()
|
|
554
|
+
if remaining <= 0:
|
|
555
|
+
return False
|
|
556
|
+
self._changed.wait(remaining)
|
|
557
|
+
return True
|
|
558
|
+
|
|
559
|
+
def _staff(self) -> None:
|
|
560
|
+
if self._started < self.threads:
|
|
561
|
+
self._started += 1
|
|
562
|
+
threading.Thread(target=self._run, name=f"world-host-finish-{self._started}", daemon=True).start()
|
|
563
|
+
|
|
564
|
+
def _run(self) -> None:
|
|
565
|
+
while True:
|
|
566
|
+
with self._changed:
|
|
567
|
+
self._changed.wait_for(lambda: self._next() is not None)
|
|
568
|
+
version = self._next()
|
|
569
|
+
assert version is not None
|
|
570
|
+
job = self._waiting[version].popleft()
|
|
571
|
+
self._serving.add(version)
|
|
572
|
+
self._work(version, job)
|
|
573
|
+
|
|
574
|
+
def _next(self) -> str | None:
|
|
575
|
+
return next((version for version, jobs in self._waiting.items() if jobs and version not in self._serving), None)
|
|
576
|
+
|
|
577
|
+
def _work(self, version: str, job: Callable[[], None]) -> None:
|
|
578
|
+
try:
|
|
579
|
+
job()
|
|
580
|
+
except Exception as error: # noqa: BLE001 - one session's finish never stops the pool
|
|
581
|
+
log_line(f"world host: a session did not finish: {type(error).__name__}")
|
|
582
|
+
finally:
|
|
583
|
+
with self._changed:
|
|
584
|
+
self._serving.discard(version)
|
|
585
|
+
if not self._waiting[version]:
|
|
586
|
+
del self._waiting[version]
|
|
587
|
+
self._changed.notify_all()
|
|
588
|
+
|
|
589
|
+
|
|
442
590
|
class WorkerPool:
|
|
443
591
|
"""Every version worker on this replica, started on the first session of a
|
|
444
592
|
version and stopped once it has had none for a while."""
|
|
@@ -468,7 +616,7 @@ class WorkerPool:
|
|
|
468
616
|
"(WORLDS_WORKER_BUDGET_MB, WORLDS_WORKER_FOOTPRINT_MB)"
|
|
469
617
|
)
|
|
470
618
|
self.root = Path(cache_root)
|
|
471
|
-
self.workers_root = self.root /
|
|
619
|
+
self.workers_root = self.root / WORKERS_DIR
|
|
472
620
|
# Workers live and die with the supervisor; whatever a previous one
|
|
473
621
|
# left (sockets, session files) is unreachable.
|
|
474
622
|
shutil.rmtree(self.workers_root, ignore_errors=True)
|
|
@@ -491,24 +639,54 @@ class WorkerPool:
|
|
|
491
639
|
def get(self, version_id: str) -> Worker:
|
|
492
640
|
"""The live worker for a version, started if there is none. Starting one
|
|
493
641
|
past the budget first stops workers holding no sessions (whatever their
|
|
494
|
-
idle time); when every one is in use, raises :class:`WorkerCapacity`.
|
|
642
|
+
idle time); when every one is in use, raises :class:`WorkerCapacity`. A
|
|
643
|
+
dead worker that still holds sessions is never replaced here: its
|
|
644
|
+
sessions' files are on disk, and the supervisor recovers them through
|
|
645
|
+
:meth:`replace` (raises :class:`WorkerGone`)."""
|
|
495
646
|
with self._lock:
|
|
496
647
|
worker = self._workers.get(version_id)
|
|
497
648
|
if worker is not None and worker.alive():
|
|
498
649
|
return worker
|
|
650
|
+
if worker is not None and worker.sessions:
|
|
651
|
+
raise WorkerGone(f"worker for version {version_id} (pid {worker.pid}) exited holding sessions")
|
|
499
652
|
if worker is not None:
|
|
500
653
|
worker.stop()
|
|
501
654
|
del self._workers[version_id]
|
|
502
|
-
self.
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
655
|
+
return self._start(version_id)
|
|
656
|
+
|
|
657
|
+
def replace(self, dead: Worker) -> Worker:
|
|
658
|
+
"""A fresh worker in ``dead``'s place, its sessions left for the caller to
|
|
659
|
+
reopen; ``dead`` is stopped if it still runs. Refused (:class:`WorkerGone`)
|
|
660
|
+
when the pool no longer holds ``dead``."""
|
|
661
|
+
with self._lock:
|
|
662
|
+
if self._workers.get(dead.version_id) is not dead:
|
|
663
|
+
raise WorkerGone(f"worker for version {dead.version_id} (pid {dead.pid}) was replaced already")
|
|
664
|
+
del self._workers[dead.version_id]
|
|
665
|
+
dead.stop()
|
|
666
|
+
return self._start(dead.version_id)
|
|
667
|
+
|
|
668
|
+
def _start(self, version_id: str) -> Worker:
|
|
669
|
+
self._make_room()
|
|
670
|
+
worker = self._spawn(version_id)
|
|
671
|
+
self._workers[version_id] = worker
|
|
672
|
+
return worker
|
|
673
|
+
|
|
674
|
+
def dead(self) -> list[Worker]:
|
|
675
|
+
"""The workers that exited still holding sessions: theirs to recover."""
|
|
676
|
+
with self._lock:
|
|
677
|
+
return [w for w in self._workers.values() if w.sessions and not w.alive()]
|
|
678
|
+
|
|
679
|
+
def live(self) -> list[Worker]:
|
|
680
|
+
"""The workers still running, holding sessions or not."""
|
|
681
|
+
with self._lock:
|
|
682
|
+
return [w for w in self._workers.values() if w.alive()]
|
|
506
683
|
|
|
507
684
|
def _make_room(self) -> None:
|
|
508
685
|
"""Under ``_lock``: leave one worker's worth of the budget free, by
|
|
509
|
-
forgetting exited workers first, then stopping
|
|
510
|
-
longest idle first, and refusing when every live
|
|
511
|
-
|
|
686
|
+
forgetting exited workers that hold no sessions first, then stopping
|
|
687
|
+
session-less ones longest idle first, and refusing when every live
|
|
688
|
+
worker is in use."""
|
|
689
|
+
exited = [v for v, w in self._workers.items() if not w.alive() and not w.sessions]
|
|
512
690
|
for version_id in exited:
|
|
513
691
|
self._workers.pop(version_id).stop()
|
|
514
692
|
spare = sorted(
|
|
@@ -527,15 +705,6 @@ class WorkerPool:
|
|
|
527
705
|
with self._lock:
|
|
528
706
|
return self._workers.get(version_id)
|
|
529
707
|
|
|
530
|
-
def drop(self, version_id: str) -> Worker | None:
|
|
531
|
-
"""Stop and forget a version's worker; returns it (with the sessions
|
|
532
|
-
it held) so the caller can forget those too."""
|
|
533
|
-
with self._lock:
|
|
534
|
-
worker = self._workers.pop(version_id, None)
|
|
535
|
-
if worker is not None:
|
|
536
|
-
worker.stop()
|
|
537
|
-
return worker
|
|
538
|
-
|
|
539
708
|
def reap_idle(self, now: float | None = None) -> list[str]:
|
|
540
709
|
"""Stop workers that have had no session for ``idle_s``; returns their versions."""
|
|
541
710
|
now = time.time() if now is None else now
|
|
@@ -604,13 +773,8 @@ class WorkerPool:
|
|
|
604
773
|
raise WorkerGone(
|
|
605
774
|
f"worker for version {worker.version_id} exited with {worker.process.returncode} on start"
|
|
606
775
|
)
|
|
607
|
-
if worker.socket_path.exists():
|
|
608
|
-
|
|
609
|
-
status, _, _ = worker.request("GET", "/healthz", {}, timeout=2.0)
|
|
610
|
-
except WorkerGone:
|
|
611
|
-
status = 0
|
|
612
|
-
if status == 200:
|
|
613
|
-
return
|
|
776
|
+
if worker.socket_path.exists() and worker.answers_health():
|
|
777
|
+
return
|
|
614
778
|
time.sleep(0.05)
|
|
615
779
|
worker.stop()
|
|
616
780
|
raise WorkerGone(f"worker for version {worker.version_id} did not come up in {WORKER_START_TIMEOUT_S:.0f}s")
|