@withgateway/sdk 0.3.3 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (158) hide show
  1. package/README.md +4 -3
  2. package/dist/cli.js +3005 -1401
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +2996 -1392
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.js +128 -37
  7. package/dist/files/index.js.map +1 -1
  8. package/dist/files/index.mjs +126 -35
  9. package/dist/files/index.mjs.map +1 -1
  10. package/dist/hooks/index.d.mts +1 -1
  11. package/dist/hooks/index.d.ts +1 -1
  12. package/dist/hooks/index.js +124 -11
  13. package/dist/hooks/index.js.map +1 -1
  14. package/dist/hooks/index.mjs +124 -11
  15. package/dist/hooks/index.mjs.map +1 -1
  16. package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
  17. package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
  18. package/dist/{index-mJhG3vZU.d.mts → index-Di9nNqOY.d.mts} +26 -3
  19. package/dist/{index-DUHmS74p.d.ts → index-LA-PMRJT.d.ts} +26 -3
  20. package/dist/index.d.mts +4 -4
  21. package/dist/index.d.ts +4 -4
  22. package/dist/index.js +1078 -260
  23. package/dist/index.js.map +1 -1
  24. package/dist/index.mjs +1040 -251
  25. package/dist/index.mjs.map +1 -1
  26. package/dist/replay/index.d.mts +1 -1
  27. package/dist/replay/index.d.ts +1 -1
  28. package/dist/replay/index.js.map +1 -1
  29. package/dist/replay/index.mjs.map +1 -1
  30. package/dist/traces/index.js +125 -34
  31. package/dist/traces/index.js.map +1 -1
  32. package/dist/traces/index.mjs +123 -32
  33. package/dist/traces/index.mjs.map +1 -1
  34. package/dist/tracing/index.d.mts +1 -1
  35. package/dist/tracing/index.d.ts +1 -1
  36. package/dist/tracing/index.js +7 -1
  37. package/dist/tracing/index.js.map +1 -1
  38. package/dist/tracing/index.mjs +7 -1
  39. package/dist/tracing/index.mjs.map +1 -1
  40. package/dist/worlds/index.d.mts +819 -438
  41. package/dist/worlds/index.d.ts +819 -438
  42. package/dist/worlds/index.js +912 -236
  43. package/dist/worlds/index.js.map +1 -1
  44. package/dist/worlds/index.mjs +908 -232
  45. package/dist/worlds/index.mjs.map +1 -1
  46. package/package.json +1 -1
  47. package/runtime/gatewaysdk/environment/_bundle.py +4 -6
  48. package/runtime/gatewaysdk/environment/_materialize.py +5 -25
  49. package/runtime/gatewaysdk/environment/_world.py +82 -66
  50. package/runtime/gatewaysdk/environment/backing.py +20 -24
  51. package/runtime/gatewaysdk/environment/core.py +1 -16
  52. package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
  53. package/runtime/gatewaysdk/environment/runtime.py +45 -2
  54. package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
  55. package/runtime/gatewaysdk/environment/schema/api.py +198 -474
  56. package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
  57. package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
  58. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
  59. package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
  60. package/runtime/gatewaysdk/environment/schema/call_capture.py +321 -0
  61. package/runtime/gatewaysdk/environment/schema/call_log.py +276 -18
  62. package/runtime/gatewaysdk/environment/schema/call_redaction.py +207 -19
  63. package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
  64. package/runtime/gatewaysdk/environment/schema/compiler.py +4 -1
  65. package/runtime/gatewaysdk/environment/schema/conform.py +38 -35
  66. package/runtime/gatewaysdk/environment/schema/connector.py +94 -152
  67. package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
  68. package/runtime/gatewaysdk/environment/schema/connectors/custom/handlers.py +2 -1
  69. package/runtime/gatewaysdk/environment/schema/connectors/github/api/handlers.py +217 -58
  70. package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +10 -19
  71. package/runtime/gatewaysdk/environment/schema/connectors/github/scenarios/default/seed/github_webhooks.json +10 -2
  72. package/runtime/gatewaysdk/environment/schema/connectors/github/tests/webhooks.json +47 -0
  73. package/runtime/gatewaysdk/environment/schema/connectors/github/world.json +21 -0
  74. package/runtime/gatewaysdk/environment/schema/connectors/jamf/api/handlers.py +185 -18
  75. package/runtime/gatewaysdk/environment/schema/connectors/jamf/connector.toml +29 -14
  76. package/runtime/gatewaysdk/environment/schema/connectors/jamf/provenance.json +43 -12
  77. package/runtime/gatewaysdk/environment/schema/connectors/jamf/scenarios/default/seed/jamf_webhooks.json +2 -2
  78. package/runtime/gatewaysdk/environment/schema/connectors/jamf/scope.toml +4 -17
  79. package/runtime/gatewaysdk/environment/schema/connectors/jamf/tests/webhooks.json +107 -0
  80. package/runtime/gatewaysdk/environment/schema/connectors/jamf/vendor-openapi.json +1 -1
  81. package/runtime/gatewaysdk/environment/schema/connectors/linear/api/handlers.py +85 -1
  82. package/runtime/gatewaysdk/environment/schema/connectors/linear/connector.toml +19 -31
  83. package/runtime/gatewaysdk/environment/schema/connectors/linear/provenance.json +18 -8
  84. package/runtime/gatewaysdk/environment/schema/connectors/linear/scenarios/default/seed/webhooks.json +19 -3
  85. package/runtime/gatewaysdk/environment/schema/connectors/linear/tests/test_webhooks.py +71 -0
  86. package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
  87. package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
  88. package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
  89. package/runtime/gatewaysdk/environment/schema/data.py +14 -13
  90. package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
  91. package/runtime/gatewaysdk/environment/schema/database/guard.py +45 -37
  92. package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
  93. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
  94. package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
  95. package/runtime/gatewaysdk/environment/schema/door.py +899 -0
  96. package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
  97. package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
  98. package/runtime/gatewaysdk/environment/schema/handler_errors.py +52 -5
  99. package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
  100. package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
  101. package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
  102. package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
  103. package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
  104. package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
  105. package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
  106. package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
  107. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
  108. package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
  109. package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
  110. package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
  111. package/runtime/gatewaysdk/environment/schema/journal.py +438 -0
  112. package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
  113. package/runtime/gatewaysdk/environment/schema/local.py +32 -116
  114. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +20 -5
  115. package/runtime/gatewaysdk/environment/schema/platform.py +29 -234
  116. package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
  117. package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
  118. package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
  119. package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
  120. package/runtime/gatewaysdk/environment/schema/runtime.py +136 -176
  121. package/runtime/gatewaysdk/environment/schema/scaffold.py +2 -1
  122. package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
  123. package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
  124. package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
  125. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
  126. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/CONNECTORS.md +3 -3
  127. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +50 -28
  128. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
  129. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +20 -22
  130. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +25 -25
  131. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
  132. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +71 -7
  133. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
  134. package/runtime/gatewaysdk/environment/schema/snapshot.py +150 -30
  135. package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
  136. package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
  137. package/runtime/gatewaysdk/environment/schema/tools_world.py +103 -203
  138. package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
  139. package/runtime/gatewaysdk/environment/schema/verbs.py +38 -36
  140. package/runtime/gatewaysdk/environment/schema/webhooks.py +687 -0
  141. package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
  142. package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
  143. package/runtime/gatewaysdk/environment/schema/world_host.py +970 -0
  144. package/runtime/gatewaysdk/environment/schema/world_tests.py +83 -11
  145. package/runtime/gatewaysdk/environment/world.py +1 -1
  146. package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
  147. package/skills/agents/connector-template-author.md +5 -1
  148. package/skills/benchmarks-getting-started/SKILL.md +8 -9
  149. package/skills/connector-schema-authoring/CONNECTORS.md +3 -3
  150. package/skills/connector-schema-authoring/SKILL.md +50 -28
  151. package/skills/connector-schema-authoring/connector-worlds.md +282 -18
  152. package/skills/connector-schema-authoring/world-schema.md +7 -6
  153. package/skills/world-data-ingestion/ROWS.md +12 -8
  154. package/skills/world-data-ingestion/SKILL.md +20 -22
  155. package/skills/worlds-getting-started/SKILL.md +25 -25
  156. package/skills/worlds-getting-started/references/cli-mcp-map.md +24 -16
  157. package/skills/worlds-getting-started/references/gotchas.md +71 -7
  158. package/skills/worlds-getting-started/references/journal-checks.md +182 -0
@@ -15,7 +15,8 @@ import socket
15
15
  import sys
16
16
  from pathlib import Path
17
17
 
18
- from .host import WorldHost, make_host_handler
18
+ from .cache_layout import VERSIONS_DIR, WORKERS_DIR
19
+ from .host import make_host_handler
19
20
  from .workers import (
20
21
  DEFAULT_WORKER_MEMORY_MB,
21
22
  WORKER_LEASE_S,
@@ -23,6 +24,7 @@ from .workers import (
23
24
  apply_limits,
24
25
  seal_process,
25
26
  )
27
+ from .world_host import WorldHost
26
28
 
27
29
 
28
30
  def main(argv: list[str] | None = None) -> int:
@@ -39,7 +41,7 @@ def main(argv: list[str] | None = None) -> int:
39
41
  raise SystemExit("world worker: WORLDS_WORKER_SECRET is unset; a worker only answers its supervisor")
40
42
  supervisor_pid: int | None = None
41
43
  if args.socket_fd is not None:
42
- supervisor_pid = int(os.environ.get("WORLDS_SUPERVISOR_PID") or os.getppid())
44
+ supervisor_pid = os.getppid()
43
45
  listening: socket.socket | str = socket.socket(fileno=args.socket_fd)
44
46
  else:
45
47
  listening = args.socket
@@ -48,8 +50,8 @@ def main(argv: list[str] | None = None) -> int:
48
50
 
49
51
  cache_root = Path(args.cache_root)
50
52
  world_host = WorldHost(
51
- cache_root / "workers" / args.version,
52
- versions_root=cache_root / "versions",
53
+ cache_root / WORKERS_DIR / args.version,
54
+ versions_root=cache_root / VERSIONS_DIR,
53
55
  lease_s=WORKER_LEASE_S,
54
56
  run_handlers=True,
55
57
  pinned_version=args.version,
@@ -35,6 +35,7 @@ fence (a sandboxed runtime class, a read-only root) when it is wanted; see
35
35
 
36
36
  from __future__ import annotations
37
37
 
38
+ import collections
38
39
  import ctypes
39
40
  import hashlib
40
41
  import http.client
@@ -49,13 +50,16 @@ import sys
49
50
  import tempfile
50
51
  import threading
51
52
  import time
52
- from collections.abc import Mapping
53
+ from collections.abc import Callable, Iterator, Mapping
54
+ from contextlib import contextmanager, suppress
53
55
  from dataclasses import dataclass, field
54
56
  from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
55
57
  from pathlib import Path
56
58
  from typing import Protocol
57
59
 
60
+ from .cache_layout import WORKERS_DIR
58
61
  from .grading import VerifierSandbox
62
+ from .handler_errors import log_line
59
63
 
60
64
  #: Address space a worker may use, in MiB, unless ``WORLDS_WORKER_MEMORY_MB`` says otherwise.
61
65
  DEFAULT_WORKER_MEMORY_MB = 2048
@@ -81,10 +85,12 @@ FLOOR_PRECISION_BYTES = 1 << 20
81
85
  #: The largest limit a kernel reports, which ``RLIM_INFINITY`` is read as in arithmetic: macOS
82
86
  #: spells it so, Linux spells it -1.
83
87
  MAX_RLIMIT = 2**63 - 1
84
- #: How long a worker with no sessions is kept before it is stopped.
85
88
  DEFAULT_WORKER_IDLE_S = 300.0
86
- #: How long the supervisor waits for a fresh worker to answer on its socket.
87
89
  WORKER_START_TIMEOUT_S = 30.0
90
+ #: How long one health probe waits: a worker that cannot answer it in this time is not serving.
91
+ HEALTH_TIMEOUT_S = 2.0
92
+ #: The most a probe reads of a worker's answer: the worker runs tenant code, so its size is not its to choose.
93
+ PROBE_LIMIT_BYTES = 64 << 10
88
94
  #: A worker's sessions live as long as the supervisor's do; the supervisor
89
95
  #: holds the leases, so the worker's own are effectively unbounded.
90
96
  WORKER_LEASE_S = 365.0 * 86400.0
@@ -94,10 +100,20 @@ INHERITED_ENV = ("PATH", "PYTHONPATH", "LANG", "LC_ALL", "TZ", "VIRTUAL_ENV")
94
100
  #: Headers the supervisor never forwards to a worker: the ones that describe
95
101
  #: the supervisor's connection, and its own control-plane secret.
96
102
  UNFORWARDED_HEADERS = frozenset({"connection", "content-length", "transfer-encoding", "x-host-secret"})
103
+ FINISH_THREADS = 8
104
+ FINISH_SETTLE_S = 10.0
97
105
 
98
106
 
99
107
  class WorkerGone(Exception):
100
- """The worker did not answer: it exited, or never came up."""
108
+ """The worker did not answer: it exited, never came up, or dropped the request."""
109
+
110
+
111
+ class WorkerTimeout(WorkerGone):
112
+ """The worker did not answer in time; it may be alive and still working on the request."""
113
+
114
+
115
+ class WorkerUnreached(WorkerGone):
116
+ """The worker took no connection: the request never reached it."""
101
117
 
102
118
 
103
119
  class WorkerCapacity(Exception):
@@ -117,6 +133,24 @@ class UnixHTTPConnection(http.client.HTTPConnection):
117
133
  self.sock = sock
118
134
 
119
135
 
136
+ def _hang_up(sock: socket.socket, cut: threading.Event) -> None:
137
+ cut.set()
138
+ with suppress(OSError):
139
+ sock.shutdown(socket.SHUT_RDWR)
140
+
141
+
142
+ @contextmanager
143
+ def _deadline(sock: socket.socket, seconds: float) -> Iterator[threading.Event]:
144
+ """Shut ``sock`` once ``seconds`` pass, so nothing read from it waits longer; the event says it was."""
145
+ cut = threading.Event()
146
+ timer = threading.Timer(seconds, _hang_up, (sock, cut))
147
+ timer.start()
148
+ try:
149
+ yield cut
150
+ finally:
151
+ timer.cancel()
152
+
153
+
120
154
  def bind_worker_socket(path: str | Path) -> socket.socket:
121
155
  """The supervisor's end of a worker: a listening unix socket at ``path``,
122
156
  ``0600`` inside a ``0700`` directory, ready to hand to the worker."""
@@ -333,7 +367,6 @@ def worker_environment(
333
367
  WORLDS_WORKER_SECRET=secret,
334
368
  WORLDS_WORKER_MEMORY_MB=str(memory_mb),
335
369
  WORLDS_INGRESS_DOMAIN=worlds_domain,
336
- WORLDS_SUPERVISOR_PID=str(os.getpid()),
337
370
  )
338
371
  return env
339
372
 
@@ -389,7 +422,6 @@ class Worker:
389
422
  socket_path: Path
390
423
  secret: str
391
424
  sessions: set[str] = field(default_factory=set)
392
- #: When the worker last had a session, for idle reaping.
393
425
  idle_since: float = field(default_factory=time.time)
394
426
 
395
427
  @property
@@ -408,25 +440,78 @@ class Worker:
408
440
  *,
409
441
  control: bool = False,
410
442
  timeout: float = 60.0,
443
+ limit: int | None = None,
411
444
  ) -> tuple[int, list[tuple[str, str]], bytes]:
412
445
  """One request over the socket. ``control`` adds the worker's own
413
- secret, so the worker's control plane answers. A worker that cannot be
414
- reached raises :class:`WorkerGone`."""
446
+ secret, so the worker's control plane answers. A worker that takes no connection raises
447
+ :class:`WorkerUnreached`; one that drops the request or answers past ``limit`` bytes raises
448
+ :class:`WorkerGone`. The whole exchange gets ``timeout`` seconds, not each read: an answer
449
+ sent slowly is cut off when they pass (the connection is shut, not waited out) and raises
450
+ :class:`WorkerTimeout`."""
415
451
  forward = {k: v for k, v in headers.items() if k.lower() not in UNFORWARDED_HEADERS}
416
452
  if control:
417
453
  forward["X-Host-Secret"] = self.secret
418
- conn = UnixHTTPConnection(self.socket_path, timeout=timeout)
454
+ started = time.monotonic()
455
+ conn = self._connected(timeout)
456
+ cut = threading.Event()
419
457
  try:
420
- conn.request(method, path, body=body or None, headers=forward)
421
- answer = conn.getresponse()
422
- payload = answer.read()
458
+ with _deadline(conn.sock, timeout - (time.monotonic() - started)) as cut:
459
+ conn.request(method, path, body=body or None, headers=forward)
460
+ answer = conn.getresponse()
461
+ payload = answer.read() if limit is None else answer.read(limit + 1)
462
+ if cut.is_set():
463
+ raise WorkerTimeout(
464
+ f"worker for version {self.version_id} (pid {self.pid}) did not answer in {timeout:g} s"
465
+ )
466
+ if limit is not None and len(payload) > limit:
467
+ raise WorkerGone(f"worker for version {self.version_id} (pid {self.pid}) answered past {limit} bytes")
423
468
  return answer.status, answer.getheaders(), payload
424
469
  except (OSError, http.client.HTTPException) as error:
425
- raise WorkerGone(
426
- f"worker for version {self.version_id} (pid {self.pid}) did not answer: {error}"
470
+ gone = WorkerTimeout if cut.is_set() or isinstance(error, TimeoutError) else WorkerGone
471
+ raise gone(f"worker for version {self.version_id} (pid {self.pid}) did not answer: {error}") from error
472
+ finally:
473
+ conn.close()
474
+
475
+ def _connected(self, timeout: float) -> UnixHTTPConnection:
476
+ conn = UnixHTTPConnection(self.socket_path, timeout=timeout)
477
+ try:
478
+ conn.connect()
479
+ except OSError as error:
480
+ conn.close()
481
+ raise WorkerUnreached(
482
+ f"worker for version {self.version_id} (pid {self.pid}) took no connection: {error}"
427
483
  ) from error
484
+ return conn
485
+
486
+ def probe(
487
+ self, path: str, *, deadline_s: float = HEALTH_TIMEOUT_S, limit: int = PROBE_LIMIT_BYTES
488
+ ) -> tuple[int, bytes]:
489
+ """One control GET whose whole answer must arrive within ``deadline_s`` and ``limit`` bytes.
490
+ The worker runs tenant code: an answer sent slowly or without end is cut off when the
491
+ deadline passes (the connection is shut, not waited out), and raises :class:`WorkerGone`."""
492
+ unanswered = f"worker for version {self.version_id} (pid {self.pid}) did not answer {path}"
493
+ conn = UnixHTTPConnection(self.socket_path, timeout=deadline_s)
494
+ try:
495
+ conn.connect()
496
+ with _deadline(conn.sock, deadline_s) as cut:
497
+ conn.request("GET", path, headers={"X-Host-Secret": self.secret})
498
+ answer = conn.getresponse()
499
+ payload = answer.read(limit + 1)
500
+ except (OSError, http.client.HTTPException) as error:
501
+ raise WorkerGone(unanswered) from error
428
502
  finally:
429
503
  conn.close()
504
+ if cut.is_set() or len(payload) > limit:
505
+ raise WorkerGone(f"{unanswered} within {deadline_s:g} s and {limit} bytes")
506
+ return answer.status, payload
507
+
508
+ def answers_health(self, timeout_s: float = HEALTH_TIMEOUT_S) -> bool:
509
+ """Whether the worker is alive and answers its health check, whole, in time."""
510
+ try:
511
+ status, _ = self.probe("/healthz", deadline_s=timeout_s)
512
+ except WorkerGone:
513
+ return False
514
+ return status == 200
430
515
 
431
516
  def stop(self, grace_s: float = 5.0) -> None:
432
517
  if self.alive():
@@ -439,6 +524,69 @@ class Worker:
439
524
  self.socket_path.unlink(missing_ok=True)
440
525
 
441
526
 
527
+ class Finishing:
528
+ """What leaves a session on a version worker, finished off the lease sweeper's thread: a few daemon
529
+ threads, one job at a time for a version in the order they came (so a worker that never answers
530
+ holds one thread, not the pool). A session with no worker is finished at once, where it is."""
531
+
532
+ def __init__(self, threads: int = FINISH_THREADS) -> None:
533
+ self.threads = threads
534
+ self._changed = threading.Condition()
535
+ self._waiting: dict[str, collections.deque[Callable[[], None]]] = {}
536
+ self._serving: set[str] = set()
537
+ self._started = 0
538
+
539
+ def leave(self, worker: Worker | None, job: Callable[[], None]) -> None:
540
+ if worker is None:
541
+ job()
542
+ return
543
+ with self._changed:
544
+ self._waiting.setdefault(worker.version_id, collections.deque()).append(job)
545
+ self._staff()
546
+ self._changed.notify()
547
+
548
+ def settle(self, deadline_s: float = FINISH_SETTLE_S) -> bool:
549
+ """Wait up to ``deadline_s`` for every job handed off to be done; whether they are."""
550
+ until = time.monotonic() + deadline_s
551
+ with self._changed:
552
+ while self._waiting:
553
+ remaining = until - time.monotonic()
554
+ if remaining <= 0:
555
+ return False
556
+ self._changed.wait(remaining)
557
+ return True
558
+
559
+ def _staff(self) -> None:
560
+ if self._started < self.threads:
561
+ self._started += 1
562
+ threading.Thread(target=self._run, name=f"world-host-finish-{self._started}", daemon=True).start()
563
+
564
+ def _run(self) -> None:
565
+ while True:
566
+ with self._changed:
567
+ self._changed.wait_for(lambda: self._next() is not None)
568
+ version = self._next()
569
+ assert version is not None
570
+ job = self._waiting[version].popleft()
571
+ self._serving.add(version)
572
+ self._work(version, job)
573
+
574
+ def _next(self) -> str | None:
575
+ return next((version for version, jobs in self._waiting.items() if jobs and version not in self._serving), None)
576
+
577
+ def _work(self, version: str, job: Callable[[], None]) -> None:
578
+ try:
579
+ job()
580
+ except Exception as error: # noqa: BLE001 - one session's finish never stops the pool
581
+ log_line(f"world host: a session did not finish: {type(error).__name__}")
582
+ finally:
583
+ with self._changed:
584
+ self._serving.discard(version)
585
+ if not self._waiting[version]:
586
+ del self._waiting[version]
587
+ self._changed.notify_all()
588
+
589
+
442
590
  class WorkerPool:
443
591
  """Every version worker on this replica, started on the first session of a
444
592
  version and stopped once it has had none for a while."""
@@ -468,7 +616,7 @@ class WorkerPool:
468
616
  "(WORLDS_WORKER_BUDGET_MB, WORLDS_WORKER_FOOTPRINT_MB)"
469
617
  )
470
618
  self.root = Path(cache_root)
471
- self.workers_root = self.root / "workers"
619
+ self.workers_root = self.root / WORKERS_DIR
472
620
  # Workers live and die with the supervisor; whatever a previous one
473
621
  # left (sockets, session files) is unreachable.
474
622
  shutil.rmtree(self.workers_root, ignore_errors=True)
@@ -491,24 +639,54 @@ class WorkerPool:
491
639
  def get(self, version_id: str) -> Worker:
492
640
  """The live worker for a version, started if there is none. Starting one
493
641
  past the budget first stops workers holding no sessions (whatever their
494
- idle time); when every one is in use, raises :class:`WorkerCapacity`."""
642
+ idle time); when every one is in use, raises :class:`WorkerCapacity`. A
643
+ dead worker that still holds sessions is never replaced here: its
644
+ sessions' files are on disk, and the supervisor recovers them through
645
+ :meth:`replace` (raises :class:`WorkerGone`)."""
495
646
  with self._lock:
496
647
  worker = self._workers.get(version_id)
497
648
  if worker is not None and worker.alive():
498
649
  return worker
650
+ if worker is not None and worker.sessions:
651
+ raise WorkerGone(f"worker for version {version_id} (pid {worker.pid}) exited holding sessions")
499
652
  if worker is not None:
500
653
  worker.stop()
501
654
  del self._workers[version_id]
502
- self._make_room()
503
- worker = self._spawn(version_id)
504
- self._workers[version_id] = worker
505
- return worker
655
+ return self._start(version_id)
656
+
657
+ def replace(self, dead: Worker) -> Worker:
658
+ """A fresh worker in ``dead``'s place, its sessions left for the caller to
659
+ reopen; ``dead`` is stopped if it still runs. Refused (:class:`WorkerGone`)
660
+ when the pool no longer holds ``dead``."""
661
+ with self._lock:
662
+ if self._workers.get(dead.version_id) is not dead:
663
+ raise WorkerGone(f"worker for version {dead.version_id} (pid {dead.pid}) was replaced already")
664
+ del self._workers[dead.version_id]
665
+ dead.stop()
666
+ return self._start(dead.version_id)
667
+
668
+ def _start(self, version_id: str) -> Worker:
669
+ self._make_room()
670
+ worker = self._spawn(version_id)
671
+ self._workers[version_id] = worker
672
+ return worker
673
+
674
+ def dead(self) -> list[Worker]:
675
+ """The workers that exited still holding sessions: theirs to recover."""
676
+ with self._lock:
677
+ return [w for w in self._workers.values() if w.sessions and not w.alive()]
678
+
679
+ def live(self) -> list[Worker]:
680
+ """The workers still running, holding sessions or not."""
681
+ with self._lock:
682
+ return [w for w in self._workers.values() if w.alive()]
506
683
 
507
684
  def _make_room(self) -> None:
508
685
  """Under ``_lock``: leave one worker's worth of the budget free, by
509
- forgetting exited workers first, then stopping session-less ones
510
- longest idle first, and refusing when every live worker is in use."""
511
- exited = [v for v, w in self._workers.items() if not w.alive()]
686
+ forgetting exited workers that hold no sessions first, then stopping
687
+ session-less ones longest idle first, and refusing when every live
688
+ worker is in use."""
689
+ exited = [v for v, w in self._workers.items() if not w.alive() and not w.sessions]
512
690
  for version_id in exited:
513
691
  self._workers.pop(version_id).stop()
514
692
  spare = sorted(
@@ -527,15 +705,6 @@ class WorkerPool:
527
705
  with self._lock:
528
706
  return self._workers.get(version_id)
529
707
 
530
- def drop(self, version_id: str) -> Worker | None:
531
- """Stop and forget a version's worker; returns it (with the sessions
532
- it held) so the caller can forget those too."""
533
- with self._lock:
534
- worker = self._workers.pop(version_id, None)
535
- if worker is not None:
536
- worker.stop()
537
- return worker
538
-
539
708
  def reap_idle(self, now: float | None = None) -> list[str]:
540
709
  """Stop workers that have had no session for ``idle_s``; returns their versions."""
541
710
  now = time.time() if now is None else now
@@ -604,13 +773,8 @@ class WorkerPool:
604
773
  raise WorkerGone(
605
774
  f"worker for version {worker.version_id} exited with {worker.process.returncode} on start"
606
775
  )
607
- if worker.socket_path.exists():
608
- try:
609
- status, _, _ = worker.request("GET", "/healthz", {}, timeout=2.0)
610
- except WorkerGone:
611
- status = 0
612
- if status == 200:
613
- return
776
+ if worker.socket_path.exists() and worker.answers_health():
777
+ return
614
778
  time.sleep(0.05)
615
779
  worker.stop()
616
780
  raise WorkerGone(f"worker for version {worker.version_id} did not come up in {WORKER_START_TIMEOUT_S:.0f}s")