@withgateway/sdk 0.3.3 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (123) hide show
  1. package/README.md +4 -3
  2. package/dist/cli.js +2221 -962
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +2212 -953
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.js +81 -19
  7. package/dist/files/index.js.map +1 -1
  8. package/dist/files/index.mjs +79 -17
  9. package/dist/files/index.mjs.map +1 -1
  10. package/dist/hooks/index.d.mts +1 -1
  11. package/dist/hooks/index.d.ts +1 -1
  12. package/dist/hooks/index.js +124 -11
  13. package/dist/hooks/index.js.map +1 -1
  14. package/dist/hooks/index.mjs +124 -11
  15. package/dist/hooks/index.mjs.map +1 -1
  16. package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
  17. package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
  18. package/dist/index.d.mts +2 -2
  19. package/dist/index.d.ts +2 -2
  20. package/dist/index.js +903 -221
  21. package/dist/index.js.map +1 -1
  22. package/dist/index.mjs +868 -212
  23. package/dist/index.mjs.map +1 -1
  24. package/dist/traces/index.js +78 -16
  25. package/dist/traces/index.js.map +1 -1
  26. package/dist/traces/index.mjs +76 -14
  27. package/dist/traces/index.mjs.map +1 -1
  28. package/dist/tracing/index.js +7 -1
  29. package/dist/tracing/index.js.map +1 -1
  30. package/dist/tracing/index.mjs +7 -1
  31. package/dist/tracing/index.mjs.map +1 -1
  32. package/dist/worlds/index.d.mts +692 -410
  33. package/dist/worlds/index.d.ts +692 -410
  34. package/dist/worlds/index.js +740 -197
  35. package/dist/worlds/index.js.map +1 -1
  36. package/dist/worlds/index.mjs +736 -193
  37. package/dist/worlds/index.mjs.map +1 -1
  38. package/package.json +1 -1
  39. package/runtime/gatewaysdk/environment/_materialize.py +5 -25
  40. package/runtime/gatewaysdk/environment/_world.py +58 -65
  41. package/runtime/gatewaysdk/environment/backing.py +20 -24
  42. package/runtime/gatewaysdk/environment/core.py +1 -16
  43. package/runtime/gatewaysdk/environment/legacy_sqlite.py +14 -27
  44. package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
  45. package/runtime/gatewaysdk/environment/schema/api.py +174 -471
  46. package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
  47. package/runtime/gatewaysdk/environment/schema/batch.py +3 -3
  48. package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
  49. package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
  50. package/runtime/gatewaysdk/environment/schema/call_log.py +230 -18
  51. package/runtime/gatewaysdk/environment/schema/call_redaction.py +201 -19
  52. package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
  53. package/runtime/gatewaysdk/environment/schema/conform.py +6 -27
  54. package/runtime/gatewaysdk/environment/schema/connector.py +89 -152
  55. package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
  56. package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
  57. package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
  58. package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
  59. package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
  60. package/runtime/gatewaysdk/environment/schema/data.py +14 -13
  61. package/runtime/gatewaysdk/environment/schema/database/base.py +106 -165
  62. package/runtime/gatewaysdk/environment/schema/database/guard.py +43 -36
  63. package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
  64. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +57 -31
  65. package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
  66. package/runtime/gatewaysdk/environment/schema/door.py +899 -0
  67. package/runtime/gatewaysdk/environment/schema/envelope.py +86 -28
  68. package/runtime/gatewaysdk/environment/schema/grading.py +58 -86
  69. package/runtime/gatewaysdk/environment/schema/handler_errors.py +25 -4
  70. package/runtime/gatewaysdk/environment/schema/host.py +229 -1695
  71. package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
  72. package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
  73. package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
  74. package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
  75. package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
  76. package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
  77. package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
  78. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
  79. package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
  80. package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
  81. package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
  82. package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
  83. package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
  84. package/runtime/gatewaysdk/environment/schema/local.py +31 -116
  85. package/runtime/gatewaysdk/environment/schema/platform.py +15 -17
  86. package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
  87. package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
  88. package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
  89. package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
  90. package/runtime/gatewaysdk/environment/schema/runtime.py +129 -176
  91. package/runtime/gatewaysdk/environment/schema/session_files.py +133 -24
  92. package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
  93. package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
  94. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +8 -9
  95. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +31 -6
  96. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +12 -8
  97. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +13 -13
  98. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +23 -23
  99. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
  100. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +35 -2
  101. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
  102. package/runtime/gatewaysdk/environment/schema/snapshot.py +149 -29
  103. package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
  104. package/runtime/gatewaysdk/environment/schema/tasks.py +229 -323
  105. package/runtime/gatewaysdk/environment/schema/tools_world.py +84 -203
  106. package/runtime/gatewaysdk/environment/schema/validation.py +69 -1
  107. package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
  108. package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
  109. package/runtime/gatewaysdk/environment/schema/workers.py +203 -39
  110. package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
  111. package/runtime/gatewaysdk/environment/schema/world_tests.py +9 -11
  112. package/runtime/gatewaysdk/environment/world.py +1 -1
  113. package/runtime/gatewaysdk/environment/world_session_contract.py +14 -0
  114. package/skills/agents/connector-template-author.md +5 -1
  115. package/skills/benchmarks-getting-started/SKILL.md +8 -9
  116. package/skills/connector-schema-authoring/SKILL.md +31 -6
  117. package/skills/connector-schema-authoring/connector-worlds.md +129 -7
  118. package/skills/world-data-ingestion/ROWS.md +12 -8
  119. package/skills/world-data-ingestion/SKILL.md +13 -13
  120. package/skills/worlds-getting-started/SKILL.md +23 -23
  121. package/skills/worlds-getting-started/references/cli-mcp-map.md +11 -6
  122. package/skills/worlds-getting-started/references/gotchas.md +35 -2
  123. package/skills/worlds-getting-started/references/journal-checks.md +182 -0
@@ -0,0 +1,577 @@
1
+ """The supervisor's half of its version workers (the worker's half is :mod:`worker`).
2
+
3
+ A session of a world that runs its own code lives in its version's worker; the
4
+ supervisor keeps the session's record and forwards its requests there. What a
5
+ forwarded request answers when the worker does not answer it:
6
+
7
+ * past its time, while the worker still answers its health check: that request's
8
+ ``call_timeout`` (504); the worker and every session stay, and the session's next
9
+ requests answer ``call_running`` (503) while that call still holds its turn;
10
+ * dropped, while the worker still answers: ``busy`` (503, retry);
11
+ * the worker died or stopped answering its health check: ``worker_exited`` (502), at
12
+ once. The worker is replaced behind that answer and each of its sessions reopens on
13
+ the new one from what it had committed: the session's directory (its store, its
14
+ dependency worlds' and its call journal) is set aside before the new worker starts,
15
+ and the new worker continues the session from it
16
+ (:class:`session_files.RecoveredSession`); the session's next requests wait for
17
+ that. A worker that died idle is replaced the same way by the next request or the
18
+ lease sweeper, which also recycles any worker holding a turn past ``STUCK_TURN_S``,
19
+ its session still open here or not. What a worker answers about its turns is read
20
+ as untrusted: it runs tenant code.
21
+
22
+ Recovery never raises: a session that cannot be reopened leaves the registry (the
23
+ platform reads it as lost and reopens it from its last export), never left on a dead
24
+ worker; one the new worker may still have opened is deleted there before its id opens again.
25
+ """
26
+
27
+ from __future__ import annotations
28
+
29
+ import json
30
+ import math
31
+ import os
32
+ import shutil
33
+ import threading
34
+ import time
35
+ import urllib.parse
36
+ from collections.abc import Callable
37
+ from pathlib import Path
38
+ from typing import TYPE_CHECKING, Any, Protocol
39
+
40
+ from .admission import BUSY, RETRY_AFTER_S
41
+ from .call_log import CallJournal
42
+ from .handler_errors import log_line
43
+ from .lanes import Lanes
44
+ from .request_ledger import DRAIN_ANSWER_BYTES, LEFT_JOURNAL_BYTES
45
+ from .session_files import LEDGER_JOURNAL, RECOVERED_DIR, ExportFile, clear, real_dir
46
+ from .validation import ContractError, fail
47
+ from .workers import HEALTH_TIMEOUT_S, Worker, WorkerCapacity, WorkerGone, WorkerPool, WorkerTimeout, WorkerUnreached
48
+
49
+ if TYPE_CHECKING:
50
+ from .host_session import Session
51
+
52
+ #: The platform treats exactly this code (with a 404) as "the host lost the session".
53
+ UNKNOWN_SESSION = "unknown_session"
54
+ CALL_TIMEOUT = "call_timeout"
55
+ CALL_RUNNING = "call_running"
56
+ WORKER_EXITED = "worker_exited"
57
+ MOVING = "moving"
58
+ LOST = "lost"
59
+ #: A handoff export of a session this host is not moving: only its holder hands a session off.
60
+ NOT_MOVING = "not_moving"
61
+ #: A forwarded call's time on the worker: with a queue wait of up to ``admission.QUEUE_WAIT_S`` the host
62
+ #: answers before the platform's 60 s control timeout, so a slow handler is named, not read as a dead host.
63
+ CALL_TIMEOUT_S = 45.0
64
+ OPEN_TIMEOUT_S = 60.0
65
+ #: Under the platform's 330 s transfer timeout: an export copies a whole store.
66
+ EXPORT_TIMEOUT_S = 300.0
67
+ #: A turn held this long serves no request the supervisor still waits on: its worker is recycled.
68
+ STUCK_TURN_S = 3 * CALL_TIMEOUT_S
69
+ RUNNING_RETRY_AFTER_S = 5
70
+ JSON = {"content-type": "application/json"}
71
+ #: An open refused ``draining`` or ``at_capacity`` (503) goes to another replica; any other 503, ``busy``
72
+ #: for an id still closing included, sends the platform back later; 404 is the one answer it reads as a loss.
73
+ REFUSALS = {
74
+ "draining": 503,
75
+ "at_capacity": 503,
76
+ BUSY: 503,
77
+ CALL_RUNNING: 503,
78
+ MOVING: 503,
79
+ NOT_MOVING: 409,
80
+ UNKNOWN_SESSION: 404,
81
+ CALL_TIMEOUT: 504,
82
+ WORKER_EXITED: 502,
83
+ }
84
+
85
+
86
+ def refusal(error: ContractError) -> tuple[int, dict, dict[str, str]]:
87
+ """A refused session request as the host answers it: status, body and headers."""
88
+ detail = error.detail
89
+ code = str(detail.get("code", "host"))
90
+ retry = detail.get("retryAfter")
91
+ body = {"error": code, "detail": str(error), "message": str(detail.get("message", ""))}
92
+ if retry is None:
93
+ return REFUSALS.get(code, 400), body, {}
94
+ return REFUSALS.get(code, 400), {**body, "retryAfter": retry}, {"Retry-After": str(retry)}
95
+
96
+
97
+ def _age(value: object) -> float | None:
98
+ """A turn's age as a worker reported it, or None unless it is a finite, non-negative number."""
99
+ if isinstance(value, bool) or not isinstance(value, (int, float)):
100
+ return None
101
+ try:
102
+ age = float(value)
103
+ except OverflowError:
104
+ return None
105
+ return age if math.isfinite(age) and age >= 0 else None
106
+
107
+
108
+ class SessionRegistry(Protocol):
109
+ """What the supervisor's registry does for :class:`VersionWorkers`."""
110
+
111
+ def recovering(self, worker: Worker) -> tuple[list[Session], list[Session]]: ...
112
+
113
+ def given_back(self, session: Session) -> None: ...
114
+
115
+ def worker_died(self, session: Session, entries: list[dict]) -> None: ...
116
+
117
+ def forget_lost(self, session: Session) -> None: ...
118
+
119
+ def close(self, session_id: str, reason: str = "closed") -> dict: ...
120
+
121
+ def tally(self, key: str) -> None: ...
122
+
123
+
124
+ class VersionWorkers:
125
+ """Placement on, requests to and recovery of this supervisor's version workers."""
126
+
127
+ def __init__(self, pool: WorkerPool, registry: SessionRegistry) -> None:
128
+ self.pool = pool
129
+ self.registry = registry
130
+ self._claims = threading.Lock()
131
+ self._recoveries = Lanes()
132
+ self._overdue: dict[tuple[int, str], Worker] = {}
133
+
134
+ def home(self, version_id: str) -> Path:
135
+ """Where a version's worker runs: its own ``WorldHost`` cache root (:mod:`worker`)."""
136
+ return self.pool.workers_root / version_id
137
+
138
+ def session_dir(self, version_id: str, session_id: str) -> Path:
139
+ return self.home(version_id) / "sessions" / session_id
140
+
141
+ def place(
142
+ self, version_id: str, session_id: str, body: dict, fetch: Callable[[Path], ExportFile | None], opens: Lanes
143
+ ) -> Worker:
144
+ """Open ``session_id`` on its version's worker with ``body``, claimed before placement so
145
+ another open cannot stop the worker as session-less meanwhile. ``fetch`` downloads an
146
+ export to restore into the session's directory once the worker runs (a worker clears
147
+ that directory when it starts); the worker reads it by path. One open per version at a
148
+ time (``opens``), so the worker's open timeout counts one open, never a queue."""
149
+ worker = self._worker_for(version_id)
150
+ self._claim(worker, session_id)
151
+ source = None
152
+ try:
153
+ source = fetch(self.session_dir(version_id, session_id))
154
+ placement = {**body, "restore": source.for_worker() if source is not None else None}
155
+ self._open_on(worker, session_id, placement, opens)
156
+ except BaseException:
157
+ self._release(worker, session_id)
158
+ raise
159
+ finally:
160
+ if source is not None:
161
+ source.discard()
162
+ return worker
163
+
164
+ def _open_on(self, worker: Worker, session_id: str, placement: dict, opens: Lanes) -> None:
165
+ """The open on ``worker``, in its version's turn. One that raised after reaching the worker may still
166
+ complete there (an answer past the cut-off) or have left a copy, which no open of the id may find
167
+ later, so it is deleted there, an open still running abandoned, before the caller lets the claim go.
168
+ One that never reached it (the refusal's cause a :class:`WorkerUnreached`) left nothing to delete."""
169
+ try:
170
+ with opens.turn(worker.version_id):
171
+ self._control(worker, "POST", "/__host/sessions", placement, OPEN_TIMEOUT_S, "the open")
172
+ except Exception as error:
173
+ if not isinstance(error.__cause__, WorkerUnreached):
174
+ self._delete(worker, session_id)
175
+ raise
176
+
177
+ def control(
178
+ self, session: Session, method: str, verb: str, body: dict | None, *, timeout: float = CALL_TIMEOUT_S
179
+ ) -> dict:
180
+ """One session verb on the session's worker, answered as this host would answer it."""
181
+ what = f"tool '{body.get('tool')}'" if verb == "call" and body else verb
182
+ worker = self._serving(session)
183
+ path = f"/__host/sessions/{session.session_id}/{verb}"
184
+ return self._control(worker, method, path, body, timeout, what, session.session_id)
185
+
186
+ def outbox(self, session: Session, page: dict, timeout: float = CALL_TIMEOUT_S) -> dict:
187
+ """A page of the session's request ledger outbox on its worker, asked even while an
188
+ overdue call holds the session's turn there (the outbox takes no turn), and read whole
189
+ within ``timeout``. A dead worker is refused, not recovered here: the drain keeps the
190
+ time it was given, and the reaper's recovery hands the session's journal over."""
191
+ path = f"/__host/sessions/{session.session_id}/requests"
192
+ worker = session.worker
193
+ assert worker is not None
194
+ if not worker.alive():
195
+ fail(WORKER_EXITED, f"the worker of session {session.session_id} exited; recovery hands its entries on")
196
+ return self._control(worker, "POST", path, page, timeout, "requests", session.session_id, DRAIN_ANSWER_BYTES)
197
+
198
+ def relay(self, session: Session, method: str, path: str, headers: dict[str, str], raw: bytes):
199
+ """A data-plane request relayed as it came; the worker's own answer, whatever its status."""
200
+ what = f"{method} {urllib.parse.urlsplit(path).path}"
201
+ worker = self._serving(session)
202
+ return self._forward(worker, method, path, headers, raw, False, CALL_TIMEOUT_S, what, session.session_id)
203
+
204
+ def holds_turn(self, session: Session) -> bool:
205
+ """Whether a call of the session past its time may still hold the session's turn on its worker:
206
+ one marked overdue that the worker does not say has ended. A worker that does not say holds it."""
207
+ worker = session.worker
208
+ if worker is None or not self._overdue_on(worker, session.session_id):
209
+ return False
210
+ turns = self._probe_turns(worker)
211
+ return turns is None or session.session_id in turns
212
+
213
+ def _overdue_on(self, worker: Worker, session_id: str) -> bool:
214
+ with self._claims:
215
+ return (id(worker), session_id) in self._overdue
216
+
217
+ def live(self, session: Session) -> Worker:
218
+ """The session's worker, replaced first (and the session reopened on the new one) if it
219
+ died; a session no live worker holds afterwards is forgotten and answers ``unknown_session``.
220
+ A session that began closing meanwhile is served like any other: whoever then deletes it
221
+ on the new worker has drained the entries it resumed first."""
222
+ worker = session.worker
223
+ assert worker is not None
224
+ if worker.alive():
225
+ return worker
226
+ self.recover(worker)
227
+ current = session.worker
228
+ if current is None or not current.alive():
229
+ self.registry.forget_lost(session)
230
+ fail(UNKNOWN_SESSION, f"no session {session.session_id} on this host")
231
+ return current
232
+
233
+ def finish(self, session: Session) -> None:
234
+ """Close a session that left the registry on its worker; a worker that is gone has closed it,
235
+ and what its journal kept of the session is handed over to the ledger."""
236
+ worker = session.worker
237
+ assert worker is not None
238
+ self._release(worker, session.session_id)
239
+ if not worker.alive():
240
+ return self._hand_over(session, self.session_dir(session.version_id, session.session_id))
241
+ self._delete(worker, session.session_id)
242
+
243
+ def _delete(self, worker: Worker, session_id: str) -> None:
244
+ """``session_id`` deleted on ``worker`` while it runs; an open of it still running there is abandoned.
245
+ One the worker answers late still lands. One it never received would leave its copy holding an id this
246
+ host frees, so it is sent once more if the worker answers its health check, and the worker is recycled
247
+ when that one does not reach it either."""
248
+ if self._delivered(worker, session_id):
249
+ return
250
+ if worker.answers_health() and self._delivered(worker, session_id):
251
+ return
252
+ self._recycle(worker)
253
+
254
+ @staticmethod
255
+ def _delivered(worker: Worker, session_id: str) -> bool:
256
+ """Whether a ``DELETE`` of ``session_id`` reached ``worker``, or there is no live worker to reach."""
257
+ if not worker.alive():
258
+ return True
259
+ try:
260
+ worker.request("DELETE", f"/__host/sessions/{session_id}", JSON, control=True, timeout=CALL_TIMEOUT_S)
261
+ except WorkerUnreached:
262
+ return False
263
+ except WorkerGone:
264
+ return True
265
+ return True
266
+
267
+ def export_path(self, session: Session, answer: dict) -> Path:
268
+ """The file a worker's export landed in, admitted only from inside that worker's
269
+ directory for this session: the worker runs tenant code, so what it names is
270
+ checked, and the bytes are hashed by the supervisor, not trusted."""
271
+ exports = self.session_dir(session.version_id, session.session_id) / "exports"
272
+ path = Path(str(answer.get("path") or ""))
273
+ try:
274
+ resolved = path.resolve(strict=True)
275
+ resolved.relative_to(exports.resolve())
276
+ except (OSError, ValueError):
277
+ fail("host", f"the worker's export is not where a worker's exports go: {path}")
278
+ return resolved
279
+
280
+ def recover(self, dead: Worker, reason: str = "workers_replaced") -> None:
281
+ """Replace ``dead`` (stopped first if it still runs) and reopen every session it held on
282
+ the new worker from what each committed; a session that cannot be reopened leaves the
283
+ registry (:meth:`_reopen_failed`). One recovery per version at a time; a later caller finds
284
+ it done. Never raises. Every session it takes is its own until it gives the session back
285
+ (:meth:`_give_back`)."""
286
+ held: list[Session] = []
287
+ try:
288
+ with self._recoveries.turn(dead.version_id):
289
+ if self.pool.peek(dead.version_id) is not dead:
290
+ return
291
+ self._stop(dead)
292
+ sessions, closing = self.registry.recovering(dead)
293
+ held = [*sessions, *closing]
294
+ aside = self._set_aside(dead.version_id, held)
295
+ fresh = self._replacement(dead)
296
+ for session in closing:
297
+ self._hand_over(session, aside.get(session.session_id))
298
+ for session in sessions:
299
+ self._reopen(fresh, session, aside.get(session.session_id))
300
+ self._forget_overdue(lambda worker: worker is dead)
301
+ self.registry.tally(reason)
302
+ finally:
303
+ self._give_back(held)
304
+
305
+ def _give_back(self, sessions: list[Session]) -> None:
306
+ """Each session a recovery took, given back once the version's turn is; one that closed meanwhile
307
+ is drained and deleted on its new worker behind :class:`Finishing`, at tenant code's pace, holding
308
+ neither the version's next recovery nor a sweep. Each fails alone."""
309
+ for session in sessions:
310
+ try:
311
+ self.registry.given_back(session)
312
+ except Exception as error: # noqa: BLE001 - recovery never raises
313
+ log_line(f"session {session.session_id} did not leave its new worker: {type(error).__name__}")
314
+
315
+ def recover_dead(self) -> list[str]:
316
+ """Recover every worker that died holding sessions; returns their versions."""
317
+ dead = self.pool.dead()
318
+ for worker in dead:
319
+ self.recover(worker)
320
+ return [worker.version_id for worker in dead]
321
+
322
+ def recover_stuck(self) -> list[str]:
323
+ """Recycle every live worker holding a turn past ``STUCK_TURN_S`` (a handler that never
324
+ returned), whether or not that turn's session is still open here: its sessions reopen
325
+ from what they committed, and an id it still held opens again. Returns their versions."""
326
+ self._forget_overdue(lambda worker: not worker.alive())
327
+ stuck = [worker for worker in self.pool.live() if max(self._turns(worker).values(), default=0) > STUCK_TURN_S]
328
+ for worker in stuck:
329
+ self.recover(worker, "workers_recycled")
330
+ return [worker.version_id for worker in stuck]
331
+
332
+ def _serving(self, session: Session) -> Worker:
333
+ """The session's live worker, refused ``unknown_session`` once its recovery could not reopen it (it
334
+ stays on its new worker only to be deleted there), and ``call_running`` while a call of the session
335
+ past its time still holds the session's turn there."""
336
+ worker = self.live(session)
337
+ if session.unrecovered:
338
+ fail(UNKNOWN_SESSION, f"no session {session.session_id} on this host")
339
+ key = (id(worker), session.session_id)
340
+ with self._claims:
341
+ overdue = key in self._overdue
342
+ held = self._turns(worker).get(session.session_id) if overdue else None
343
+ if held is None:
344
+ self._unmark_overdue(key)
345
+ return worker
346
+ fail(
347
+ CALL_RUNNING,
348
+ f"an earlier call on this session has run {held:.0f} s, past its time; "
349
+ f"retry after {RUNNING_RETRY_AFTER_S} s",
350
+ retryAfter=RUNNING_RETRY_AFTER_S,
351
+ )
352
+
353
+ @classmethod
354
+ def _turns(cls, worker: Worker) -> dict[str, float]:
355
+ return cls._probe_turns(worker) or {}
356
+
357
+ @staticmethod
358
+ def _probe_turns(worker: Worker) -> dict[str, float] | None:
359
+ """Each session holding a turn on ``worker`` and how long it has held it, as the worker
360
+ says within ``HEALTH_TIMEOUT_S``; None when it cannot say. Never raises: the worker runs
361
+ tenant code, so its answer is cut off at the probe's deadline and size, and only a
362
+ finite, non-negative age of a session is kept."""
363
+ try:
364
+ status, payload = worker.probe("/__host/turns", deadline_s=HEALTH_TIMEOUT_S)
365
+ answer = json.loads(payload) if status == 200 else None
366
+ except (WorkerGone, ValueError, RecursionError):
367
+ return None
368
+ turns = answer.get("turns") if isinstance(answer, dict) else None
369
+ if not isinstance(turns, dict):
370
+ return None
371
+ ages = {str(sid): _age(age) for sid, age in turns.items()}
372
+ return {sid: age for sid, age in ages.items() if age is not None}
373
+
374
+ @staticmethod
375
+ def _stop(dead: Worker) -> None:
376
+ try:
377
+ dead.stop()
378
+ except Exception as error: # noqa: BLE001 - a worker that will not stop is replaced all the same
379
+ log_line(f"worker for version {dead.version_id} did not stop: {type(error).__name__}")
380
+
381
+ def _replacement(self, dead: Worker) -> Worker | None:
382
+ try:
383
+ return self.pool.replace(dead)
384
+ except Exception as error: # noqa: BLE001 - its sessions are lost either way; the platform reopens them
385
+ log_line(f"worker for version {dead.version_id} could not be replaced: {type(error).__name__}")
386
+ return None
387
+
388
+ def _set_aside(self, version_id: str, sessions: list[Session]) -> dict[str, Path]:
389
+ """Each session's directory moved under the worker's ``recovered/``, before a new worker
390
+ clears its sessions directory; a session whose directory cannot be moved is left out."""
391
+ root = self.home(version_id) / RECOVERED_DIR
392
+ try:
393
+ if not real_dir(root):
394
+ clear(root)
395
+ root.mkdir(mode=0o700, exist_ok=True)
396
+ except OSError as error:
397
+ log_line(f"no directory to set version {version_id}'s sessions aside in: {type(error).__name__}")
398
+ return {}
399
+ moved = {session.session_id: self._move_aside(root, version_id, session.session_id) for session in sessions}
400
+ return {session_id: directory for session_id, directory in moved.items() if directory is not None}
401
+
402
+ def _move_aside(self, root: Path, version_id: str, session_id: str) -> Path | None:
403
+ source = self.session_dir(version_id, session_id)
404
+ target = root / session_id
405
+ try:
406
+ if not (real_dir(root) and real_dir(source.parent) and real_dir(source)):
407
+ return None
408
+ clear(target)
409
+ os.replace(source, target)
410
+ except OSError as error:
411
+ log_line(f"session {session_id} could not be set aside: {type(error).__name__}")
412
+ return None
413
+ return target
414
+
415
+ def _reopen(self, fresh: Worker | None, session: Session, directory: Path | None) -> None:
416
+ if fresh is None or directory is None or session.placement is None:
417
+ return self._lose(session, directory)
418
+ body = {**session.placement, "restore": {"recovered": str(directory)}}
419
+ self._claim(fresh, session.session_id)
420
+ try:
421
+ status, _, payload = fresh.request(
422
+ "POST", "/__host/sessions", JSON, json.dumps(body).encode("utf-8"), control=True, timeout=OPEN_TIMEOUT_S
423
+ )
424
+ self._answer(status, payload)
425
+ except Exception as error: # noqa: BLE001 - one session that cannot be reopened never strands the rest
426
+ return self._reopen_failed(fresh, session, directory, error)
427
+ session.worker = fresh
428
+ session.export = None
429
+ self.registry.worker_died(session, [])
430
+ self.registry.tally("sessions_recovered")
431
+
432
+ def _reopen_failed(self, fresh: Worker, session: Session, directory: Path, error: Exception) -> None:
433
+ """A reopen that failed while ``fresh`` runs may still open there (an answer past the cut-off, or one
434
+ not read). The session is marked unrecovered and closed first, whatever closed it already, so no request
435
+ is relayed to ``fresh`` before both, then kept there, claimed: the recovery's give-back has it drained
436
+ and deleted there before its id takes a new open. With ``fresh`` gone too, it is forgotten."""
437
+ if not fresh.alive():
438
+ self._release(fresh, session.session_id)
439
+ return self._lose(session, directory, error)
440
+ self._unrecovered(session, error)
441
+ self.registry.close(session.session_id, reason=LOST)
442
+ session.worker = fresh
443
+ self._hand_over(session, directory)
444
+
445
+ def _lose(self, session: Session, directory: Path | None, error: BaseException | None = None) -> None:
446
+ self._hand_over(session, directory)
447
+ self.registry.forget_lost(session)
448
+ self._unrecovered(session, error)
449
+
450
+ def _unrecovered(self, session: Session, error: BaseException | None) -> None:
451
+ session.unrecovered = True
452
+ self.registry.tally("sessions_unrecovered")
453
+ why = f": {type(error).__name__}" if error is not None else ""
454
+ log_line(f"session {session.session_id} was not reopened on a new worker{why}; the platform reopens it")
455
+
456
+ def _hand_over(self, session: Session, directory: Path | None) -> None:
457
+ """The request ledger entries a session's dead worker had not sent, given to the host's
458
+ ledger before the session goes (none when its directory was not set aside); a journal
459
+ that cannot be handed over costs that session its entries and no other session anything."""
460
+ try:
461
+ self.registry.worker_died(session, self._left(directory))
462
+ except Exception as error: # noqa: BLE001 - one session's journal never strands the rest
463
+ log_line(f"session {session.session_id} journal was not handed over: {type(error).__name__}")
464
+
465
+ @staticmethod
466
+ def _left(directory: Path | None) -> list[dict]:
467
+ if directory is None or not (real_dir(directory.parent) and real_dir(directory)):
468
+ return []
469
+ try:
470
+ return CallJournal.read_left(directory / LEDGER_JOURNAL, LEFT_JOURNAL_BYTES)
471
+ finally:
472
+ shutil.rmtree(directory, ignore_errors=True)
473
+
474
+ def _worker_for(self, version_id: str) -> Worker:
475
+ current = self.pool.peek(version_id)
476
+ if current is not None and not current.alive():
477
+ self.recover(current)
478
+ try:
479
+ return self.pool.get(version_id)
480
+ except WorkerCapacity as error:
481
+ fail("at_capacity", str(error))
482
+
483
+ def _claim(self, worker: Worker, session_id: str) -> None:
484
+ with self._claims:
485
+ worker.sessions.add(session_id)
486
+
487
+ def _release(self, worker: Worker, session_id: str) -> None:
488
+ with self._claims:
489
+ worker.sessions.discard(session_id)
490
+ if not worker.sessions:
491
+ worker.idle_since = time.time()
492
+
493
+ def _control(
494
+ self,
495
+ worker: Worker,
496
+ method: str,
497
+ path: str,
498
+ body: dict | None,
499
+ timeout: float,
500
+ what: str,
501
+ session_id: str | None = None,
502
+ limit: int | None = None,
503
+ ) -> dict:
504
+ raw = json.dumps(body).encode("utf-8") if body is not None else b""
505
+ status, _, payload = self._forward(worker, method, path, JSON, raw, True, timeout, what, session_id, limit)
506
+ return self._answer(status, payload)
507
+
508
+ @staticmethod
509
+ def _answer(status: int, payload: bytes) -> dict:
510
+ """A worker's control answer as this host's: its refusal re-raised as ours."""
511
+ answer = json.loads(payload or b"null")
512
+ if status == 200:
513
+ return answer if isinstance(answer, dict) else {"result": answer}
514
+ detail: dict[str, Any] = answer if isinstance(answer, dict) else {}
515
+ retry = {"retryAfter": detail["retryAfter"]} if "retryAfter" in detail else {}
516
+ fail(str(detail.get("error") or "host"), str(detail.get("message") or detail.get("detail") or answer), **retry)
517
+
518
+ def _forward(
519
+ self,
520
+ worker: Worker,
521
+ method: str,
522
+ path: str,
523
+ headers: dict[str, str],
524
+ raw: bytes,
525
+ control: bool,
526
+ timeout: float,
527
+ what: str,
528
+ session_id: str | None = None,
529
+ limit: int | None = None,
530
+ ) -> tuple[int, list[tuple[str, str]], bytes]:
531
+ try:
532
+ return worker.request(method, path, headers, raw, control=control, timeout=timeout, limit=limit)
533
+ except WorkerGone as error:
534
+ raise self._unanswered(worker, error, what, timeout, session_id) from error
535
+
536
+ def _unanswered(
537
+ self, worker: Worker, error: WorkerGone, what: str, timeout: float, session_id: str | None
538
+ ) -> ContractError:
539
+ if worker.alive() and worker.answers_health():
540
+ return self._refusal(worker, error, what, timeout, session_id)
541
+ self._recycle(worker)
542
+ return ContractError(
543
+ WORKER_EXITED,
544
+ f"the world's worker exited during {what}; retry: the session continues from what it committed",
545
+ )
546
+
547
+ def _recycle(self, worker: Worker) -> None:
548
+ worker.process.kill()
549
+ threading.Thread(target=self.recover, args=(worker,), name="world-worker-recovery", daemon=True).start()
550
+
551
+ def _refusal(
552
+ self, worker: Worker, error: WorkerGone, what: str, timeout: float, session_id: str | None
553
+ ) -> ContractError:
554
+ if isinstance(error, WorkerTimeout):
555
+ self._mark_overdue(worker, session_id)
556
+ return ContractError(
557
+ CALL_TIMEOUT,
558
+ f"{what} did not finish within {timeout:g} s; it may still complete, and the session keeps serving",
559
+ )
560
+ return ContractError(
561
+ BUSY, f"the world's worker did not take {what}; retry after {RETRY_AFTER_S} s", retryAfter=RETRY_AFTER_S
562
+ )
563
+
564
+ def _mark_overdue(self, worker: Worker, session_id: str | None) -> None:
565
+ """A call of ``session_id`` past its time may still hold the session's turn on ``worker``."""
566
+ if session_id is None:
567
+ return
568
+ with self._claims:
569
+ self._overdue[(id(worker), session_id)] = worker
570
+
571
+ def _unmark_overdue(self, key: tuple[int, str]) -> None:
572
+ with self._claims:
573
+ self._overdue.pop(key, None)
574
+
575
+ def _forget_overdue(self, gone: Callable[[Worker], bool]) -> None:
576
+ with self._claims:
577
+ self._overdue = {key: worker for key, worker in self._overdue.items() if not gone(worker)}