@withgateway/sdk 0.9.1 → 0.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agents/index.d.mts +12 -4
- package/dist/agents/index.d.ts +12 -4
- package/dist/agents/index.js +30 -0
- package/dist/agents/index.js.map +1 -1
- package/dist/agents/index.mjs +30 -0
- package/dist/agents/index.mjs.map +1 -1
- package/dist/cli.js +2291 -945
- package/dist/cli.js.map +1 -1
- package/dist/cli.mjs +2290 -944
- package/dist/cli.mjs.map +1 -1
- package/dist/files/index.js.map +1 -1
- package/dist/files/index.mjs.map +1 -1
- package/dist/index.js +325 -184
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +324 -183
- package/dist/index.mjs.map +1 -1
- package/dist/traces/index.js.map +1 -1
- package/dist/traces/index.mjs.map +1 -1
- package/dist/tracing/index.js +1 -1
- package/dist/tracing/index.js.map +1 -1
- package/dist/tracing/index.mjs +1 -1
- package/dist/tracing/index.mjs.map +1 -1
- package/dist/worlds/index.d.mts +3 -1
- package/dist/worlds/index.d.ts +3 -1
- package/dist/worlds/index.js +178 -67
- package/dist/worlds/index.js.map +1 -1
- package/dist/worlds/index.mjs +177 -66
- package/dist/worlds/index.mjs.map +1 -1
- package/package.json +1 -1
- package/runtime/gatewaysdk/environment/bundle_manifest.py +297 -9
- package/runtime/gatewaysdk/environment/factory_documents.py +670 -0
- package/runtime/gatewaysdk/environment/factory_rules.py +973 -0
- package/runtime/gatewaysdk/environment/factory_shapes.py +253 -0
- package/runtime/gatewaysdk/environment/factory_spine.py +1155 -0
- package/runtime/gatewaysdk/environment/factory_spine_shapes.py +480 -0
- package/runtime/gatewaysdk/environment/schema/admission.py +61 -59
- package/runtime/gatewaysdk/environment/schema/cast.py +133 -67
- package/runtime/gatewaysdk/environment/schema/compiler.py +16 -17
- package/runtime/gatewaysdk/environment/schema/handler_errors.py +58 -4
- package/runtime/gatewaysdk/environment/schema/host.py +21 -2
- package/runtime/gatewaysdk/environment/schema/host_control.py +3 -1
- package/runtime/gatewaysdk/environment/schema/host_loops.py +27 -4
- package/runtime/gatewaysdk/environment/schema/host_metrics.py +69 -0
- package/runtime/gatewaysdk/environment/schema/host_requests.py +21 -0
- package/runtime/gatewaysdk/environment/schema/host_session.py +7 -0
- package/runtime/gatewaysdk/environment/schema/host_versions.py +4 -0
- package/runtime/gatewaysdk/environment/schema/host_workers.py +172 -50
- package/runtime/gatewaysdk/environment/schema/ingest.py +12 -3
- package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
- package/runtime/gatewaysdk/environment/schema/measured_tests.py +187 -0
- package/runtime/gatewaysdk/environment/schema/platform.py +57 -3
- package/runtime/gatewaysdk/environment/schema/platform_query.py +4 -3
- package/runtime/gatewaysdk/environment/schema/platform_templates.py +105 -17
- package/runtime/gatewaysdk/environment/schema/process_memory.py +76 -0
- package/runtime/gatewaysdk/environment/schema/redaction.py +13 -0
- package/runtime/gatewaysdk/environment/schema/runtime.py +14 -0
- package/runtime/gatewaysdk/environment/schema/scaffold.py +21 -4
- package/runtime/gatewaysdk/environment/schema/shape_catalog.py +493 -0
- package/runtime/gatewaysdk/environment/schema/shape_declarations.py +219 -0
- package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/factory-orchestrator/SKILL.md +147 -0
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +5 -1
- package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +4 -4
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +2 -2
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +1 -0
- package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/shape-catalog.md +59 -0
- package/runtime/gatewaysdk/environment/schema/validation.py +14 -0
- package/runtime/gatewaysdk/environment/schema/worker.py +7 -4
- package/runtime/gatewaysdk/environment/schema/worker_key.py +38 -0
- package/runtime/gatewaysdk/environment/schema/worker_peaks.py +38 -0
- package/runtime/gatewaysdk/environment/schema/worker_watch.py +50 -0
- package/runtime/gatewaysdk/environment/schema/workers.py +237 -68
- package/runtime/gatewaysdk/environment/schema/world_host.py +56 -62
- package/runtime/gatewaysdk/environment/schema/world_tests.py +3 -1
- package/skills/connector-schema-authoring/SKILL.md +1 -0
- package/skills/connector-schema-authoring/connector-worlds.md +1 -0
- package/skills/connector-schema-authoring/world-schema.md +7 -0
- package/skills/factory-orchestrator/SKILL.md +147 -0
- package/skills/world-data-ingestion/ROWS.md +5 -1
- package/skills/world-data-ingestion/SKILL.md +4 -4
- package/skills/worlds-getting-started/SKILL.md +2 -2
- package/skills/worlds-getting-started/references/cli-mcp-map.md +1 -0
- package/skills/worlds-getting-started/references/shape-catalog.md +59 -0
|
@@ -39,6 +39,27 @@ def worker_snapshot(snapshot: dict[str, str] | None) -> dict[str, str] | None:
|
|
|
39
39
|
return {key: value for key, value in snapshot.items() if key != "url"} if snapshot else None
|
|
40
40
|
|
|
41
41
|
|
|
42
|
+
MAX_MEMORY_PEAK_MB = 1 << 20
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def parse_memory_peak(value: Any) -> int | None:
|
|
46
|
+
"""An open's ``memoryPeakMb``: the most memory the platform measured the session's world to need, a whole
|
|
47
|
+
MiB from 1 to 1 TiB; None when it has not measured one."""
|
|
48
|
+
if value is None:
|
|
49
|
+
return None
|
|
50
|
+
if isinstance(value, bool) or not isinstance(value, int) or not 0 < value <= MAX_MEMORY_PEAK_MB:
|
|
51
|
+
fail("host", "memoryPeakMb is a whole number of MiB from 1 to 1048576")
|
|
52
|
+
return value
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def parse_world_state(value: Any) -> bool | None:
|
|
56
|
+
"""An open's ``worldState``: whether the session is on one of its world's states, None when the open names
|
|
57
|
+
none (a platform from before a version's states had a worker of their own)."""
|
|
58
|
+
if value is None or isinstance(value, bool):
|
|
59
|
+
return value
|
|
60
|
+
fail("host", "worldState is true or false")
|
|
61
|
+
|
|
62
|
+
|
|
42
63
|
def parse_dependency_refs(dependencies: Any) -> list[DependencyRef]:
|
|
43
64
|
if dependencies is None:
|
|
44
65
|
return []
|
|
@@ -13,6 +13,7 @@ from .door import OPEN_MODE, VENDOR_MODE, Door, same
|
|
|
13
13
|
from .host_surfaces import SessionHost, capability_host, session_host
|
|
14
14
|
from .request_ledger import LedgerLife
|
|
15
15
|
from .session_move import SessionMove
|
|
16
|
+
from .worker_key import WorkerKey
|
|
16
17
|
|
|
17
18
|
if TYPE_CHECKING:
|
|
18
19
|
from .api import ApiSettings, WorldApp
|
|
@@ -85,6 +86,7 @@ class Session:
|
|
|
85
86
|
recoveries: int = field(default=0, init=False)
|
|
86
87
|
close_reason: str = field(default="closed", init=False)
|
|
87
88
|
unrecovered: bool = field(default=False, init=False)
|
|
89
|
+
world_state: bool = False
|
|
88
90
|
|
|
89
91
|
def __post_init__(self, pending: bool) -> None:
|
|
90
92
|
self.move = SessionMove(self.session_id, pending=pending)
|
|
@@ -94,6 +96,11 @@ class Session:
|
|
|
94
96
|
world serves browsers: from the world's declared settings, never its worker."""
|
|
95
97
|
return None if self.settings is None else self.settings.cors.preflight(method, headers)
|
|
96
98
|
|
|
99
|
+
@property
|
|
100
|
+
def key(self) -> WorkerKey:
|
|
101
|
+
"""Which of its version's lanes serves the session: its turns, and its worker when it has one."""
|
|
102
|
+
return WorkerKey.of(self.version_id, self.world_state)
|
|
103
|
+
|
|
97
104
|
@property
|
|
98
105
|
def app(self) -> WorldApp | None:
|
|
99
106
|
"""The world's routes over this session's store, when this process
|
|
@@ -142,6 +142,10 @@ class CachedVersion:
|
|
|
142
142
|
def needs_worker(self) -> bool:
|
|
143
143
|
return self.authored_code is not None
|
|
144
144
|
|
|
145
|
+
@property
|
|
146
|
+
def world(self) -> str:
|
|
147
|
+
return self.runtime_class.world_name(self)
|
|
148
|
+
|
|
145
149
|
def served_names(self, dependencies: dict[str, DependencyWorld]) -> ServedNames:
|
|
146
150
|
"""The tools a session task may allow, beside the routes it may not."""
|
|
147
151
|
return ServedNames.of(self.runtime_class.served_by(self, dependencies), self.spec)
|
|
@@ -17,7 +17,17 @@ forwarded request answers when the worker does not answer it:
|
|
|
17
17
|
that. A worker that died idle is replaced the same way by the next request or the
|
|
18
18
|
lease sweeper, which also recycles any worker holding a turn past ``STUCK_TURN_S``,
|
|
19
19
|
its session still open here or not. What a worker answers about its turns is read
|
|
20
|
-
as untrusted: it runs tenant code
|
|
20
|
+
as untrusted: it runs tenant code;
|
|
21
|
+
* world code raised ``MemoryError`` there (the worker's own mark on that answer,
|
|
22
|
+
:class:`handler_errors.OutOfMemory`, which no data it serves can set): that answer goes
|
|
23
|
+
back as it is, and the worker, if the kernel counts it at ``POISONED_HELD_FRACTION`` of its
|
|
24
|
+
cap or more, is poisoned. It takes no new request, the requests already
|
|
25
|
+
on it get up to ``POISONED_SETTLE_S`` to finish, and it is then recycled like a stuck one,
|
|
26
|
+
at most once per ``POISONED_BACKOFF_S`` for its key; the next request waits for the fresh
|
|
27
|
+
worker, and an open that met the poison is told to come back (``busy``).
|
|
28
|
+
|
|
29
|
+
A version's sessions on its world states have a worker of their own beside main's
|
|
30
|
+
(:class:`WorkerKey`), so a state that exhausts its worker recycles that worker only.
|
|
21
31
|
|
|
22
32
|
Recovery never raises: a session that cannot be reopened leaves the registry (the
|
|
23
33
|
platform reads it as lost and reopens it from its last export), never left on a dead
|
|
@@ -33,17 +43,21 @@ import shutil
|
|
|
33
43
|
import threading
|
|
34
44
|
import time
|
|
35
45
|
import urllib.parse
|
|
36
|
-
from collections
|
|
46
|
+
from collections import Counter
|
|
47
|
+
from collections.abc import Callable, Iterator
|
|
48
|
+
from contextlib import contextmanager
|
|
37
49
|
from pathlib import Path
|
|
38
50
|
from typing import TYPE_CHECKING, Any, Protocol
|
|
39
51
|
|
|
40
52
|
from .admission import BUSY, RETRY_AFTER_S
|
|
41
53
|
from .call_log import CallJournal
|
|
42
|
-
from .handler_errors import log_line
|
|
54
|
+
from .handler_errors import OutOfMemory, log_line
|
|
55
|
+
from .process_memory import SIZE
|
|
43
56
|
from .lanes import Lanes
|
|
44
57
|
from .request_ledger import DRAIN_ANSWER_BYTES, LEFT_JOURNAL_BYTES
|
|
45
58
|
from .session_files import LEDGER_JOURNAL, RECOVERED_DIR, ExportFile, clear, real_dir
|
|
46
59
|
from .validation import ContractError, fail
|
|
60
|
+
from .worker_key import WorkerKey
|
|
47
61
|
from .workers import HEALTH_TIMEOUT_S, Worker, WorkerCapacity, WorkerGone, WorkerPool, WorkerTimeout, WorkerUnreached
|
|
48
62
|
|
|
49
63
|
if TYPE_CHECKING:
|
|
@@ -66,7 +80,16 @@ OPEN_TIMEOUT_S = 60.0
|
|
|
66
80
|
EXPORT_TIMEOUT_S = 300.0
|
|
67
81
|
#: A turn held this long serves no request the supervisor still waits on: its worker is recycled.
|
|
68
82
|
STUCK_TURN_S = 3 * CALL_TIMEOUT_S
|
|
83
|
+
#: What a poisoned worker's requests already in flight get to finish before it is recycled under them.
|
|
84
|
+
POISONED_SETTLE_S = 5.0
|
|
85
|
+
#: A key whose worker was recycled for running out of memory is not recycled for it again sooner.
|
|
86
|
+
POISONED_BACKOFF_S = 60.0
|
|
87
|
+
#: A worker that ran out of memory is recycled only while it holds this much of its cap: one request asking
|
|
88
|
+
#: for more than the cap is refused its allocation and leaves the process as it was.
|
|
89
|
+
POISONED_HELD_FRACTION = 0.75
|
|
90
|
+
WORKERS_POISONED = "workers_poisoned"
|
|
69
91
|
RUNNING_RETRY_AFTER_S = 5
|
|
92
|
+
OUT_OF_MEMORY_RETRY = "the world's worker ran out of memory and is being replaced; retry"
|
|
70
93
|
JSON = {"content-type": "application/json"}
|
|
71
94
|
#: An open refused ``draining`` or ``at_capacity`` (503) goes to another replica; any other 503, ``busy``
|
|
72
95
|
#: for an id still closing included, sends the platform back later; 404 is the one answer it reads as a loss.
|
|
@@ -94,6 +117,10 @@ def refusal(error: ContractError) -> tuple[int, dict, dict[str, str]]:
|
|
|
94
117
|
return REFUSALS.get(code, 400), {**body, "retryAfter": retry}, {"Retry-After": str(retry)}
|
|
95
118
|
|
|
96
119
|
|
|
120
|
+
def _busy(error: BaseException) -> bool:
|
|
121
|
+
return isinstance(error, ContractError) and error.detail.get("code") == BUSY
|
|
122
|
+
|
|
123
|
+
|
|
97
124
|
def _age(value: object) -> float | None:
|
|
98
125
|
"""A turn's age as a worker reported it, or None unless it is a finite, non-negative number."""
|
|
99
126
|
if isinstance(value, bool) or not isinstance(value, (int, float)):
|
|
@@ -130,45 +157,74 @@ class VersionWorkers:
|
|
|
130
157
|
self._claims = threading.Lock()
|
|
131
158
|
self._recoveries = Lanes()
|
|
132
159
|
self._overdue: dict[tuple[int, str], Worker] = {}
|
|
160
|
+
self._forwarding = threading.Condition()
|
|
161
|
+
self._inflight: Counter[int] = Counter()
|
|
162
|
+
self._poisoned_at: dict[WorkerKey, float] = {}
|
|
133
163
|
|
|
134
|
-
def home(self,
|
|
135
|
-
"""Where a
|
|
136
|
-
return self.pool.workers_root
|
|
164
|
+
def home(self, key: WorkerKey) -> Path:
|
|
165
|
+
"""Where a key's worker runs: its own ``WorldHost`` cache root (:mod:`worker`)."""
|
|
166
|
+
return key.home(self.pool.workers_root)
|
|
137
167
|
|
|
138
|
-
def session_dir(self,
|
|
139
|
-
return self.home(
|
|
168
|
+
def session_dir(self, key: WorkerKey, session_id: str) -> Path:
|
|
169
|
+
return self.home(key) / "sessions" / session_id
|
|
140
170
|
|
|
141
171
|
def place(
|
|
142
|
-
self,
|
|
172
|
+
self,
|
|
173
|
+
key: WorkerKey,
|
|
174
|
+
session_id: str,
|
|
175
|
+
body: dict,
|
|
176
|
+
fetch: Callable[[Path], ExportFile | None],
|
|
177
|
+
opens: Lanes,
|
|
178
|
+
world: str,
|
|
179
|
+
need_mb: int = 0,
|
|
143
180
|
) -> Worker:
|
|
144
|
-
"""Open ``session_id`` on its
|
|
181
|
+
"""Open ``session_id`` on its key's worker with ``body``, claimed before placement so
|
|
145
182
|
another open cannot stop the worker as session-less meanwhile. ``fetch`` downloads an
|
|
146
183
|
export to restore into the session's directory once the worker runs (a worker clears
|
|
147
|
-
that directory when it starts); the worker reads it by path. One open per
|
|
148
|
-
time (``opens``), so the worker's open timeout counts one open, never a queue.
|
|
149
|
-
worker
|
|
184
|
+
that directory when it starts); the worker reads it by path. One open per key at a
|
|
185
|
+
time (``opens``), so the worker's open timeout counts one open, never a queue. ``world``
|
|
186
|
+
labels the worker in what the host logs about it; ``need_mb`` is what the session's world was
|
|
187
|
+
measured to need of it (:class:`MemoryNeed`), 0 when unmeasured."""
|
|
188
|
+
worker = self._worker_for(key, world, need_mb)
|
|
150
189
|
self._claim(worker, session_id)
|
|
151
190
|
source = None
|
|
152
191
|
try:
|
|
153
|
-
source = fetch(self.session_dir(
|
|
192
|
+
source = fetch(self.session_dir(key, session_id))
|
|
154
193
|
placement = {**body, "restore": source.for_worker() if source is not None else None}
|
|
155
194
|
self._open_on(worker, session_id, placement, opens)
|
|
156
|
-
|
|
195
|
+
self._refuse_poisoned(worker, session_id)
|
|
196
|
+
except BaseException as error:
|
|
157
197
|
self._release(worker, session_id)
|
|
198
|
+
if worker.poisoned and not _busy(error):
|
|
199
|
+
raise ContractError(BUSY, OUT_OF_MEMORY_RETRY, retryAfter=RETRY_AFTER_S) from error
|
|
158
200
|
raise
|
|
159
201
|
finally:
|
|
160
202
|
if source is not None:
|
|
161
203
|
source.discard()
|
|
162
204
|
return worker
|
|
163
205
|
|
|
206
|
+
def _refuse_poisoned(self, worker: Worker, session_id: str) -> None:
|
|
207
|
+
"""An open that landed on a worker poisoned meanwhile is not on any recovery's list: it is deleted
|
|
208
|
+
there rather than kept on a worker about to stop."""
|
|
209
|
+
if not worker.poisoned:
|
|
210
|
+
return
|
|
211
|
+
self._delete(worker, session_id)
|
|
212
|
+
fail(BUSY, OUT_OF_MEMORY_RETRY, retryAfter=RETRY_AFTER_S)
|
|
213
|
+
|
|
164
214
|
def _open_on(self, worker: Worker, session_id: str, placement: dict, opens: Lanes) -> None:
|
|
165
215
|
"""The open on ``worker``, in its version's turn. One that raised after reaching the worker may still
|
|
166
216
|
complete there (an answer past the cut-off) or have left a copy, which no open of the id may find
|
|
167
217
|
later, so it is deleted there, an open still running abandoned, before the caller lets the claim go.
|
|
168
218
|
One that never reached it (the refusal's cause a :class:`WorkerUnreached`) left nothing to delete."""
|
|
219
|
+
raw = json.dumps(placement).encode("utf-8")
|
|
169
220
|
try:
|
|
170
|
-
with opens.turn(worker.
|
|
171
|
-
|
|
221
|
+
with opens.turn(worker.key.name):
|
|
222
|
+
status, headers, payload = self._forward(
|
|
223
|
+
worker, "POST", "/__host/sessions", JSON, raw, True, OPEN_TIMEOUT_S, "the open"
|
|
224
|
+
)
|
|
225
|
+
if OutOfMemory.in_headers(headers) and (worker.poisoned or self._exhausted(worker)):
|
|
226
|
+
fail(BUSY, OUT_OF_MEMORY_RETRY, retryAfter=RETRY_AFTER_S)
|
|
227
|
+
self._answer(status, payload)
|
|
172
228
|
except Exception as error:
|
|
173
229
|
if not isinstance(error.__cause__, WorkerUnreached):
|
|
174
230
|
self._delete(worker, session_id)
|
|
@@ -216,12 +272,12 @@ class VersionWorkers:
|
|
|
216
272
|
|
|
217
273
|
def live(self, session: Session) -> Worker:
|
|
218
274
|
"""The session's worker, replaced first (and the session reopened on the new one) if it
|
|
219
|
-
died; a session no live worker holds afterwards is forgotten and answers
|
|
220
|
-
A session that began closing meanwhile is served like any other:
|
|
221
|
-
on the new worker has drained the entries it resumed first."""
|
|
275
|
+
died or was poisoned; a session no live worker holds afterwards is forgotten and answers
|
|
276
|
+
``unknown_session``. A session that began closing meanwhile is served like any other:
|
|
277
|
+
whoever then deletes it on the new worker has drained the entries it resumed first."""
|
|
222
278
|
worker = session.worker
|
|
223
279
|
assert worker is not None
|
|
224
|
-
if worker.
|
|
280
|
+
if worker.serves():
|
|
225
281
|
return worker
|
|
226
282
|
self.recover(worker)
|
|
227
283
|
current = session.worker
|
|
@@ -237,7 +293,7 @@ class VersionWorkers:
|
|
|
237
293
|
assert worker is not None
|
|
238
294
|
self._release(worker, session.session_id)
|
|
239
295
|
if not worker.alive():
|
|
240
|
-
return self._hand_over(session, self.session_dir(
|
|
296
|
+
return self._hand_over(session, self.session_dir(worker.key, session.session_id))
|
|
241
297
|
self._delete(worker, session.session_id)
|
|
242
298
|
|
|
243
299
|
def _delete(self, worker: Worker, session_id: str) -> None:
|
|
@@ -268,7 +324,7 @@ class VersionWorkers:
|
|
|
268
324
|
"""The file a worker's export landed in, admitted only from inside that worker's
|
|
269
325
|
directory for this session: the worker runs tenant code, so what it names is
|
|
270
326
|
checked, and the bytes are hashed by the supervisor, not trusted."""
|
|
271
|
-
exports = self.session_dir(session.
|
|
327
|
+
exports = self.session_dir(session.key, session.session_id) / "exports"
|
|
272
328
|
path = Path(str(answer.get("path") or ""))
|
|
273
329
|
try:
|
|
274
330
|
resolved = path.resolve(strict=True)
|
|
@@ -278,27 +334,29 @@ class VersionWorkers:
|
|
|
278
334
|
return resolved
|
|
279
335
|
|
|
280
336
|
def recover(self, dead: Worker, reason: str = "workers_replaced") -> None:
|
|
281
|
-
"""Replace ``dead`` (stopped first if it still runs
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
(:meth:`_give_back`)."""
|
|
337
|
+
"""Replace ``dead`` (stopped first if it still runs; a poisoned one once the requests already on it
|
|
338
|
+
are answered or ``POISONED_SETTLE_S`` passes) and reopen every session it held on the new worker
|
|
339
|
+
from what each committed; a session that cannot be reopened leaves the registry
|
|
340
|
+
(:meth:`_reopen_failed`). One recovery per key at a time; a later caller finds it done. Never
|
|
341
|
+
raises. Every session it takes is its own until it gives the session back (:meth:`_give_back`)."""
|
|
286
342
|
held: list[Session] = []
|
|
287
343
|
try:
|
|
288
|
-
with self._recoveries.turn(dead.
|
|
289
|
-
if self.pool.peek(dead.
|
|
344
|
+
with self._recoveries.turn(dead.key.name):
|
|
345
|
+
if self.pool.peek(dead.key) is not dead:
|
|
290
346
|
return
|
|
347
|
+
self._settle(dead)
|
|
348
|
+
self._outgrown(dead)
|
|
291
349
|
self._stop(dead)
|
|
292
350
|
sessions, closing = self.registry.recovering(dead)
|
|
293
351
|
held = [*sessions, *closing]
|
|
294
|
-
aside = self._set_aside(dead.
|
|
352
|
+
aside = self._set_aside(dead.key, held)
|
|
295
353
|
fresh = self._replacement(dead)
|
|
296
354
|
for session in closing:
|
|
297
355
|
self._hand_over(session, aside.get(session.session_id))
|
|
298
356
|
for session in sessions:
|
|
299
357
|
self._reopen(fresh, session, aside.get(session.session_id))
|
|
300
358
|
self._forget_overdue(lambda worker: worker is dead)
|
|
301
|
-
self.registry.tally(reason)
|
|
359
|
+
self.registry.tally(WORKERS_POISONED if dead.poisoned else reason)
|
|
302
360
|
finally:
|
|
303
361
|
self._give_back(held)
|
|
304
362
|
|
|
@@ -313,21 +371,21 @@ class VersionWorkers:
|
|
|
313
371
|
log_line(f"session {session.session_id} did not leave its new worker: {type(error).__name__}")
|
|
314
372
|
|
|
315
373
|
def recover_dead(self) -> list[str]:
|
|
316
|
-
"""Recover every worker that died holding sessions; returns their
|
|
374
|
+
"""Recover every worker that died holding sessions; returns their keys' names."""
|
|
317
375
|
dead = self.pool.dead()
|
|
318
376
|
for worker in dead:
|
|
319
377
|
self.recover(worker)
|
|
320
|
-
return [worker.
|
|
378
|
+
return [worker.key.name for worker in dead]
|
|
321
379
|
|
|
322
380
|
def recover_stuck(self) -> list[str]:
|
|
323
381
|
"""Recycle every live worker holding a turn past ``STUCK_TURN_S`` (a handler that never
|
|
324
382
|
returned), whether or not that turn's session is still open here: its sessions reopen
|
|
325
|
-
from what they committed, and an id it still held opens again. Returns their
|
|
383
|
+
from what they committed, and an id it still held opens again. Returns their keys' names."""
|
|
326
384
|
self._forget_overdue(lambda worker: not worker.alive())
|
|
327
385
|
stuck = [worker for worker in self.pool.live() if max(self._turns(worker).values(), default=0) > STUCK_TURN_S]
|
|
328
386
|
for worker in stuck:
|
|
329
387
|
self.recover(worker, "workers_recycled")
|
|
330
|
-
return [worker.
|
|
388
|
+
return [worker.key.name for worker in stuck]
|
|
331
389
|
|
|
332
390
|
def _serving(self, session: Session) -> Worker:
|
|
333
391
|
"""The session's live worker, refused ``unknown_session`` once its recovery could not reopen it (it
|
|
@@ -371,36 +429,49 @@ class VersionWorkers:
|
|
|
371
429
|
ages = {str(sid): _age(age) for sid, age in turns.items()}
|
|
372
430
|
return {sid: age for sid, age in ages.items() if age is not None}
|
|
373
431
|
|
|
432
|
+
def _outgrown(self, worker: Worker) -> None:
|
|
433
|
+
"""A poisoned worker's replacement is sized by what it was seen to hold, read before it stops."""
|
|
434
|
+
if not worker.poisoned:
|
|
435
|
+
return
|
|
436
|
+
self.pool.outgrown(worker)
|
|
437
|
+
|
|
438
|
+
def _settle(self, worker: Worker) -> None:
|
|
439
|
+
"""A poisoned worker's requests already forwarded get up to ``POISONED_SETTLE_S`` to be answered."""
|
|
440
|
+
if not worker.poisoned:
|
|
441
|
+
return
|
|
442
|
+
with self._forwarding:
|
|
443
|
+
self._forwarding.wait_for(lambda: not self._inflight[id(worker)], timeout=POISONED_SETTLE_S)
|
|
444
|
+
|
|
374
445
|
@staticmethod
|
|
375
446
|
def _stop(dead: Worker) -> None:
|
|
376
447
|
try:
|
|
377
448
|
dead.stop()
|
|
378
449
|
except Exception as error: # noqa: BLE001 - a worker that will not stop is replaced all the same
|
|
379
|
-
log_line(f"worker for
|
|
450
|
+
log_line(f"worker for {dead.key.name} did not stop: {type(error).__name__}")
|
|
380
451
|
|
|
381
452
|
def _replacement(self, dead: Worker) -> Worker | None:
|
|
382
453
|
try:
|
|
383
454
|
return self.pool.replace(dead)
|
|
384
455
|
except Exception as error: # noqa: BLE001 - its sessions are lost either way; the platform reopens them
|
|
385
|
-
log_line(f"worker for
|
|
456
|
+
log_line(f"worker for {dead.key.name} could not be replaced: {type(error).__name__}")
|
|
386
457
|
return None
|
|
387
458
|
|
|
388
|
-
def _set_aside(self,
|
|
459
|
+
def _set_aside(self, key: WorkerKey, sessions: list[Session]) -> dict[str, Path]:
|
|
389
460
|
"""Each session's directory moved under the worker's ``recovered/``, before a new worker
|
|
390
461
|
clears its sessions directory; a session whose directory cannot be moved is left out."""
|
|
391
|
-
root = self.home(
|
|
462
|
+
root = self.home(key) / RECOVERED_DIR
|
|
392
463
|
try:
|
|
393
464
|
if not real_dir(root):
|
|
394
465
|
clear(root)
|
|
395
466
|
root.mkdir(mode=0o700, exist_ok=True)
|
|
396
467
|
except OSError as error:
|
|
397
|
-
log_line(f"no directory to set
|
|
468
|
+
log_line(f"no directory to set {key.name}'s sessions aside in: {type(error).__name__}")
|
|
398
469
|
return {}
|
|
399
|
-
moved = {session.session_id: self._move_aside(root,
|
|
470
|
+
moved = {session.session_id: self._move_aside(root, key, session.session_id) for session in sessions}
|
|
400
471
|
return {session_id: directory for session_id, directory in moved.items() if directory is not None}
|
|
401
472
|
|
|
402
|
-
def _move_aside(self, root: Path,
|
|
403
|
-
source = self.session_dir(
|
|
473
|
+
def _move_aside(self, root: Path, key: WorkerKey, session_id: str) -> Path | None:
|
|
474
|
+
source = self.session_dir(key, session_id)
|
|
404
475
|
target = root / session_id
|
|
405
476
|
try:
|
|
406
477
|
if not (real_dir(root) and real_dir(source.parent) and real_dir(source)):
|
|
@@ -418,9 +489,10 @@ class VersionWorkers:
|
|
|
418
489
|
body = {**session.placement, "restore": {"recovered": str(directory)}}
|
|
419
490
|
self._claim(fresh, session.session_id)
|
|
420
491
|
try:
|
|
421
|
-
status,
|
|
492
|
+
status, headers, payload = fresh.request(
|
|
422
493
|
"POST", "/__host/sessions", JSON, json.dumps(body).encode("utf-8"), control=True, timeout=OPEN_TIMEOUT_S
|
|
423
494
|
)
|
|
495
|
+
self._watch(fresh, headers)
|
|
424
496
|
self._answer(status, payload)
|
|
425
497
|
except Exception as error: # noqa: BLE001 - one session that cannot be reopened never strands the rest
|
|
426
498
|
return self._reopen_failed(fresh, session, directory, error)
|
|
@@ -471,12 +543,12 @@ class VersionWorkers:
|
|
|
471
543
|
finally:
|
|
472
544
|
shutil.rmtree(directory, ignore_errors=True)
|
|
473
545
|
|
|
474
|
-
def _worker_for(self,
|
|
475
|
-
current = self.pool.peek(
|
|
476
|
-
if current is not None and not current.
|
|
546
|
+
def _worker_for(self, key: WorkerKey, world: str, need_mb: int) -> Worker:
|
|
547
|
+
current = self.pool.peek(key)
|
|
548
|
+
if current is not None and not current.serves():
|
|
477
549
|
self.recover(current)
|
|
478
550
|
try:
|
|
479
|
-
return self.pool.get(
|
|
551
|
+
return self.pool.get(key, world=world, need_mb=need_mb)
|
|
480
552
|
except WorkerCapacity as error:
|
|
481
553
|
fail("at_capacity", str(error))
|
|
482
554
|
|
|
@@ -529,9 +601,59 @@ class VersionWorkers:
|
|
|
529
601
|
limit: int | None = None,
|
|
530
602
|
) -> tuple[int, list[tuple[str, str]], bytes]:
|
|
531
603
|
try:
|
|
532
|
-
|
|
604
|
+
with self._forwarded(worker):
|
|
605
|
+
answer = worker.request(method, path, headers, raw, control=control, timeout=timeout, limit=limit)
|
|
533
606
|
except WorkerGone as error:
|
|
534
607
|
raise self._unanswered(worker, error, what, timeout, session_id) from error
|
|
608
|
+
self._watch(worker, answer[1])
|
|
609
|
+
return answer
|
|
610
|
+
|
|
611
|
+
def _watch(self, worker: Worker, headers: list[tuple[str, str]]) -> None:
|
|
612
|
+
if OutOfMemory.in_headers(headers):
|
|
613
|
+
self._poison(worker)
|
|
614
|
+
|
|
615
|
+
@contextmanager
|
|
616
|
+
def _forwarded(self, worker: Worker) -> Iterator[None]:
|
|
617
|
+
with self._forwarding:
|
|
618
|
+
self._inflight[id(worker)] += 1
|
|
619
|
+
try:
|
|
620
|
+
yield
|
|
621
|
+
finally:
|
|
622
|
+
with self._forwarding:
|
|
623
|
+
self._inflight[id(worker)] -= 1
|
|
624
|
+
if self._inflight[id(worker)] <= 0:
|
|
625
|
+
del self._inflight[id(worker)]
|
|
626
|
+
self._forwarding.notify_all()
|
|
627
|
+
|
|
628
|
+
def _exhausted(self, worker: Worker) -> bool:
|
|
629
|
+
"""Whether the kernel counts ``worker`` at ``POISONED_HELD_FRACTION`` of its address-space cap or more
|
|
630
|
+
(``VmSize``); a worker the kernel cannot say of (no ``/proc``) is taken at its word."""
|
|
631
|
+
size = self.pool.memory.status(worker.pid).mb(SIZE)
|
|
632
|
+
return size is None or size >= worker.cap_mb * POISONED_HELD_FRACTION
|
|
633
|
+
|
|
634
|
+
def _poison(self, worker: Worker) -> None:
|
|
635
|
+
"""World code on ``worker`` raised ``MemoryError`` while it holds most of its memory: it takes no new
|
|
636
|
+
request and is recycled behind this answer, once, with a line naming its world and version; not when
|
|
637
|
+
its key was recycled for this within ``POISONED_BACKOFF_S``, so a lane that keeps running out cuts its
|
|
638
|
+
calls once a minute at most, and never for a request that only asked for more than the cap."""
|
|
639
|
+
if not self._exhausted(worker):
|
|
640
|
+
return
|
|
641
|
+
now = time.monotonic()
|
|
642
|
+
with self._claims:
|
|
643
|
+
backed_off = now - self._poisoned_at.get(worker.key, -math.inf) < POISONED_BACKOFF_S
|
|
644
|
+
first = not worker.poisoned and not backed_off
|
|
645
|
+
if first:
|
|
646
|
+
worker.poisoned = True
|
|
647
|
+
self._poisoned_at[worker.key] = now
|
|
648
|
+
if not first:
|
|
649
|
+
return
|
|
650
|
+
log_line(
|
|
651
|
+
f"world {worker.world!r} version {worker.version_id} ({worker.key.lane}): out of memory in its worker "
|
|
652
|
+
f"(pid {worker.pid}); recycling it"
|
|
653
|
+
)
|
|
654
|
+
threading.Thread(
|
|
655
|
+
target=self.recover, args=(worker, WORKERS_POISONED), name="world-worker-recycle", daemon=True
|
|
656
|
+
).start()
|
|
535
657
|
|
|
536
658
|
def _unanswered(
|
|
537
659
|
self, worker: Worker, error: WorkerGone, what: str, timeout: float, session_id: str | None
|
|
@@ -407,9 +407,7 @@ def ingest_records(
|
|
|
407
407
|
else:
|
|
408
408
|
fail("ingest", "mapping is an ingest.toml text, a parsed table, a Mapping or a file path", "/mapping")
|
|
409
409
|
if declared is not None:
|
|
410
|
-
|
|
411
|
-
validate_mapping(declared, entities)
|
|
412
|
-
projected, report = apply_mapping(declared, listed, entities)
|
|
410
|
+
projected, report = map_records(root, listed, declared)
|
|
413
411
|
else:
|
|
414
412
|
spec = load_spec(root) if (root / SPEC).is_file() else None
|
|
415
413
|
if spec is None or not any(operation.entity for operation in spec.operations.values()):
|
|
@@ -446,6 +444,17 @@ def ingest_records(
|
|
|
446
444
|
return report
|
|
447
445
|
|
|
448
446
|
|
|
447
|
+
def map_records(root: str | Path, records: list, mapping: Any) -> tuple[dict[str, list[dict]], dict]:
|
|
448
|
+
"""``records`` through a declared ``ingest.toml`` against the world's contract, nothing imported:
|
|
449
|
+
the rows and the mapping's report, for a caller that lands the rows through its own gate."""
|
|
450
|
+
from .mapping import Mapping, apply_mapping, parse_mapping, validate_mapping
|
|
451
|
+
|
|
452
|
+
declared = mapping if isinstance(mapping, Mapping) else parse_mapping(mapping)
|
|
453
|
+
entities = _entities(Path(root))
|
|
454
|
+
validate_mapping(declared, entities)
|
|
455
|
+
return apply_mapping(declared, records, entities)
|
|
456
|
+
|
|
457
|
+
|
|
449
458
|
def _refused(via: str, listed: list, reason: str, redact: str, dry_run: bool, keep_rows: bool) -> dict:
|
|
450
459
|
report = {
|
|
451
460
|
"via": via,
|
|
@@ -729,7 +729,7 @@ def apply_mapping(mapping: Mapping, records: Iterable[Any], entities: Entities)
|
|
|
729
729
|
candidates = list(by_name.get(ANY_SOURCE, []))
|
|
730
730
|
if tool_calls and isinstance(tool, str) and tool:
|
|
731
731
|
candidates = by_name.get(tool, []) + candidates
|
|
732
|
-
record = {**raw, "args": raw.get("args") if isinstance(raw.get("args"), dict) else {}}
|
|
732
|
+
record = {**raw, "args": raw.get("args") if isinstance(raw.get("args"), dict) else {}} if tool_calls else raw
|
|
733
733
|
meta = record_meta(record, index)
|
|
734
734
|
landed = 0
|
|
735
735
|
try:
|