@withgateway/sdk 0.9.1 → 0.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/dist/agents/index.d.mts +12 -4
  2. package/dist/agents/index.d.ts +12 -4
  3. package/dist/agents/index.js +30 -0
  4. package/dist/agents/index.js.map +1 -1
  5. package/dist/agents/index.mjs +30 -0
  6. package/dist/agents/index.mjs.map +1 -1
  7. package/dist/cli.js +2291 -945
  8. package/dist/cli.js.map +1 -1
  9. package/dist/cli.mjs +2290 -944
  10. package/dist/cli.mjs.map +1 -1
  11. package/dist/files/index.js.map +1 -1
  12. package/dist/files/index.mjs.map +1 -1
  13. package/dist/index.js +325 -184
  14. package/dist/index.js.map +1 -1
  15. package/dist/index.mjs +324 -183
  16. package/dist/index.mjs.map +1 -1
  17. package/dist/traces/index.js.map +1 -1
  18. package/dist/traces/index.mjs.map +1 -1
  19. package/dist/tracing/index.js +1 -1
  20. package/dist/tracing/index.js.map +1 -1
  21. package/dist/tracing/index.mjs +1 -1
  22. package/dist/tracing/index.mjs.map +1 -1
  23. package/dist/worlds/index.d.mts +3 -1
  24. package/dist/worlds/index.d.ts +3 -1
  25. package/dist/worlds/index.js +178 -67
  26. package/dist/worlds/index.js.map +1 -1
  27. package/dist/worlds/index.mjs +177 -66
  28. package/dist/worlds/index.mjs.map +1 -1
  29. package/package.json +1 -1
  30. package/runtime/gatewaysdk/environment/bundle_manifest.py +297 -9
  31. package/runtime/gatewaysdk/environment/factory_documents.py +670 -0
  32. package/runtime/gatewaysdk/environment/factory_rules.py +973 -0
  33. package/runtime/gatewaysdk/environment/factory_shapes.py +253 -0
  34. package/runtime/gatewaysdk/environment/factory_spine.py +1155 -0
  35. package/runtime/gatewaysdk/environment/factory_spine_shapes.py +480 -0
  36. package/runtime/gatewaysdk/environment/schema/admission.py +61 -59
  37. package/runtime/gatewaysdk/environment/schema/cast.py +133 -67
  38. package/runtime/gatewaysdk/environment/schema/compiler.py +16 -17
  39. package/runtime/gatewaysdk/environment/schema/handler_errors.py +58 -4
  40. package/runtime/gatewaysdk/environment/schema/host.py +21 -2
  41. package/runtime/gatewaysdk/environment/schema/host_control.py +3 -1
  42. package/runtime/gatewaysdk/environment/schema/host_loops.py +27 -4
  43. package/runtime/gatewaysdk/environment/schema/host_metrics.py +69 -0
  44. package/runtime/gatewaysdk/environment/schema/host_requests.py +21 -0
  45. package/runtime/gatewaysdk/environment/schema/host_session.py +7 -0
  46. package/runtime/gatewaysdk/environment/schema/host_versions.py +4 -0
  47. package/runtime/gatewaysdk/environment/schema/host_workers.py +172 -50
  48. package/runtime/gatewaysdk/environment/schema/ingest.py +12 -3
  49. package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
  50. package/runtime/gatewaysdk/environment/schema/measured_tests.py +187 -0
  51. package/runtime/gatewaysdk/environment/schema/platform.py +57 -3
  52. package/runtime/gatewaysdk/environment/schema/platform_query.py +4 -3
  53. package/runtime/gatewaysdk/environment/schema/platform_templates.py +105 -17
  54. package/runtime/gatewaysdk/environment/schema/process_memory.py +76 -0
  55. package/runtime/gatewaysdk/environment/schema/redaction.py +13 -0
  56. package/runtime/gatewaysdk/environment/schema/runtime.py +14 -0
  57. package/runtime/gatewaysdk/environment/schema/scaffold.py +21 -4
  58. package/runtime/gatewaysdk/environment/schema/shape_catalog.py +493 -0
  59. package/runtime/gatewaysdk/environment/schema/shape_declarations.py +219 -0
  60. package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
  61. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +1 -0
  62. package/runtime/gatewaysdk/environment/schema/skills/factory-orchestrator/SKILL.md +147 -0
  63. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +5 -1
  64. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +4 -4
  65. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +2 -2
  66. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +1 -0
  67. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/shape-catalog.md +59 -0
  68. package/runtime/gatewaysdk/environment/schema/validation.py +14 -0
  69. package/runtime/gatewaysdk/environment/schema/worker.py +7 -4
  70. package/runtime/gatewaysdk/environment/schema/worker_key.py +38 -0
  71. package/runtime/gatewaysdk/environment/schema/worker_peaks.py +38 -0
  72. package/runtime/gatewaysdk/environment/schema/worker_watch.py +50 -0
  73. package/runtime/gatewaysdk/environment/schema/workers.py +237 -68
  74. package/runtime/gatewaysdk/environment/schema/world_host.py +56 -62
  75. package/runtime/gatewaysdk/environment/schema/world_tests.py +3 -1
  76. package/skills/connector-schema-authoring/SKILL.md +1 -0
  77. package/skills/connector-schema-authoring/connector-worlds.md +1 -0
  78. package/skills/connector-schema-authoring/world-schema.md +7 -0
  79. package/skills/factory-orchestrator/SKILL.md +147 -0
  80. package/skills/world-data-ingestion/ROWS.md +5 -1
  81. package/skills/world-data-ingestion/SKILL.md +4 -4
  82. package/skills/worlds-getting-started/SKILL.md +2 -2
  83. package/skills/worlds-getting-started/references/cli-mcp-map.md +1 -0
  84. package/skills/worlds-getting-started/references/shape-catalog.md +59 -0
@@ -39,6 +39,27 @@ def worker_snapshot(snapshot: dict[str, str] | None) -> dict[str, str] | None:
39
39
  return {key: value for key, value in snapshot.items() if key != "url"} if snapshot else None
40
40
 
41
41
 
42
+ MAX_MEMORY_PEAK_MB = 1 << 20
43
+
44
+
45
+ def parse_memory_peak(value: Any) -> int | None:
46
+ """An open's ``memoryPeakMb``: the most memory the platform measured the session's world to need, a whole
47
+ MiB from 1 to 1 TiB; None when it has not measured one."""
48
+ if value is None:
49
+ return None
50
+ if isinstance(value, bool) or not isinstance(value, int) or not 0 < value <= MAX_MEMORY_PEAK_MB:
51
+ fail("host", "memoryPeakMb is a whole number of MiB from 1 to 1048576")
52
+ return value
53
+
54
+
55
+ def parse_world_state(value: Any) -> bool | None:
56
+ """An open's ``worldState``: whether the session is on one of its world's states, None when the open names
57
+ none (a platform from before a version's states had a worker of their own)."""
58
+ if value is None or isinstance(value, bool):
59
+ return value
60
+ fail("host", "worldState is true or false")
61
+
62
+
42
63
  def parse_dependency_refs(dependencies: Any) -> list[DependencyRef]:
43
64
  if dependencies is None:
44
65
  return []
@@ -13,6 +13,7 @@ from .door import OPEN_MODE, VENDOR_MODE, Door, same
13
13
  from .host_surfaces import SessionHost, capability_host, session_host
14
14
  from .request_ledger import LedgerLife
15
15
  from .session_move import SessionMove
16
+ from .worker_key import WorkerKey
16
17
 
17
18
  if TYPE_CHECKING:
18
19
  from .api import ApiSettings, WorldApp
@@ -85,6 +86,7 @@ class Session:
85
86
  recoveries: int = field(default=0, init=False)
86
87
  close_reason: str = field(default="closed", init=False)
87
88
  unrecovered: bool = field(default=False, init=False)
89
+ world_state: bool = False
88
90
 
89
91
  def __post_init__(self, pending: bool) -> None:
90
92
  self.move = SessionMove(self.session_id, pending=pending)
@@ -94,6 +96,11 @@ class Session:
94
96
  world serves browsers: from the world's declared settings, never its worker."""
95
97
  return None if self.settings is None else self.settings.cors.preflight(method, headers)
96
98
 
99
+ @property
100
+ def key(self) -> WorkerKey:
101
+ """Which of its version's lanes serves the session: its turns, and its worker when it has one."""
102
+ return WorkerKey.of(self.version_id, self.world_state)
103
+
97
104
  @property
98
105
  def app(self) -> WorldApp | None:
99
106
  """The world's routes over this session's store, when this process
@@ -142,6 +142,10 @@ class CachedVersion:
142
142
  def needs_worker(self) -> bool:
143
143
  return self.authored_code is not None
144
144
 
145
+ @property
146
+ def world(self) -> str:
147
+ return self.runtime_class.world_name(self)
148
+
145
149
  def served_names(self, dependencies: dict[str, DependencyWorld]) -> ServedNames:
146
150
  """The tools a session task may allow, beside the routes it may not."""
147
151
  return ServedNames.of(self.runtime_class.served_by(self, dependencies), self.spec)
@@ -17,7 +17,17 @@ forwarded request answers when the worker does not answer it:
17
17
  that. A worker that died idle is replaced the same way by the next request or the
18
18
  lease sweeper, which also recycles any worker holding a turn past ``STUCK_TURN_S``,
19
19
  its session still open here or not. What a worker answers about its turns is read
20
- as untrusted: it runs tenant code.
20
+ as untrusted: it runs tenant code;
21
+ * world code raised ``MemoryError`` there (the worker's own mark on that answer,
22
+ :class:`handler_errors.OutOfMemory`, which no data it serves can set): that answer goes
23
+ back as it is, and the worker, if the kernel counts it at ``POISONED_HELD_FRACTION`` of its
24
+ cap or more, is poisoned. It takes no new request, the requests already
25
+ on it get up to ``POISONED_SETTLE_S`` to finish, and it is then recycled like a stuck one,
26
+ at most once per ``POISONED_BACKOFF_S`` for its key; the next request waits for the fresh
27
+ worker, and an open that met the poison is told to come back (``busy``).
28
+
29
+ A version's sessions on its world states have a worker of their own beside main's
30
+ (:class:`WorkerKey`), so a state that exhausts its worker recycles that worker only.
21
31
 
22
32
  Recovery never raises: a session that cannot be reopened leaves the registry (the
23
33
  platform reads it as lost and reopens it from its last export), never left on a dead
@@ -33,17 +43,21 @@ import shutil
33
43
  import threading
34
44
  import time
35
45
  import urllib.parse
36
- from collections.abc import Callable
46
+ from collections import Counter
47
+ from collections.abc import Callable, Iterator
48
+ from contextlib import contextmanager
37
49
  from pathlib import Path
38
50
  from typing import TYPE_CHECKING, Any, Protocol
39
51
 
40
52
  from .admission import BUSY, RETRY_AFTER_S
41
53
  from .call_log import CallJournal
42
- from .handler_errors import log_line
54
+ from .handler_errors import OutOfMemory, log_line
55
+ from .process_memory import SIZE
43
56
  from .lanes import Lanes
44
57
  from .request_ledger import DRAIN_ANSWER_BYTES, LEFT_JOURNAL_BYTES
45
58
  from .session_files import LEDGER_JOURNAL, RECOVERED_DIR, ExportFile, clear, real_dir
46
59
  from .validation import ContractError, fail
60
+ from .worker_key import WorkerKey
47
61
  from .workers import HEALTH_TIMEOUT_S, Worker, WorkerCapacity, WorkerGone, WorkerPool, WorkerTimeout, WorkerUnreached
48
62
 
49
63
  if TYPE_CHECKING:
@@ -66,7 +80,16 @@ OPEN_TIMEOUT_S = 60.0
66
80
  EXPORT_TIMEOUT_S = 300.0
67
81
  #: A turn held this long serves no request the supervisor still waits on: its worker is recycled.
68
82
  STUCK_TURN_S = 3 * CALL_TIMEOUT_S
83
+ #: What a poisoned worker's requests already in flight get to finish before it is recycled under them.
84
+ POISONED_SETTLE_S = 5.0
85
+ #: A key whose worker was recycled for running out of memory is not recycled for it again sooner.
86
+ POISONED_BACKOFF_S = 60.0
87
+ #: A worker that ran out of memory is recycled only while it holds this much of its cap: one request asking
88
+ #: for more than the cap is refused its allocation and leaves the process as it was.
89
+ POISONED_HELD_FRACTION = 0.75
90
+ WORKERS_POISONED = "workers_poisoned"
69
91
  RUNNING_RETRY_AFTER_S = 5
92
+ OUT_OF_MEMORY_RETRY = "the world's worker ran out of memory and is being replaced; retry"
70
93
  JSON = {"content-type": "application/json"}
71
94
  #: An open refused ``draining`` or ``at_capacity`` (503) goes to another replica; any other 503, ``busy``
72
95
  #: for an id still closing included, sends the platform back later; 404 is the one answer it reads as a loss.
@@ -94,6 +117,10 @@ def refusal(error: ContractError) -> tuple[int, dict, dict[str, str]]:
94
117
  return REFUSALS.get(code, 400), {**body, "retryAfter": retry}, {"Retry-After": str(retry)}
95
118
 
96
119
 
120
+ def _busy(error: BaseException) -> bool:
121
+ return isinstance(error, ContractError) and error.detail.get("code") == BUSY
122
+
123
+
97
124
  def _age(value: object) -> float | None:
98
125
  """A turn's age as a worker reported it, or None unless it is a finite, non-negative number."""
99
126
  if isinstance(value, bool) or not isinstance(value, (int, float)):
@@ -130,45 +157,74 @@ class VersionWorkers:
130
157
  self._claims = threading.Lock()
131
158
  self._recoveries = Lanes()
132
159
  self._overdue: dict[tuple[int, str], Worker] = {}
160
+ self._forwarding = threading.Condition()
161
+ self._inflight: Counter[int] = Counter()
162
+ self._poisoned_at: dict[WorkerKey, float] = {}
133
163
 
134
- def home(self, version_id: str) -> Path:
135
- """Where a version's worker runs: its own ``WorldHost`` cache root (:mod:`worker`)."""
136
- return self.pool.workers_root / version_id
164
+ def home(self, key: WorkerKey) -> Path:
165
+ """Where a key's worker runs: its own ``WorldHost`` cache root (:mod:`worker`)."""
166
+ return key.home(self.pool.workers_root)
137
167
 
138
- def session_dir(self, version_id: str, session_id: str) -> Path:
139
- return self.home(version_id) / "sessions" / session_id
168
+ def session_dir(self, key: WorkerKey, session_id: str) -> Path:
169
+ return self.home(key) / "sessions" / session_id
140
170
 
141
171
  def place(
142
- self, version_id: str, session_id: str, body: dict, fetch: Callable[[Path], ExportFile | None], opens: Lanes
172
+ self,
173
+ key: WorkerKey,
174
+ session_id: str,
175
+ body: dict,
176
+ fetch: Callable[[Path], ExportFile | None],
177
+ opens: Lanes,
178
+ world: str,
179
+ need_mb: int = 0,
143
180
  ) -> Worker:
144
- """Open ``session_id`` on its version's worker with ``body``, claimed before placement so
181
+ """Open ``session_id`` on its key's worker with ``body``, claimed before placement so
145
182
  another open cannot stop the worker as session-less meanwhile. ``fetch`` downloads an
146
183
  export to restore into the session's directory once the worker runs (a worker clears
147
- that directory when it starts); the worker reads it by path. One open per version at a
148
- time (``opens``), so the worker's open timeout counts one open, never a queue."""
149
- worker = self._worker_for(version_id)
184
+ that directory when it starts); the worker reads it by path. One open per key at a
185
+ time (``opens``), so the worker's open timeout counts one open, never a queue. ``world``
186
+ labels the worker in what the host logs about it; ``need_mb`` is what the session's world was
187
+ measured to need of it (:class:`MemoryNeed`), 0 when unmeasured."""
188
+ worker = self._worker_for(key, world, need_mb)
150
189
  self._claim(worker, session_id)
151
190
  source = None
152
191
  try:
153
- source = fetch(self.session_dir(version_id, session_id))
192
+ source = fetch(self.session_dir(key, session_id))
154
193
  placement = {**body, "restore": source.for_worker() if source is not None else None}
155
194
  self._open_on(worker, session_id, placement, opens)
156
- except BaseException:
195
+ self._refuse_poisoned(worker, session_id)
196
+ except BaseException as error:
157
197
  self._release(worker, session_id)
198
+ if worker.poisoned and not _busy(error):
199
+ raise ContractError(BUSY, OUT_OF_MEMORY_RETRY, retryAfter=RETRY_AFTER_S) from error
158
200
  raise
159
201
  finally:
160
202
  if source is not None:
161
203
  source.discard()
162
204
  return worker
163
205
 
206
+ def _refuse_poisoned(self, worker: Worker, session_id: str) -> None:
207
+ """An open that landed on a worker poisoned meanwhile is not on any recovery's list: it is deleted
208
+ there rather than kept on a worker about to stop."""
209
+ if not worker.poisoned:
210
+ return
211
+ self._delete(worker, session_id)
212
+ fail(BUSY, OUT_OF_MEMORY_RETRY, retryAfter=RETRY_AFTER_S)
213
+
164
214
  def _open_on(self, worker: Worker, session_id: str, placement: dict, opens: Lanes) -> None:
165
215
  """The open on ``worker``, in its version's turn. One that raised after reaching the worker may still
166
216
  complete there (an answer past the cut-off) or have left a copy, which no open of the id may find
167
217
  later, so it is deleted there, an open still running abandoned, before the caller lets the claim go.
168
218
  One that never reached it (the refusal's cause a :class:`WorkerUnreached`) left nothing to delete."""
219
+ raw = json.dumps(placement).encode("utf-8")
169
220
  try:
170
- with opens.turn(worker.version_id):
171
- self._control(worker, "POST", "/__host/sessions", placement, OPEN_TIMEOUT_S, "the open")
221
+ with opens.turn(worker.key.name):
222
+ status, headers, payload = self._forward(
223
+ worker, "POST", "/__host/sessions", JSON, raw, True, OPEN_TIMEOUT_S, "the open"
224
+ )
225
+ if OutOfMemory.in_headers(headers) and (worker.poisoned or self._exhausted(worker)):
226
+ fail(BUSY, OUT_OF_MEMORY_RETRY, retryAfter=RETRY_AFTER_S)
227
+ self._answer(status, payload)
172
228
  except Exception as error:
173
229
  if not isinstance(error.__cause__, WorkerUnreached):
174
230
  self._delete(worker, session_id)
@@ -216,12 +272,12 @@ class VersionWorkers:
216
272
 
217
273
  def live(self, session: Session) -> Worker:
218
274
  """The session's worker, replaced first (and the session reopened on the new one) if it
219
- died; a session no live worker holds afterwards is forgotten and answers ``unknown_session``.
220
- A session that began closing meanwhile is served like any other: whoever then deletes it
221
- on the new worker has drained the entries it resumed first."""
275
+ died or was poisoned; a session no live worker holds afterwards is forgotten and answers
276
+ ``unknown_session``. A session that began closing meanwhile is served like any other:
277
+ whoever then deletes it on the new worker has drained the entries it resumed first."""
222
278
  worker = session.worker
223
279
  assert worker is not None
224
- if worker.alive():
280
+ if worker.serves():
225
281
  return worker
226
282
  self.recover(worker)
227
283
  current = session.worker
@@ -237,7 +293,7 @@ class VersionWorkers:
237
293
  assert worker is not None
238
294
  self._release(worker, session.session_id)
239
295
  if not worker.alive():
240
- return self._hand_over(session, self.session_dir(session.version_id, session.session_id))
296
+ return self._hand_over(session, self.session_dir(worker.key, session.session_id))
241
297
  self._delete(worker, session.session_id)
242
298
 
243
299
  def _delete(self, worker: Worker, session_id: str) -> None:
@@ -268,7 +324,7 @@ class VersionWorkers:
268
324
  """The file a worker's export landed in, admitted only from inside that worker's
269
325
  directory for this session: the worker runs tenant code, so what it names is
270
326
  checked, and the bytes are hashed by the supervisor, not trusted."""
271
- exports = self.session_dir(session.version_id, session.session_id) / "exports"
327
+ exports = self.session_dir(session.key, session.session_id) / "exports"
272
328
  path = Path(str(answer.get("path") or ""))
273
329
  try:
274
330
  resolved = path.resolve(strict=True)
@@ -278,27 +334,29 @@ class VersionWorkers:
278
334
  return resolved
279
335
 
280
336
  def recover(self, dead: Worker, reason: str = "workers_replaced") -> None:
281
- """Replace ``dead`` (stopped first if it still runs) and reopen every session it held on
282
- the new worker from what each committed; a session that cannot be reopened leaves the
283
- registry (:meth:`_reopen_failed`). One recovery per version at a time; a later caller finds
284
- it done. Never raises. Every session it takes is its own until it gives the session back
285
- (:meth:`_give_back`)."""
337
+ """Replace ``dead`` (stopped first if it still runs; a poisoned one once the requests already on it
338
+ are answered or ``POISONED_SETTLE_S`` passes) and reopen every session it held on the new worker
339
+ from what each committed; a session that cannot be reopened leaves the registry
340
+ (:meth:`_reopen_failed`). One recovery per key at a time; a later caller finds it done. Never
341
+ raises. Every session it takes is its own until it gives the session back (:meth:`_give_back`)."""
286
342
  held: list[Session] = []
287
343
  try:
288
- with self._recoveries.turn(dead.version_id):
289
- if self.pool.peek(dead.version_id) is not dead:
344
+ with self._recoveries.turn(dead.key.name):
345
+ if self.pool.peek(dead.key) is not dead:
290
346
  return
347
+ self._settle(dead)
348
+ self._outgrown(dead)
291
349
  self._stop(dead)
292
350
  sessions, closing = self.registry.recovering(dead)
293
351
  held = [*sessions, *closing]
294
- aside = self._set_aside(dead.version_id, held)
352
+ aside = self._set_aside(dead.key, held)
295
353
  fresh = self._replacement(dead)
296
354
  for session in closing:
297
355
  self._hand_over(session, aside.get(session.session_id))
298
356
  for session in sessions:
299
357
  self._reopen(fresh, session, aside.get(session.session_id))
300
358
  self._forget_overdue(lambda worker: worker is dead)
301
- self.registry.tally(reason)
359
+ self.registry.tally(WORKERS_POISONED if dead.poisoned else reason)
302
360
  finally:
303
361
  self._give_back(held)
304
362
 
@@ -313,21 +371,21 @@ class VersionWorkers:
313
371
  log_line(f"session {session.session_id} did not leave its new worker: {type(error).__name__}")
314
372
 
315
373
  def recover_dead(self) -> list[str]:
316
- """Recover every worker that died holding sessions; returns their versions."""
374
+ """Recover every worker that died holding sessions; returns their keys' names."""
317
375
  dead = self.pool.dead()
318
376
  for worker in dead:
319
377
  self.recover(worker)
320
- return [worker.version_id for worker in dead]
378
+ return [worker.key.name for worker in dead]
321
379
 
322
380
  def recover_stuck(self) -> list[str]:
323
381
  """Recycle every live worker holding a turn past ``STUCK_TURN_S`` (a handler that never
324
382
  returned), whether or not that turn's session is still open here: its sessions reopen
325
- from what they committed, and an id it still held opens again. Returns their versions."""
383
+ from what they committed, and an id it still held opens again. Returns their keys' names."""
326
384
  self._forget_overdue(lambda worker: not worker.alive())
327
385
  stuck = [worker for worker in self.pool.live() if max(self._turns(worker).values(), default=0) > STUCK_TURN_S]
328
386
  for worker in stuck:
329
387
  self.recover(worker, "workers_recycled")
330
- return [worker.version_id for worker in stuck]
388
+ return [worker.key.name for worker in stuck]
331
389
 
332
390
  def _serving(self, session: Session) -> Worker:
333
391
  """The session's live worker, refused ``unknown_session`` once its recovery could not reopen it (it
@@ -371,36 +429,49 @@ class VersionWorkers:
371
429
  ages = {str(sid): _age(age) for sid, age in turns.items()}
372
430
  return {sid: age for sid, age in ages.items() if age is not None}
373
431
 
432
+ def _outgrown(self, worker: Worker) -> None:
433
+ """A poisoned worker's replacement is sized by what it was seen to hold, read before it stops."""
434
+ if not worker.poisoned:
435
+ return
436
+ self.pool.outgrown(worker)
437
+
438
+ def _settle(self, worker: Worker) -> None:
439
+ """A poisoned worker's requests already forwarded get up to ``POISONED_SETTLE_S`` to be answered."""
440
+ if not worker.poisoned:
441
+ return
442
+ with self._forwarding:
443
+ self._forwarding.wait_for(lambda: not self._inflight[id(worker)], timeout=POISONED_SETTLE_S)
444
+
374
445
  @staticmethod
375
446
  def _stop(dead: Worker) -> None:
376
447
  try:
377
448
  dead.stop()
378
449
  except Exception as error: # noqa: BLE001 - a worker that will not stop is replaced all the same
379
- log_line(f"worker for version {dead.version_id} did not stop: {type(error).__name__}")
450
+ log_line(f"worker for {dead.key.name} did not stop: {type(error).__name__}")
380
451
 
381
452
  def _replacement(self, dead: Worker) -> Worker | None:
382
453
  try:
383
454
  return self.pool.replace(dead)
384
455
  except Exception as error: # noqa: BLE001 - its sessions are lost either way; the platform reopens them
385
- log_line(f"worker for version {dead.version_id} could not be replaced: {type(error).__name__}")
456
+ log_line(f"worker for {dead.key.name} could not be replaced: {type(error).__name__}")
386
457
  return None
387
458
 
388
- def _set_aside(self, version_id: str, sessions: list[Session]) -> dict[str, Path]:
459
+ def _set_aside(self, key: WorkerKey, sessions: list[Session]) -> dict[str, Path]:
389
460
  """Each session's directory moved under the worker's ``recovered/``, before a new worker
390
461
  clears its sessions directory; a session whose directory cannot be moved is left out."""
391
- root = self.home(version_id) / RECOVERED_DIR
462
+ root = self.home(key) / RECOVERED_DIR
392
463
  try:
393
464
  if not real_dir(root):
394
465
  clear(root)
395
466
  root.mkdir(mode=0o700, exist_ok=True)
396
467
  except OSError as error:
397
- log_line(f"no directory to set version {version_id}'s sessions aside in: {type(error).__name__}")
468
+ log_line(f"no directory to set {key.name}'s sessions aside in: {type(error).__name__}")
398
469
  return {}
399
- moved = {session.session_id: self._move_aside(root, version_id, session.session_id) for session in sessions}
470
+ moved = {session.session_id: self._move_aside(root, key, session.session_id) for session in sessions}
400
471
  return {session_id: directory for session_id, directory in moved.items() if directory is not None}
401
472
 
402
- def _move_aside(self, root: Path, version_id: str, session_id: str) -> Path | None:
403
- source = self.session_dir(version_id, session_id)
473
+ def _move_aside(self, root: Path, key: WorkerKey, session_id: str) -> Path | None:
474
+ source = self.session_dir(key, session_id)
404
475
  target = root / session_id
405
476
  try:
406
477
  if not (real_dir(root) and real_dir(source.parent) and real_dir(source)):
@@ -418,9 +489,10 @@ class VersionWorkers:
418
489
  body = {**session.placement, "restore": {"recovered": str(directory)}}
419
490
  self._claim(fresh, session.session_id)
420
491
  try:
421
- status, _, payload = fresh.request(
492
+ status, headers, payload = fresh.request(
422
493
  "POST", "/__host/sessions", JSON, json.dumps(body).encode("utf-8"), control=True, timeout=OPEN_TIMEOUT_S
423
494
  )
495
+ self._watch(fresh, headers)
424
496
  self._answer(status, payload)
425
497
  except Exception as error: # noqa: BLE001 - one session that cannot be reopened never strands the rest
426
498
  return self._reopen_failed(fresh, session, directory, error)
@@ -471,12 +543,12 @@ class VersionWorkers:
471
543
  finally:
472
544
  shutil.rmtree(directory, ignore_errors=True)
473
545
 
474
- def _worker_for(self, version_id: str) -> Worker:
475
- current = self.pool.peek(version_id)
476
- if current is not None and not current.alive():
546
+ def _worker_for(self, key: WorkerKey, world: str, need_mb: int) -> Worker:
547
+ current = self.pool.peek(key)
548
+ if current is not None and not current.serves():
477
549
  self.recover(current)
478
550
  try:
479
- return self.pool.get(version_id)
551
+ return self.pool.get(key, world=world, need_mb=need_mb)
480
552
  except WorkerCapacity as error:
481
553
  fail("at_capacity", str(error))
482
554
 
@@ -529,9 +601,59 @@ class VersionWorkers:
529
601
  limit: int | None = None,
530
602
  ) -> tuple[int, list[tuple[str, str]], bytes]:
531
603
  try:
532
- return worker.request(method, path, headers, raw, control=control, timeout=timeout, limit=limit)
604
+ with self._forwarded(worker):
605
+ answer = worker.request(method, path, headers, raw, control=control, timeout=timeout, limit=limit)
533
606
  except WorkerGone as error:
534
607
  raise self._unanswered(worker, error, what, timeout, session_id) from error
608
+ self._watch(worker, answer[1])
609
+ return answer
610
+
611
+ def _watch(self, worker: Worker, headers: list[tuple[str, str]]) -> None:
612
+ if OutOfMemory.in_headers(headers):
613
+ self._poison(worker)
614
+
615
+ @contextmanager
616
+ def _forwarded(self, worker: Worker) -> Iterator[None]:
617
+ with self._forwarding:
618
+ self._inflight[id(worker)] += 1
619
+ try:
620
+ yield
621
+ finally:
622
+ with self._forwarding:
623
+ self._inflight[id(worker)] -= 1
624
+ if self._inflight[id(worker)] <= 0:
625
+ del self._inflight[id(worker)]
626
+ self._forwarding.notify_all()
627
+
628
+ def _exhausted(self, worker: Worker) -> bool:
629
+ """Whether the kernel counts ``worker`` at ``POISONED_HELD_FRACTION`` of its address-space cap or more
630
+ (``VmSize``); a worker the kernel cannot say of (no ``/proc``) is taken at its word."""
631
+ size = self.pool.memory.status(worker.pid).mb(SIZE)
632
+ return size is None or size >= worker.cap_mb * POISONED_HELD_FRACTION
633
+
634
+ def _poison(self, worker: Worker) -> None:
635
+ """World code on ``worker`` raised ``MemoryError`` while it holds most of its memory: it takes no new
636
+ request and is recycled behind this answer, once, with a line naming its world and version; not when
637
+ its key was recycled for this within ``POISONED_BACKOFF_S``, so a lane that keeps running out cuts its
638
+ calls once a minute at most, and never for a request that only asked for more than the cap."""
639
+ if not self._exhausted(worker):
640
+ return
641
+ now = time.monotonic()
642
+ with self._claims:
643
+ backed_off = now - self._poisoned_at.get(worker.key, -math.inf) < POISONED_BACKOFF_S
644
+ first = not worker.poisoned and not backed_off
645
+ if first:
646
+ worker.poisoned = True
647
+ self._poisoned_at[worker.key] = now
648
+ if not first:
649
+ return
650
+ log_line(
651
+ f"world {worker.world!r} version {worker.version_id} ({worker.key.lane}): out of memory in its worker "
652
+ f"(pid {worker.pid}); recycling it"
653
+ )
654
+ threading.Thread(
655
+ target=self.recover, args=(worker, WORKERS_POISONED), name="world-worker-recycle", daemon=True
656
+ ).start()
535
657
 
536
658
  def _unanswered(
537
659
  self, worker: Worker, error: WorkerGone, what: str, timeout: float, session_id: str | None
@@ -407,9 +407,7 @@ def ingest_records(
407
407
  else:
408
408
  fail("ingest", "mapping is an ingest.toml text, a parsed table, a Mapping or a file path", "/mapping")
409
409
  if declared is not None:
410
- entities = _entities(root)
411
- validate_mapping(declared, entities)
412
- projected, report = apply_mapping(declared, listed, entities)
410
+ projected, report = map_records(root, listed, declared)
413
411
  else:
414
412
  spec = load_spec(root) if (root / SPEC).is_file() else None
415
413
  if spec is None or not any(operation.entity for operation in spec.operations.values()):
@@ -446,6 +444,17 @@ def ingest_records(
446
444
  return report
447
445
 
448
446
 
447
+ def map_records(root: str | Path, records: list, mapping: Any) -> tuple[dict[str, list[dict]], dict]:
448
+ """``records`` through a declared ``ingest.toml`` against the world's contract, nothing imported:
449
+ the rows and the mapping's report, for a caller that lands the rows through its own gate."""
450
+ from .mapping import Mapping, apply_mapping, parse_mapping, validate_mapping
451
+
452
+ declared = mapping if isinstance(mapping, Mapping) else parse_mapping(mapping)
453
+ entities = _entities(Path(root))
454
+ validate_mapping(declared, entities)
455
+ return apply_mapping(declared, records, entities)
456
+
457
+
449
458
  def _refused(via: str, listed: list, reason: str, redact: str, dry_run: bool, keep_rows: bool) -> dict:
450
459
  report = {
451
460
  "via": via,
@@ -729,7 +729,7 @@ def apply_mapping(mapping: Mapping, records: Iterable[Any], entities: Entities)
729
729
  candidates = list(by_name.get(ANY_SOURCE, []))
730
730
  if tool_calls and isinstance(tool, str) and tool:
731
731
  candidates = by_name.get(tool, []) + candidates
732
- record = {**raw, "args": raw.get("args") if isinstance(raw.get("args"), dict) else {}}
732
+ record = {**raw, "args": raw.get("args") if isinstance(raw.get("args"), dict) else {}} if tool_calls else raw
733
733
  meta = record_meta(record, index)
734
734
  landed = 0
735
735
  try: