@withgateway/sdk 0.6.0 → 0.9.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/dist/agents/index.d.mts +581 -0
  2. package/dist/agents/index.d.ts +581 -0
  3. package/dist/agents/index.js +1473 -0
  4. package/dist/agents/index.js.map +1 -0
  5. package/dist/agents/index.mjs +1424 -0
  6. package/dist/agents/index.mjs.map +1 -0
  7. package/dist/apiError-DHpfbmdx.d.mts +17 -0
  8. package/dist/apiError-DHpfbmdx.d.ts +17 -0
  9. package/dist/cli.js +11611 -6845
  10. package/dist/cli.js.map +1 -1
  11. package/dist/cli.mjs +10090 -5325
  12. package/dist/cli.mjs.map +1 -1
  13. package/dist/files/index.d.mts +3 -1
  14. package/dist/files/index.d.ts +3 -1
  15. package/dist/files/index.js +371 -102
  16. package/dist/files/index.js.map +1 -1
  17. package/dist/files/index.mjs +371 -102
  18. package/dist/files/index.mjs.map +1 -1
  19. package/dist/hooks/index.js +346 -136
  20. package/dist/hooks/index.js.map +1 -1
  21. package/dist/hooks/index.mjs +346 -136
  22. package/dist/hooks/index.mjs.map +1 -1
  23. package/dist/{index-LA-PMRJT.d.ts → index-DIN14Lhg.d.ts} +1 -1
  24. package/dist/{index-Di9nNqOY.d.mts → index-DuB-eQ0Z.d.mts} +1 -1
  25. package/dist/index.d.mts +7 -4
  26. package/dist/index.d.ts +7 -4
  27. package/dist/index.js +3177 -748
  28. package/dist/index.js.map +1 -1
  29. package/dist/index.mjs +2987 -590
  30. package/dist/index.mjs.map +1 -1
  31. package/dist/media-CQUC7TkS.d.mts +39 -0
  32. package/dist/media-DQnBloaX.d.ts +39 -0
  33. package/dist/traces/index.d.mts +3 -1
  34. package/dist/traces/index.d.ts +3 -1
  35. package/dist/traces/index.js +371 -102
  36. package/dist/traces/index.js.map +1 -1
  37. package/dist/traces/index.mjs +371 -102
  38. package/dist/traces/index.mjs.map +1 -1
  39. package/dist/tracing/index.d.mts +3 -2
  40. package/dist/tracing/index.d.ts +3 -2
  41. package/dist/tracing/index.js +257 -16
  42. package/dist/tracing/index.js.map +1 -1
  43. package/dist/tracing/index.mjs +257 -16
  44. package/dist/tracing/index.mjs.map +1 -1
  45. package/dist/transport-CpRe7zgd.d.mts +123 -0
  46. package/dist/transport-CpRe7zgd.d.ts +123 -0
  47. package/dist/worlds/index.d.mts +1986 -1746
  48. package/dist/worlds/index.d.ts +1986 -1746
  49. package/dist/worlds/index.js +1470 -290
  50. package/dist/worlds/index.js.map +1 -1
  51. package/dist/worlds/index.mjs +1465 -286
  52. package/dist/worlds/index.mjs.map +1 -1
  53. package/package.json +6 -1
  54. package/runtime/gatewaysdk/environment/bundle_manifest.py +531 -0
  55. package/runtime/gatewaysdk/environment/factory_documents.py +670 -0
  56. package/runtime/gatewaysdk/environment/factory_rules.py +973 -0
  57. package/runtime/gatewaysdk/environment/factory_shapes.py +253 -0
  58. package/runtime/gatewaysdk/environment/factory_spine.py +1155 -0
  59. package/runtime/gatewaysdk/environment/factory_spine_shapes.py +480 -0
  60. package/runtime/gatewaysdk/environment/schema/__main__.py +2 -2
  61. package/runtime/gatewaysdk/environment/schema/_slack_fidelity.py +8 -23
  62. package/runtime/gatewaysdk/environment/schema/admission.py +61 -59
  63. package/runtime/gatewaysdk/environment/schema/api.py +3 -4
  64. package/runtime/gatewaysdk/environment/schema/assertions.py +7 -1
  65. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +1 -0
  66. package/runtime/gatewaysdk/environment/schema/call_log.py +20 -0
  67. package/runtime/gatewaysdk/environment/schema/cast.py +133 -67
  68. package/runtime/gatewaysdk/environment/schema/compiler.py +16 -17
  69. package/runtime/gatewaysdk/environment/schema/database/base.py +2 -2
  70. package/runtime/gatewaysdk/environment/schema/handler_errors.py +58 -4
  71. package/runtime/gatewaysdk/environment/schema/host.py +94 -12
  72. package/runtime/gatewaysdk/environment/schema/host_browser.py +48 -0
  73. package/runtime/gatewaysdk/environment/schema/host_control.py +35 -2
  74. package/runtime/gatewaysdk/environment/schema/host_handoff.py +4 -1
  75. package/runtime/gatewaysdk/environment/schema/host_loops.py +27 -4
  76. package/runtime/gatewaysdk/environment/schema/host_metrics.py +69 -0
  77. package/runtime/gatewaysdk/environment/schema/host_requests.py +35 -0
  78. package/runtime/gatewaysdk/environment/schema/host_session.py +20 -2
  79. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +6 -0
  80. package/runtime/gatewaysdk/environment/schema/host_versions.py +100 -19
  81. package/runtime/gatewaysdk/environment/schema/host_workers.py +172 -50
  82. package/runtime/gatewaysdk/environment/schema/ingest.py +12 -3
  83. package/runtime/gatewaysdk/environment/schema/journal.py +7 -2
  84. package/runtime/gatewaysdk/environment/schema/local.py +32 -7
  85. package/runtime/gatewaysdk/environment/schema/mapping.py +1 -1
  86. package/runtime/gatewaysdk/environment/schema/measured_tests.py +187 -0
  87. package/runtime/gatewaysdk/environment/schema/page_checks.py +123 -0
  88. package/runtime/gatewaysdk/environment/schema/platform.py +80 -270
  89. package/runtime/gatewaysdk/environment/schema/platform_query.py +4 -3
  90. package/runtime/gatewaysdk/environment/schema/platform_templates.py +420 -0
  91. package/runtime/gatewaysdk/environment/schema/process_memory.py +76 -0
  92. package/runtime/gatewaysdk/environment/schema/redaction.py +13 -0
  93. package/runtime/gatewaysdk/environment/schema/request_ledger.py +4 -1
  94. package/runtime/gatewaysdk/environment/schema/run_parameters.py +25 -0
  95. package/runtime/gatewaysdk/environment/schema/runtime.py +57 -5
  96. package/runtime/gatewaysdk/environment/schema/scaffold.py +386 -219
  97. package/runtime/gatewaysdk/environment/schema/session_files.py +13 -2
  98. package/runtime/gatewaysdk/environment/schema/shape_catalog.py +493 -0
  99. package/runtime/gatewaysdk/environment/schema/shape_declarations.py +219 -0
  100. package/runtime/gatewaysdk/environment/schema/skill.py +1 -0
  101. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +1 -1
  102. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +2 -2
  103. package/runtime/gatewaysdk/environment/schema/skills/factory-orchestrator/SKILL.md +147 -0
  104. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +5 -1
  105. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +8 -6
  106. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +9 -9
  107. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +33 -2
  108. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +6 -6
  109. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +17 -2
  110. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/new-world-sandbox.md +1 -1
  111. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/shape-catalog.md +59 -0
  112. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/ui-map.md +3 -2
  113. package/runtime/gatewaysdk/environment/schema/slack.py +55 -10
  114. package/runtime/gatewaysdk/environment/schema/snapshot.py +28 -2
  115. package/runtime/gatewaysdk/environment/schema/tasks.py +25 -2
  116. package/runtime/gatewaysdk/environment/schema/template_record.py +131 -0
  117. package/runtime/gatewaysdk/environment/schema/tools_world.py +7 -5
  118. package/runtime/gatewaysdk/environment/schema/ui.py +54 -33
  119. package/runtime/gatewaysdk/environment/schema/ui_interactions.py +419 -0
  120. package/runtime/gatewaysdk/environment/schema/ui_recorder.js +895 -0
  121. package/runtime/gatewaysdk/environment/schema/ui_replay.py +366 -0
  122. package/runtime/gatewaysdk/environment/schema/ui_replay_record.LICENSE +25 -0
  123. package/runtime/gatewaysdk/environment/schema/ui_replay_record.js +109 -0
  124. package/runtime/gatewaysdk/environment/schema/validation.py +24 -1
  125. package/runtime/gatewaysdk/environment/schema/worker.py +12 -5
  126. package/runtime/gatewaysdk/environment/schema/worker_key.py +38 -0
  127. package/runtime/gatewaysdk/environment/schema/worker_peaks.py +38 -0
  128. package/runtime/gatewaysdk/environment/schema/worker_watch.py +50 -0
  129. package/runtime/gatewaysdk/environment/schema/workers.py +245 -69
  130. package/runtime/gatewaysdk/environment/schema/world_host.py +96 -73
  131. package/runtime/gatewaysdk/environment/schema/world_tests.py +3 -1
  132. package/runtime/gatewaysdk/environment/secret_values.py +1 -1
  133. package/runtime/gatewaysdk/environment/world.py +34 -43
  134. package/runtime/gatewaysdk/environment/world_session_contract.py +7 -34
  135. package/skills/benchmarks-getting-started/SKILL.md +1 -1
  136. package/skills/connector-schema-authoring/SKILL.md +2 -2
  137. package/skills/connector-schema-authoring/connector-worlds.md +3 -2
  138. package/skills/connector-schema-authoring/world-schema.md +36 -23
  139. package/skills/factory-orchestrator/SKILL.md +147 -0
  140. package/skills/world-data-ingestion/ROWS.md +5 -1
  141. package/skills/world-data-ingestion/SKILL.md +8 -6
  142. package/skills/worlds-getting-started/SKILL.md +9 -9
  143. package/skills/worlds-getting-started/references/cli-mcp-map.md +33 -2
  144. package/skills/worlds-getting-started/references/gotchas.md +6 -6
  145. package/skills/worlds-getting-started/references/journal-checks.md +17 -2
  146. package/skills/worlds-getting-started/references/new-world-sandbox.md +1 -1
  147. package/skills/worlds-getting-started/references/shape-catalog.md +59 -0
  148. package/skills/worlds-getting-started/references/ui-map.md +3 -2
  149. package/dist/media-BIApiPzS.d.mts +0 -102
  150. package/dist/media-BIApiPzS.d.ts +0 -102
@@ -17,7 +17,17 @@ forwarded request answers when the worker does not answer it:
17
17
  that. A worker that died idle is replaced the same way by the next request or the
18
18
  lease sweeper, which also recycles any worker holding a turn past ``STUCK_TURN_S``,
19
19
  its session still open here or not. What a worker answers about its turns is read
20
- as untrusted: it runs tenant code.
20
+ as untrusted: it runs tenant code;
21
+ * world code raised ``MemoryError`` there (the worker's own mark on that answer,
22
+ :class:`handler_errors.OutOfMemory`, which no data it serves can set): that answer goes
23
+ back as it is, and the worker, if the kernel counts it at ``POISONED_HELD_FRACTION`` of its
24
+ cap or more, is poisoned. It takes no new request, the requests already
25
+ on it get up to ``POISONED_SETTLE_S`` to finish, and it is then recycled like a stuck one,
26
+ at most once per ``POISONED_BACKOFF_S`` for its key; the next request waits for the fresh
27
+ worker, and an open that met the poison is told to come back (``busy``).
28
+
29
+ A version's sessions on its world states have a worker of their own beside main's
30
+ (:class:`WorkerKey`), so a state that exhausts its worker recycles that worker only.
21
31
 
22
32
  Recovery never raises: a session that cannot be reopened leaves the registry (the
23
33
  platform reads it as lost and reopens it from its last export), never left on a dead
@@ -33,17 +43,21 @@ import shutil
33
43
  import threading
34
44
  import time
35
45
  import urllib.parse
36
- from collections.abc import Callable
46
+ from collections import Counter
47
+ from collections.abc import Callable, Iterator
48
+ from contextlib import contextmanager
37
49
  from pathlib import Path
38
50
  from typing import TYPE_CHECKING, Any, Protocol
39
51
 
40
52
  from .admission import BUSY, RETRY_AFTER_S
41
53
  from .call_log import CallJournal
42
- from .handler_errors import log_line
54
+ from .handler_errors import OutOfMemory, log_line
55
+ from .process_memory import SIZE
43
56
  from .lanes import Lanes
44
57
  from .request_ledger import DRAIN_ANSWER_BYTES, LEFT_JOURNAL_BYTES
45
58
  from .session_files import LEDGER_JOURNAL, RECOVERED_DIR, ExportFile, clear, real_dir
46
59
  from .validation import ContractError, fail
60
+ from .worker_key import WorkerKey
47
61
  from .workers import HEALTH_TIMEOUT_S, Worker, WorkerCapacity, WorkerGone, WorkerPool, WorkerTimeout, WorkerUnreached
48
62
 
49
63
  if TYPE_CHECKING:
@@ -66,7 +80,16 @@ OPEN_TIMEOUT_S = 60.0
66
80
  EXPORT_TIMEOUT_S = 300.0
67
81
  #: A turn held this long serves no request the supervisor still waits on: its worker is recycled.
68
82
  STUCK_TURN_S = 3 * CALL_TIMEOUT_S
83
+ #: What a poisoned worker's requests already in flight get to finish before it is recycled under them.
84
+ POISONED_SETTLE_S = 5.0
85
+ #: A key whose worker was recycled for running out of memory is not recycled for it again sooner.
86
+ POISONED_BACKOFF_S = 60.0
87
+ #: A worker that ran out of memory is recycled only while it holds this much of its cap: one request asking
88
+ #: for more than the cap is refused its allocation and leaves the process as it was.
89
+ POISONED_HELD_FRACTION = 0.75
90
+ WORKERS_POISONED = "workers_poisoned"
69
91
  RUNNING_RETRY_AFTER_S = 5
92
+ OUT_OF_MEMORY_RETRY = "the world's worker ran out of memory and is being replaced; retry"
70
93
  JSON = {"content-type": "application/json"}
71
94
  #: An open refused ``draining`` or ``at_capacity`` (503) goes to another replica; any other 503, ``busy``
72
95
  #: for an id still closing included, sends the platform back later; 404 is the one answer it reads as a loss.
@@ -94,6 +117,10 @@ def refusal(error: ContractError) -> tuple[int, dict, dict[str, str]]:
94
117
  return REFUSALS.get(code, 400), {**body, "retryAfter": retry}, {"Retry-After": str(retry)}
95
118
 
96
119
 
120
+ def _busy(error: BaseException) -> bool:
121
+ return isinstance(error, ContractError) and error.detail.get("code") == BUSY
122
+
123
+
97
124
  def _age(value: object) -> float | None:
98
125
  """A turn's age as a worker reported it, or None unless it is a finite, non-negative number."""
99
126
  if isinstance(value, bool) or not isinstance(value, (int, float)):
@@ -130,45 +157,74 @@ class VersionWorkers:
130
157
  self._claims = threading.Lock()
131
158
  self._recoveries = Lanes()
132
159
  self._overdue: dict[tuple[int, str], Worker] = {}
160
+ self._forwarding = threading.Condition()
161
+ self._inflight: Counter[int] = Counter()
162
+ self._poisoned_at: dict[WorkerKey, float] = {}
133
163
 
134
- def home(self, version_id: str) -> Path:
135
- """Where a version's worker runs: its own ``WorldHost`` cache root (:mod:`worker`)."""
136
- return self.pool.workers_root / version_id
164
+ def home(self, key: WorkerKey) -> Path:
165
+ """Where a key's worker runs: its own ``WorldHost`` cache root (:mod:`worker`)."""
166
+ return key.home(self.pool.workers_root)
137
167
 
138
- def session_dir(self, version_id: str, session_id: str) -> Path:
139
- return self.home(version_id) / "sessions" / session_id
168
+ def session_dir(self, key: WorkerKey, session_id: str) -> Path:
169
+ return self.home(key) / "sessions" / session_id
140
170
 
141
171
  def place(
142
- self, version_id: str, session_id: str, body: dict, fetch: Callable[[Path], ExportFile | None], opens: Lanes
172
+ self,
173
+ key: WorkerKey,
174
+ session_id: str,
175
+ body: dict,
176
+ fetch: Callable[[Path], ExportFile | None],
177
+ opens: Lanes,
178
+ world: str,
179
+ need_mb: int = 0,
143
180
  ) -> Worker:
144
- """Open ``session_id`` on its version's worker with ``body``, claimed before placement so
181
+ """Open ``session_id`` on its key's worker with ``body``, claimed before placement so
145
182
  another open cannot stop the worker as session-less meanwhile. ``fetch`` downloads an
146
183
  export to restore into the session's directory once the worker runs (a worker clears
147
- that directory when it starts); the worker reads it by path. One open per version at a
148
- time (``opens``), so the worker's open timeout counts one open, never a queue."""
149
- worker = self._worker_for(version_id)
184
+ that directory when it starts); the worker reads it by path. One open per key at a
185
+ time (``opens``), so the worker's open timeout counts one open, never a queue. ``world``
186
+ labels the worker in what the host logs about it; ``need_mb`` is what the session's world was
187
+ measured to need of it (:class:`MemoryNeed`), 0 when unmeasured."""
188
+ worker = self._worker_for(key, world, need_mb)
150
189
  self._claim(worker, session_id)
151
190
  source = None
152
191
  try:
153
- source = fetch(self.session_dir(version_id, session_id))
192
+ source = fetch(self.session_dir(key, session_id))
154
193
  placement = {**body, "restore": source.for_worker() if source is not None else None}
155
194
  self._open_on(worker, session_id, placement, opens)
156
- except BaseException:
195
+ self._refuse_poisoned(worker, session_id)
196
+ except BaseException as error:
157
197
  self._release(worker, session_id)
198
+ if worker.poisoned and not _busy(error):
199
+ raise ContractError(BUSY, OUT_OF_MEMORY_RETRY, retryAfter=RETRY_AFTER_S) from error
158
200
  raise
159
201
  finally:
160
202
  if source is not None:
161
203
  source.discard()
162
204
  return worker
163
205
 
206
+ def _refuse_poisoned(self, worker: Worker, session_id: str) -> None:
207
+ """An open that landed on a worker poisoned meanwhile is not on any recovery's list: it is deleted
208
+ there rather than kept on a worker about to stop."""
209
+ if not worker.poisoned:
210
+ return
211
+ self._delete(worker, session_id)
212
+ fail(BUSY, OUT_OF_MEMORY_RETRY, retryAfter=RETRY_AFTER_S)
213
+
164
214
  def _open_on(self, worker: Worker, session_id: str, placement: dict, opens: Lanes) -> None:
165
215
  """The open on ``worker``, in its version's turn. One that raised after reaching the worker may still
166
216
  complete there (an answer past the cut-off) or have left a copy, which no open of the id may find
167
217
  later, so it is deleted there, an open still running abandoned, before the caller lets the claim go.
168
218
  One that never reached it (the refusal's cause a :class:`WorkerUnreached`) left nothing to delete."""
219
+ raw = json.dumps(placement).encode("utf-8")
169
220
  try:
170
- with opens.turn(worker.version_id):
171
- self._control(worker, "POST", "/__host/sessions", placement, OPEN_TIMEOUT_S, "the open")
221
+ with opens.turn(worker.key.name):
222
+ status, headers, payload = self._forward(
223
+ worker, "POST", "/__host/sessions", JSON, raw, True, OPEN_TIMEOUT_S, "the open"
224
+ )
225
+ if OutOfMemory.in_headers(headers) and (worker.poisoned or self._exhausted(worker)):
226
+ fail(BUSY, OUT_OF_MEMORY_RETRY, retryAfter=RETRY_AFTER_S)
227
+ self._answer(status, payload)
172
228
  except Exception as error:
173
229
  if not isinstance(error.__cause__, WorkerUnreached):
174
230
  self._delete(worker, session_id)
@@ -216,12 +272,12 @@ class VersionWorkers:
216
272
 
217
273
  def live(self, session: Session) -> Worker:
218
274
  """The session's worker, replaced first (and the session reopened on the new one) if it
219
- died; a session no live worker holds afterwards is forgotten and answers ``unknown_session``.
220
- A session that began closing meanwhile is served like any other: whoever then deletes it
221
- on the new worker has drained the entries it resumed first."""
275
+ died or was poisoned; a session no live worker holds afterwards is forgotten and answers
276
+ ``unknown_session``. A session that began closing meanwhile is served like any other:
277
+ whoever then deletes it on the new worker has drained the entries it resumed first."""
222
278
  worker = session.worker
223
279
  assert worker is not None
224
- if worker.alive():
280
+ if worker.serves():
225
281
  return worker
226
282
  self.recover(worker)
227
283
  current = session.worker
@@ -237,7 +293,7 @@ class VersionWorkers:
237
293
  assert worker is not None
238
294
  self._release(worker, session.session_id)
239
295
  if not worker.alive():
240
- return self._hand_over(session, self.session_dir(session.version_id, session.session_id))
296
+ return self._hand_over(session, self.session_dir(worker.key, session.session_id))
241
297
  self._delete(worker, session.session_id)
242
298
 
243
299
  def _delete(self, worker: Worker, session_id: str) -> None:
@@ -268,7 +324,7 @@ class VersionWorkers:
268
324
  """The file a worker's export landed in, admitted only from inside that worker's
269
325
  directory for this session: the worker runs tenant code, so what it names is
270
326
  checked, and the bytes are hashed by the supervisor, not trusted."""
271
- exports = self.session_dir(session.version_id, session.session_id) / "exports"
327
+ exports = self.session_dir(session.key, session.session_id) / "exports"
272
328
  path = Path(str(answer.get("path") or ""))
273
329
  try:
274
330
  resolved = path.resolve(strict=True)
@@ -278,27 +334,29 @@ class VersionWorkers:
278
334
  return resolved
279
335
 
280
336
  def recover(self, dead: Worker, reason: str = "workers_replaced") -> None:
281
- """Replace ``dead`` (stopped first if it still runs) and reopen every session it held on
282
- the new worker from what each committed; a session that cannot be reopened leaves the
283
- registry (:meth:`_reopen_failed`). One recovery per version at a time; a later caller finds
284
- it done. Never raises. Every session it takes is its own until it gives the session back
285
- (:meth:`_give_back`)."""
337
+ """Replace ``dead`` (stopped first if it still runs; a poisoned one once the requests already on it
338
+ are answered or ``POISONED_SETTLE_S`` passes) and reopen every session it held on the new worker
339
+ from what each committed; a session that cannot be reopened leaves the registry
340
+ (:meth:`_reopen_failed`). One recovery per key at a time; a later caller finds it done. Never
341
+ raises. Every session it takes is its own until it gives the session back (:meth:`_give_back`)."""
286
342
  held: list[Session] = []
287
343
  try:
288
- with self._recoveries.turn(dead.version_id):
289
- if self.pool.peek(dead.version_id) is not dead:
344
+ with self._recoveries.turn(dead.key.name):
345
+ if self.pool.peek(dead.key) is not dead:
290
346
  return
347
+ self._settle(dead)
348
+ self._outgrown(dead)
291
349
  self._stop(dead)
292
350
  sessions, closing = self.registry.recovering(dead)
293
351
  held = [*sessions, *closing]
294
- aside = self._set_aside(dead.version_id, held)
352
+ aside = self._set_aside(dead.key, held)
295
353
  fresh = self._replacement(dead)
296
354
  for session in closing:
297
355
  self._hand_over(session, aside.get(session.session_id))
298
356
  for session in sessions:
299
357
  self._reopen(fresh, session, aside.get(session.session_id))
300
358
  self._forget_overdue(lambda worker: worker is dead)
301
- self.registry.tally(reason)
359
+ self.registry.tally(WORKERS_POISONED if dead.poisoned else reason)
302
360
  finally:
303
361
  self._give_back(held)
304
362
 
@@ -313,21 +371,21 @@ class VersionWorkers:
313
371
  log_line(f"session {session.session_id} did not leave its new worker: {type(error).__name__}")
314
372
 
315
373
  def recover_dead(self) -> list[str]:
316
- """Recover every worker that died holding sessions; returns their versions."""
374
+ """Recover every worker that died holding sessions; returns their keys' names."""
317
375
  dead = self.pool.dead()
318
376
  for worker in dead:
319
377
  self.recover(worker)
320
- return [worker.version_id for worker in dead]
378
+ return [worker.key.name for worker in dead]
321
379
 
322
380
  def recover_stuck(self) -> list[str]:
323
381
  """Recycle every live worker holding a turn past ``STUCK_TURN_S`` (a handler that never
324
382
  returned), whether or not that turn's session is still open here: its sessions reopen
325
- from what they committed, and an id it still held opens again. Returns their versions."""
383
+ from what they committed, and an id it still held opens again. Returns their keys' names."""
326
384
  self._forget_overdue(lambda worker: not worker.alive())
327
385
  stuck = [worker for worker in self.pool.live() if max(self._turns(worker).values(), default=0) > STUCK_TURN_S]
328
386
  for worker in stuck:
329
387
  self.recover(worker, "workers_recycled")
330
- return [worker.version_id for worker in stuck]
388
+ return [worker.key.name for worker in stuck]
331
389
 
332
390
  def _serving(self, session: Session) -> Worker:
333
391
  """The session's live worker, refused ``unknown_session`` once its recovery could not reopen it (it
@@ -371,36 +429,49 @@ class VersionWorkers:
371
429
  ages = {str(sid): _age(age) for sid, age in turns.items()}
372
430
  return {sid: age for sid, age in ages.items() if age is not None}
373
431
 
432
+ def _outgrown(self, worker: Worker) -> None:
433
+ """A poisoned worker's replacement is sized by what it was seen to hold, read before it stops."""
434
+ if not worker.poisoned:
435
+ return
436
+ self.pool.outgrown(worker)
437
+
438
+ def _settle(self, worker: Worker) -> None:
439
+ """A poisoned worker's requests already forwarded get up to ``POISONED_SETTLE_S`` to be answered."""
440
+ if not worker.poisoned:
441
+ return
442
+ with self._forwarding:
443
+ self._forwarding.wait_for(lambda: not self._inflight[id(worker)], timeout=POISONED_SETTLE_S)
444
+
374
445
  @staticmethod
375
446
  def _stop(dead: Worker) -> None:
376
447
  try:
377
448
  dead.stop()
378
449
  except Exception as error: # noqa: BLE001 - a worker that will not stop is replaced all the same
379
- log_line(f"worker for version {dead.version_id} did not stop: {type(error).__name__}")
450
+ log_line(f"worker for {dead.key.name} did not stop: {type(error).__name__}")
380
451
 
381
452
  def _replacement(self, dead: Worker) -> Worker | None:
382
453
  try:
383
454
  return self.pool.replace(dead)
384
455
  except Exception as error: # noqa: BLE001 - its sessions are lost either way; the platform reopens them
385
- log_line(f"worker for version {dead.version_id} could not be replaced: {type(error).__name__}")
456
+ log_line(f"worker for {dead.key.name} could not be replaced: {type(error).__name__}")
386
457
  return None
387
458
 
388
- def _set_aside(self, version_id: str, sessions: list[Session]) -> dict[str, Path]:
459
+ def _set_aside(self, key: WorkerKey, sessions: list[Session]) -> dict[str, Path]:
389
460
  """Each session's directory moved under the worker's ``recovered/``, before a new worker
390
461
  clears its sessions directory; a session whose directory cannot be moved is left out."""
391
- root = self.home(version_id) / RECOVERED_DIR
462
+ root = self.home(key) / RECOVERED_DIR
392
463
  try:
393
464
  if not real_dir(root):
394
465
  clear(root)
395
466
  root.mkdir(mode=0o700, exist_ok=True)
396
467
  except OSError as error:
397
- log_line(f"no directory to set version {version_id}'s sessions aside in: {type(error).__name__}")
468
+ log_line(f"no directory to set {key.name}'s sessions aside in: {type(error).__name__}")
398
469
  return {}
399
- moved = {session.session_id: self._move_aside(root, version_id, session.session_id) for session in sessions}
470
+ moved = {session.session_id: self._move_aside(root, key, session.session_id) for session in sessions}
400
471
  return {session_id: directory for session_id, directory in moved.items() if directory is not None}
401
472
 
402
- def _move_aside(self, root: Path, version_id: str, session_id: str) -> Path | None:
403
- source = self.session_dir(version_id, session_id)
473
+ def _move_aside(self, root: Path, key: WorkerKey, session_id: str) -> Path | None:
474
+ source = self.session_dir(key, session_id)
404
475
  target = root / session_id
405
476
  try:
406
477
  if not (real_dir(root) and real_dir(source.parent) and real_dir(source)):
@@ -418,9 +489,10 @@ class VersionWorkers:
418
489
  body = {**session.placement, "restore": {"recovered": str(directory)}}
419
490
  self._claim(fresh, session.session_id)
420
491
  try:
421
- status, _, payload = fresh.request(
492
+ status, headers, payload = fresh.request(
422
493
  "POST", "/__host/sessions", JSON, json.dumps(body).encode("utf-8"), control=True, timeout=OPEN_TIMEOUT_S
423
494
  )
495
+ self._watch(fresh, headers)
424
496
  self._answer(status, payload)
425
497
  except Exception as error: # noqa: BLE001 - one session that cannot be reopened never strands the rest
426
498
  return self._reopen_failed(fresh, session, directory, error)
@@ -471,12 +543,12 @@ class VersionWorkers:
471
543
  finally:
472
544
  shutil.rmtree(directory, ignore_errors=True)
473
545
 
474
- def _worker_for(self, version_id: str) -> Worker:
475
- current = self.pool.peek(version_id)
476
- if current is not None and not current.alive():
546
+ def _worker_for(self, key: WorkerKey, world: str, need_mb: int) -> Worker:
547
+ current = self.pool.peek(key)
548
+ if current is not None and not current.serves():
477
549
  self.recover(current)
478
550
  try:
479
- return self.pool.get(version_id)
551
+ return self.pool.get(key, world=world, need_mb=need_mb)
480
552
  except WorkerCapacity as error:
481
553
  fail("at_capacity", str(error))
482
554
 
@@ -529,9 +601,59 @@ class VersionWorkers:
529
601
  limit: int | None = None,
530
602
  ) -> tuple[int, list[tuple[str, str]], bytes]:
531
603
  try:
532
- return worker.request(method, path, headers, raw, control=control, timeout=timeout, limit=limit)
604
+ with self._forwarded(worker):
605
+ answer = worker.request(method, path, headers, raw, control=control, timeout=timeout, limit=limit)
533
606
  except WorkerGone as error:
534
607
  raise self._unanswered(worker, error, what, timeout, session_id) from error
608
+ self._watch(worker, answer[1])
609
+ return answer
610
+
611
+ def _watch(self, worker: Worker, headers: list[tuple[str, str]]) -> None:
612
+ if OutOfMemory.in_headers(headers):
613
+ self._poison(worker)
614
+
615
+ @contextmanager
616
+ def _forwarded(self, worker: Worker) -> Iterator[None]:
617
+ with self._forwarding:
618
+ self._inflight[id(worker)] += 1
619
+ try:
620
+ yield
621
+ finally:
622
+ with self._forwarding:
623
+ self._inflight[id(worker)] -= 1
624
+ if self._inflight[id(worker)] <= 0:
625
+ del self._inflight[id(worker)]
626
+ self._forwarding.notify_all()
627
+
628
+ def _exhausted(self, worker: Worker) -> bool:
629
+ """Whether the kernel counts ``worker`` at ``POISONED_HELD_FRACTION`` of its address-space cap or more
630
+ (``VmSize``); a worker the kernel cannot say of (no ``/proc``) is taken at its word."""
631
+ size = self.pool.memory.status(worker.pid).mb(SIZE)
632
+ return size is None or size >= worker.cap_mb * POISONED_HELD_FRACTION
633
+
634
+ def _poison(self, worker: Worker) -> None:
635
+ """World code on ``worker`` raised ``MemoryError`` while it holds most of its memory: it takes no new
636
+ request and is recycled behind this answer, once, with a line naming its world and version; not when
637
+ its key was recycled for this within ``POISONED_BACKOFF_S``, so a lane that keeps running out cuts its
638
+ calls once a minute at most, and never for a request that only asked for more than the cap."""
639
+ if not self._exhausted(worker):
640
+ return
641
+ now = time.monotonic()
642
+ with self._claims:
643
+ backed_off = now - self._poisoned_at.get(worker.key, -math.inf) < POISONED_BACKOFF_S
644
+ first = not worker.poisoned and not backed_off
645
+ if first:
646
+ worker.poisoned = True
647
+ self._poisoned_at[worker.key] = now
648
+ if not first:
649
+ return
650
+ log_line(
651
+ f"world {worker.world!r} version {worker.version_id} ({worker.key.lane}): out of memory in its worker "
652
+ f"(pid {worker.pid}); recycling it"
653
+ )
654
+ threading.Thread(
655
+ target=self.recover, args=(worker, WORKERS_POISONED), name="world-worker-recycle", daemon=True
656
+ ).start()
535
657
 
536
658
  def _unanswered(
537
659
  self, worker: Worker, error: WorkerGone, what: str, timeout: float, session_id: str | None
@@ -407,9 +407,7 @@ def ingest_records(
407
407
  else:
408
408
  fail("ingest", "mapping is an ingest.toml text, a parsed table, a Mapping or a file path", "/mapping")
409
409
  if declared is not None:
410
- entities = _entities(root)
411
- validate_mapping(declared, entities)
412
- projected, report = apply_mapping(declared, listed, entities)
410
+ projected, report = map_records(root, listed, declared)
413
411
  else:
414
412
  spec = load_spec(root) if (root / SPEC).is_file() else None
415
413
  if spec is None or not any(operation.entity for operation in spec.operations.values()):
@@ -446,6 +444,17 @@ def ingest_records(
446
444
  return report
447
445
 
448
446
 
447
+ def map_records(root: str | Path, records: list, mapping: Any) -> tuple[dict[str, list[dict]], dict]:
448
+ """``records`` through a declared ``ingest.toml`` against the world's contract, nothing imported:
449
+ the rows and the mapping's report, for a caller that lands the rows through its own gate."""
450
+ from .mapping import Mapping, apply_mapping, parse_mapping, validate_mapping
451
+
452
+ declared = mapping if isinstance(mapping, Mapping) else parse_mapping(mapping)
453
+ entities = _entities(Path(root))
454
+ validate_mapping(declared, entities)
455
+ return apply_mapping(declared, records, entities)
456
+
457
+
449
458
  def _refused(via: str, listed: list, reason: str, redact: str, dry_run: bool, keep_rows: bool) -> dict:
450
459
  report = {
451
460
  "via": via,
@@ -13,7 +13,7 @@ from typing import TYPE_CHECKING, Any
13
13
  from .._toml import tomllib
14
14
  from .call_redaction import REDACTED, CallRedaction
15
15
  from .database.base import SideTable
16
- from .tasks import JOURNAL, JOURNAL_META, WITHHELD_FIELD
16
+ from .tasks import JOURNAL, JOURNAL_META, PAGE_NOTES, WITHHELD_FIELD
17
17
  from .validation import fail
18
18
 
19
19
  if TYPE_CHECKING:
@@ -26,7 +26,7 @@ JOURNAL_TABLES = frozenset(JOURNAL_KEYS)
26
26
  MAX_JOURNAL_REQUESTS = 20_000
27
27
  NOT_RECORDED = "not recorded: only a World Host session records the requests its world receives"
28
28
  LINKED_NOT_RECORDED = "not recorded for a linked world: the root world's verifiers and the task's checks read it"
29
- VIAS = ("api", "ui", "tool", "webhook")
29
+ VIAS = ("api", "ui", "tool", "webhook", "page")
30
30
  JSON_HINTS = ("json",)
31
31
  WITHHELD_ROUTE_BODY = ["body", "selectors"]
32
32
  WITHHELD_TOOL_ARGS = ["args", "selectors"]
@@ -260,8 +260,13 @@ class CallsView:
260
260
  return None
261
261
  if entry.raw["via"] == "tool" and entry.raw["method"] is None:
262
262
  return self._tool(entry) if self.admitted(entry.operation) else None
263
+ if entry.raw["via"] == "page":
264
+ return None if entry.operation in PAGE_NOTES else self._interaction(entry)
263
265
  return self._route(entry)
264
266
 
267
+ def _interaction(self, entry: JournalEntry) -> dict:
268
+ return {**self._tool(entry), "path": entry.raw["path"], "query": entry.raw["query"]}
269
+
265
270
  def _tool(self, entry: JournalEntry) -> dict:
266
271
  body = entry.request
267
272
  args = body.value if body.whole else REDACTED
@@ -10,6 +10,7 @@ import tempfile
10
10
  import threading
11
11
  import urllib.parse
12
12
  from dataclasses import dataclass, field
13
+ from functools import partial
13
14
  from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
14
15
  from pathlib import Path
15
16
  from typing import Any, Callable
@@ -33,6 +34,7 @@ from .tools_world import (
33
34
  ToolsWorldState,
34
35
  load_tools_version,
35
36
  )
37
+ from .ui_interactions import RECORDER_PATH, UiRecorder
36
38
  from .validation import ContractError
37
39
  from .verbs import refusal, session_verb, unknown_verb
38
40
 
@@ -51,7 +53,7 @@ def load_local_version(root: Path, scratch: Path) -> ToolsVersion:
51
53
  f"{root} is not a schema world (no gateway-env.toml with a [schema] table); "
52
54
  "`gateway worlds schema init` starts one",
53
55
  )
54
- return load_tools_version(root, scratch / "template.db", None, run_handlers=True)
56
+ return load_tools_version(root, scratch / "template.db", fetched=False, run_handlers=True)
55
57
 
56
58
 
57
59
  def load_local_dependencies(root: Path, version: ToolsVersion, scratch: Path) -> dict[str, DependencyWorld]:
@@ -317,10 +319,7 @@ def make_session_handler(
317
319
  raw = self.rfile.read(length) if length else b""
318
320
  path = self.path.split("?", 1)[0]
319
321
  if ui_origin:
320
- assert ui is not None
321
- if app is not None and api_target(ui, path) is not None:
322
- return self._ui_api(method, path, raw, Via.UI)
323
- return self._ui(method, path)
322
+ return self._ui_origin(method, path, raw)
324
323
  origin = self.headers.get("Origin")
325
324
  if origin and not _local_origin(origin):
326
325
  return self._reply(403, {"error": "this session answers this machine's pages, not another site's"})
@@ -344,11 +343,32 @@ def make_session_handler(
344
343
  )
345
344
  send_response(self, status, body, extra, app.settings.error_content_type)
346
345
 
346
+ def _ui_origin(self, method: str, path: str, raw: bytes) -> None:
347
+ assert ui is not None
348
+ if app is not None and api_target(ui, path) is not None:
349
+ return self._ui_api(method, path, raw, Via.UI)
350
+ if path == RECORDER_PATH:
351
+ return self._page(method, UiRecorder.script)
352
+ if UiRecorder.serves(path):
353
+ return self._recorded(method, path, raw)
354
+ return self._page(method, partial(self._served, path))
355
+
356
+ def _served(self, path: str) -> tuple[int, dict[str, str], bytes]:
357
+ assert ui is not None
358
+ status, headers, payload = static_file(ui, path)
359
+ page = UiRecorder.page(status, headers, payload)
360
+ if page is not None:
361
+ UiRecorder.served(path, calls)
362
+ return status, headers, payload if page is None else page
363
+
347
364
  def _ui(self, method: str, path: str) -> None:
365
+ assert ui is not None
366
+ return self._page(method, partial(static_file, ui, path))
367
+
368
+ def _page(self, method: str, answer: Callable[[], tuple[int, dict[str, str], bytes]]) -> None:
348
369
  if method != "GET":
349
370
  return self._reply(405, {"error": "the static ui answers GET only"})
350
- assert ui is not None
351
- status, headers, payload = static_file(ui, path)
371
+ status, headers, payload = answer()
352
372
  self.send_response(status)
353
373
  for name, value in headers.items():
354
374
  self.send_header(name, value)
@@ -356,6 +376,11 @@ def make_session_handler(
356
376
  self.end_headers()
357
377
  write_body(self, payload)
358
378
 
379
+ def _recorded(self, method: str, path: str, raw: bytes) -> None:
380
+ if cross_site({k.lower(): v for k, v in self.headers.items()}, _local_origin):
381
+ return self._reply(403, {"error": "the recorder answers the page on this server, not another site"})
382
+ return self._reply(*UiRecorder.record(method, path, raw, calls))
383
+
359
384
  def _ui_api(self, method: str, path: str, raw: bytes, via: Via) -> None:
360
385
  """Answers with the world's own credential for the pin, as the host does for the session cookie."""
361
386
  assert app is not None and ui is not None
@@ -729,7 +729,7 @@ def apply_mapping(mapping: Mapping, records: Iterable[Any], entities: Entities)
729
729
  candidates = list(by_name.get(ANY_SOURCE, []))
730
730
  if tool_calls and isinstance(tool, str) and tool:
731
731
  candidates = by_name.get(tool, []) + candidates
732
- record = {**raw, "args": raw.get("args") if isinstance(raw.get("args"), dict) else {}}
732
+ record = {**raw, "args": raw.get("args") if isinstance(raw.get("args"), dict) else {}} if tool_calls else raw
733
733
  meta = record_meta(record, index)
734
734
  landed = 0
735
735
  try: