@withgateway/sdk 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. package/README.md +8 -6
  2. package/dist/cli.js +4372 -2265
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +4362 -2256
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +259 -58
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +249 -50
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/hooks/index.d.mts +1 -1
  13. package/dist/hooks/index.d.ts +1 -1
  14. package/dist/hooks/index.js +124 -11
  15. package/dist/hooks/index.js.map +1 -1
  16. package/dist/hooks/index.mjs +124 -11
  17. package/dist/hooks/index.mjs.map +1 -1
  18. package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
  19. package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
  20. package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
  21. package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
  22. package/dist/index.d.mts +5 -5
  23. package/dist/index.d.ts +5 -5
  24. package/dist/index.js +2240 -789
  25. package/dist/index.js.map +1 -1
  26. package/dist/index.mjs +2150 -739
  27. package/dist/index.mjs.map +1 -1
  28. package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
  29. package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
  30. package/dist/traces/index.d.mts +1 -1
  31. package/dist/traces/index.d.ts +1 -1
  32. package/dist/traces/index.js +253 -52
  33. package/dist/traces/index.js.map +1 -1
  34. package/dist/traces/index.mjs +243 -44
  35. package/dist/traces/index.mjs.map +1 -1
  36. package/dist/tracing/index.d.mts +2 -2
  37. package/dist/tracing/index.d.ts +2 -2
  38. package/dist/tracing/index.js +75 -13
  39. package/dist/tracing/index.js.map +1 -1
  40. package/dist/tracing/index.mjs +75 -13
  41. package/dist/tracing/index.mjs.map +1 -1
  42. package/dist/worlds/index.d.mts +893 -482
  43. package/dist/worlds/index.d.ts +893 -482
  44. package/dist/worlds/index.js +1968 -669
  45. package/dist/worlds/index.js.map +1 -1
  46. package/dist/worlds/index.mjs +1960 -662
  47. package/dist/worlds/index.mjs.map +1 -1
  48. package/package.json +1 -1
  49. package/runtime/gatewaysdk/environment/_bundle.py +2 -27
  50. package/runtime/gatewaysdk/environment/_materialize.py +48 -88
  51. package/runtime/gatewaysdk/environment/_world.py +155 -237
  52. package/runtime/gatewaysdk/environment/backing.py +133 -0
  53. package/runtime/gatewaysdk/environment/core.py +2 -17
  54. package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
  55. package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
  56. package/runtime/gatewaysdk/environment/runtime.py +8 -13
  57. package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
  58. package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
  59. package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
  60. package/runtime/gatewaysdk/environment/schema/api.py +347 -892
  61. package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
  62. package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
  63. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
  64. package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
  65. package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
  66. package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
  67. package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
  68. package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
  69. package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
  70. package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
  71. package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
  72. package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
  73. package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
  74. package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
  75. package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
  76. package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
  77. package/runtime/gatewaysdk/environment/schema/data.py +60 -216
  78. package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
  79. package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
  80. package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
  81. package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
  82. package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
  83. package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
  84. package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
  85. package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
  86. package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
  87. package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
  88. package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
  89. package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
  90. package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
  91. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
  92. package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
  93. package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
  94. package/runtime/gatewaysdk/environment/schema/door.py +899 -0
  95. package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
  96. package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
  97. package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
  98. package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
  99. package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
  100. package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
  101. package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
  102. package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
  103. package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
  104. package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
  105. package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
  106. package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
  107. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
  108. package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
  109. package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
  110. package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
  111. package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
  112. package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
  113. package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
  114. package/runtime/gatewaysdk/environment/schema/local.py +58 -170
  115. package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
  116. package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
  117. package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
  118. package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
  119. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
  120. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
  121. package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
  122. package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
  123. package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
  124. package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
  125. package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
  126. package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
  127. package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
  128. package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
  129. package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
  130. package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
  131. package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
  132. package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
  133. package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
  134. package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
  135. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
  136. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
  137. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
  138. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
  139. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
  140. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
  141. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
  142. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
  143. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
  144. package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
  145. package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
  146. package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
  147. package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
  148. package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
  149. package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
  150. package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
  151. package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
  152. package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
  153. package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
  154. package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
  155. package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
  156. package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
  157. package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
  158. package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
  159. package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
  160. package/runtime/gatewaysdk/environment/score.py +57 -0
  161. package/runtime/gatewaysdk/environment/secret_values.py +36 -0
  162. package/runtime/gatewaysdk/environment/world.py +43 -4
  163. package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
  164. package/skills/agents/connector-template-author.md +5 -1
  165. package/skills/benchmarks-getting-started/SKILL.md +10 -11
  166. package/skills/connector-schema-authoring/SKILL.md +49 -16
  167. package/skills/connector-schema-authoring/connector-worlds.md +142 -11
  168. package/skills/connector-schema-authoring/world-schema.md +103 -51
  169. package/skills/world-data-ingestion/ROWS.md +39 -9
  170. package/skills/world-data-ingestion/SKILL.md +27 -27
  171. package/skills/worlds-getting-started/SKILL.md +33 -33
  172. package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
  173. package/skills/worlds-getting-started/references/gotchas.md +49 -1
  174. package/skills/worlds-getting-started/references/journal-checks.md +182 -0
  175. package/skills/worlds-getting-started/references/openapi-import.md +50 -21
  176. package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
@@ -0,0 +1,958 @@
1
+ """The World Host's half of the world request ledger: every request a session's world receives,
2
+ through every door, waits in the session's :class:`RequestOutbox`; the :class:`LedgerFlusher` moves
3
+ it to the host's :class:`~.ledger_spill.LedgerSpill` on disk, and from there to the platform.
4
+
5
+ Only the platform's acknowledgement lets an entry go from the disk. The grader's call log
6
+ (:class:`~.call_log.CallLog`) keeps its own bounded view; no bound of it reaches the ledger. An outbox
7
+ is moved to disk every few seconds and as soon as it passes half its ceiling, so it reaches its
8
+ ceilings (bodies shed past :data:`OUTBOX_MAX_CHARS`, entries dropped past :data:`OUTBOX_MAX_ENTRIES`)
9
+ only while that move is held up. What an outbox sheds or drops, and what the disk's cap lets go, is
10
+ counted in the session's account and stored with it. Bodies are never logged."""
11
+
12
+ from __future__ import annotations
13
+
14
+ import collections
15
+ import enum
16
+ import http.client
17
+ import json
18
+ import re
19
+ import threading
20
+ import time
21
+ import urllib.error
22
+ import urllib.request
23
+ import uuid
24
+ from dataclasses import dataclass
25
+ from datetime import datetime, timezone
26
+ from functools import partial
27
+ from typing import IO, TYPE_CHECKING, Callable, Iterator, Mapping, Protocol
28
+
29
+ from ..world_session_contract import REQUEST_LOG_PAGE_MAX
30
+ from .call_capture import Exchange
31
+ from .call_log import CallJournal
32
+ from .handler_errors import log_line
33
+ from .ledger_spill import HEX32, LEDGER_BATCH_BYTES, LedgerSpill, Piece, counted, ledger_line, runs
34
+ from .validation import fail
35
+
36
+ if TYPE_CHECKING:
37
+ from .host_session import Session
38
+ from .world_host import WorldHost
39
+
40
+ LEDGER_FLUSH_S = 2.0
41
+ LEDGER_STOP_S = 10.0
42
+ LEDGER_BARRIER_S = 20.0
43
+ BARRIER_POLL_S = 0.05
44
+ NO_PLATFORM = "this World Host sends no request ledger: it has no platform to send to"
45
+ INHERITED_REQUESTS = (
46
+ "this session was reopened from a checkpoint or recovered after its worker or host stopped, "
47
+ "so the requests its previous holder had not stored cannot be proven"
48
+ )
49
+ INHERITED_MAX_CHARS = 500
50
+ INGEST_ANSWER_BYTES = 64 * 1024
51
+ LEDGER_ROUNDS = 10
52
+ LEDGER_BACKOFF_MAX_S = 60.0
53
+ LEDGER_ROUTE = "/api/public/world-hosts/requests"
54
+ OUTBOX_MAX_ENTRIES = 20_000
55
+ OUTBOX_MAX_CHARS = 8 << 20
56
+ TAKE_MAX_PAGES = 2 * OUTBOX_MAX_ENTRIES // REQUEST_LOG_PAGE_MAX
57
+ TAKE_DEADLINE_S = 5.0
58
+ LEDGER_PASS_S = 10.0
59
+ LEFT_JOURNAL_BYTES = 64 << 20
60
+ DRAIN_BYTES = 4 << 20
61
+ DRAIN_ANSWER_BYTES = DRAIN_BYTES + (1 << 20)
62
+ OUTBOX_CURSOR = re.compile(r"[0-9a-f]{32}:[0-9]{1,18}")
63
+ RETRIED_STATUSES = frozenset({401, 403, 408, 429})
64
+
65
+
66
+ def _now() -> str:
67
+ return datetime.now(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z")
68
+
69
+
70
+ def _size(value: object) -> int:
71
+ return len(json.dumps(value, ensure_ascii=False).encode("utf-8", "replace"))
72
+
73
+
74
+ def _late(until: float | None) -> bool:
75
+ return until is not None and time.monotonic() >= until
76
+
77
+
78
+ def _timeout(until: float | None) -> float | None:
79
+ return None if until is None else max(0.05, until - time.monotonic())
80
+
81
+
82
+ class MalformedPage(ValueError):
83
+ """A drain answer the flusher cannot use: a version worker runs tenant code."""
84
+
85
+
86
+ @dataclass
87
+ class _Budget:
88
+ """What a take still may move (``bytes``) and the pages it moved."""
89
+
90
+ bytes: int
91
+ pages: int = 0
92
+
93
+ def spend(self, size: int) -> None:
94
+ self.bytes -= size
95
+ self.pages += 1
96
+
97
+ def spent(self) -> bool:
98
+ return self.bytes <= 0
99
+
100
+
101
+ @dataclass(frozen=True)
102
+ class OutboxPage:
103
+ """One drain answer as the flusher may trust it (:meth:`of`): each entry as the line the spill
104
+ keeps, None for one the platform would not parse; ``epoch`` names the outbox that answered,
105
+ ``pending`` how many entries it holds after this page and ``inherited`` why it cannot say it holds
106
+ every request it was given."""
107
+
108
+ lines: list[bytes | None]
109
+ cursor: str | None
110
+ more: bool
111
+ epoch: str
112
+ shed: int
113
+ lost: int
114
+ pending: int
115
+ inherited: str | None = None
116
+ source: str | None = None
117
+
118
+ @classmethod
119
+ def of(cls, answer: object) -> OutboxPage:
120
+ page = answer if isinstance(answer, dict) else {}
121
+ requests, cursor, epoch, inherited, source = (
122
+ page.get(key) for key in ("requests", "cursor", "epoch", "inherited", "source")
123
+ )
124
+ valid = (
125
+ isinstance(requests, list)
126
+ and isinstance(epoch, str)
127
+ and bool(HEX32.fullmatch(epoch))
128
+ and counted(page.get("shed"))
129
+ and counted(page.get("lost"))
130
+ and counted(page.get("pending"))
131
+ and isinstance(page.get("more"), bool)
132
+ and (cursor is None if not requests else isinstance(cursor, str) and bool(OUTBOX_CURSOR.fullmatch(cursor)))
133
+ and (inherited is None or (isinstance(inherited, str) and len(inherited) <= INHERITED_MAX_CHARS))
134
+ and (source is None or LedgerLife.sourced(source))
135
+ )
136
+ if not valid:
137
+ raise MalformedPage("a drain answer off the outbox's contract")
138
+ lines = [ledger_line(entry) for entry in requests]
139
+ return cls(lines, cursor, page["more"], epoch, page["shed"], page["lost"], page["pending"], inherited, source)
140
+
141
+ @property
142
+ def size(self) -> int:
143
+ return sum(len(line or b"") for line in self.lines)
144
+
145
+
146
+ class MoveFailed(Exception):
147
+ """A move of a session's outbox that failed once the outbox had answered ``page``: ``left`` is what it
148
+ still holds, the page's own entries included unless they were ``kept`` on the spill."""
149
+
150
+ def __init__(self, page: OutboxPage, kept: bool) -> None:
151
+ super().__init__("a move of a session's outbox to the spill failed")
152
+ self.left = page.pending + (0 if kept else len(page.lines))
153
+
154
+
155
+ class RequestOutbox:
156
+ """One session's requests on their way to the ledger, oldest first, each under the id
157
+ the ledger keeps it by and the outbox's own ``seq``. :meth:`drain` is the only way out:
158
+ handed the ``cursor`` of the last page the host took, it lets go of every entry up to it
159
+ and answers the next page. A cursor names the outbox it came from (``epoch``), so one a
160
+ replaced outbox answered lets go of nothing here. Past half of either ceiling it calls
161
+ ``on_pressure``, which wakes the host's flusher. A worker's session journals its outbox
162
+ beside its store (:data:`~.session_files.LEDGER_JOURNAL`), rewritten from memory once
163
+ twice the character ceiling was appended, so a worker that dies leaves its unsent
164
+ entries to the next one (:meth:`resume`)."""
165
+
166
+ def __init__(
167
+ self,
168
+ journal: CallJournal | None = None,
169
+ *,
170
+ max_entries: int = OUTBOX_MAX_ENTRIES,
171
+ max_chars: int = OUTBOX_MAX_CHARS,
172
+ on_pressure: Callable[[], None] | None = None,
173
+ ) -> None:
174
+ self.journal = journal
175
+ self.max_entries = max_entries
176
+ self.max_chars = max_chars
177
+ self.on_pressure = on_pressure
178
+ self.epoch = uuid.uuid4().hex
179
+ self.entries: collections.deque[dict] = collections.deque()
180
+ self._charges: collections.deque[int] = collections.deque()
181
+ self._unshed = 0
182
+ self.chars = 0
183
+ self.seq = 0
184
+ self.shed = 0
185
+ self.lost = 0
186
+ self.inherited: str | None = None
187
+ self._lock = threading.Lock()
188
+
189
+ def append(self, line: dict, exchange: Exchange) -> None:
190
+ at = _now()
191
+ with self._lock:
192
+ self.seq += 1
193
+ entry = {"id": uuid.uuid4().hex, "seq": self.seq, "at": at, **line, **exchange.as_json()}
194
+ self._hold(entry, exchange.charge)
195
+ self._journal(entry)
196
+ pressed = 2 * self.chars > self.max_chars or 2 * len(self.entries) > self.max_entries
197
+ self._bound()
198
+ if pressed and self.on_pressure is not None:
199
+ self.on_pressure()
200
+
201
+ def drain(self, cursor: str | None, limit: int = REQUEST_LOG_PAGE_MAX, max_bytes: int = DRAIN_BYTES) -> dict:
202
+ """Let go of every entry up to ``cursor`` and answer the entries after it, at most
203
+ ``limit`` and as many as fit ``max_bytes`` of JSON (always one), with the
204
+ ``cursor`` that lets go of them, whether ``more`` wait and how many (``pending``), and the
205
+ outbox's running counts."""
206
+ with self._lock:
207
+ if self._ack(cursor):
208
+ self._rewrite()
209
+ page, spent = [], 0
210
+ for entry in self.entries:
211
+ spent += _size(entry)
212
+ if len(page) == limit or (page and spent > max_bytes):
213
+ break
214
+ page.append(entry)
215
+ return {
216
+ "requests": page,
217
+ "cursor": f"{self.epoch}:{page[-1]['seq']}" if page else None,
218
+ "more": len(self.entries) > len(page),
219
+ "pending": len(self.entries) - len(page),
220
+ "epoch": self.epoch,
221
+ "shed": self.shed,
222
+ "lost": self.lost,
223
+ **({"inherited": self.inherited} if self.inherited else {}),
224
+ }
225
+
226
+ def resume(self, entries: list[dict], inherited: str = INHERITED_REQUESTS) -> None:
227
+ """Carry on with the entries a previous holder kept, numbered on from the last; what it
228
+ had not kept is unknown, so the outbox can no longer say it holds every request."""
229
+ with self._lock:
230
+ self.inherited = inherited
231
+ for entry in entries:
232
+ self._hold(entry, Exchange.from_json(entry).charge)
233
+ self.seq = max([self.seq, *(int(entry["seq"]) for entry in entries)])
234
+ self._unshed = 0
235
+ self._bound()
236
+ self._rewrite()
237
+
238
+ def _hold(self, entry: dict, charge: int) -> None:
239
+ self.entries.append(entry)
240
+ self._charges.append(charge)
241
+ self.chars += charge
242
+
243
+ def _pop(self) -> None:
244
+ self.entries.popleft()
245
+ self.chars -= self._charges.popleft()
246
+ self._unshed = max(0, self._unshed - 1)
247
+
248
+ def _ack(self, cursor: str | None) -> bool:
249
+ epoch, _, seq = (cursor or "").partition(":")
250
+ if epoch != self.epoch:
251
+ return False
252
+ acked = False
253
+ while self.entries and int(self.entries[0]["seq"]) <= int(seq):
254
+ self._pop()
255
+ acked = True
256
+ return acked
257
+
258
+ def _bound(self) -> None:
259
+ while self.chars > self.max_chars and self._unshed < len(self.entries):
260
+ self._shed(self._unshed)
261
+ self._unshed += 1
262
+ while len(self.entries) > self.max_entries:
263
+ self._pop()
264
+ self.lost += 1
265
+
266
+ def _shed(self, index: int) -> None:
267
+ charge = self._charges[index]
268
+ if charge == 0:
269
+ return
270
+ entry = self.entries[index]
271
+ self.entries[index] = {**entry, **Exchange.from_json(entry).shed().as_json()}
272
+ self._charges[index] = 0
273
+ self.chars -= charge
274
+ self.shed += 1
275
+
276
+ def _journal(self, entry: dict) -> None:
277
+ if self.journal is None:
278
+ return
279
+ self.journal.append(CallJournal.line(entry))
280
+ if self.journal.appended > 2 * self.max_chars:
281
+ self._rewrite()
282
+
283
+ def _rewrite(self) -> None:
284
+ if self.journal is not None:
285
+ self.journal.rewrite(list(self.entries), 8 * self.max_chars)
286
+
287
+
288
+ @dataclass
289
+ class LedgerLife:
290
+ """The platform's account of one life of a session (``source``, from its open): the host keeps the
291
+ life's first outbox (``epoch``, the first a drain answers) under it, so that account alone settles
292
+ it, and every later outbox, a dead worker's included, under an account of its own."""
293
+
294
+ source: str | None = None
295
+ epoch: str | None = None
296
+
297
+ @staticmethod
298
+ def sourced(source: object) -> bool:
299
+ return isinstance(source, str) and bool(HEX32.fullmatch(source)) and source != "0" * 32
300
+
301
+ @classmethod
302
+ def of(cls, source: object) -> LedgerLife:
303
+ if source is not None and not cls.sourced(source):
304
+ fail("host", "ledgerSource is the 32-hex request ledger account the platform opened for this life")
305
+ return cls(source if isinstance(source, str) else None)
306
+
307
+ def stamp(self, page: dict) -> dict:
308
+ epoch = page.get("epoch") if isinstance(page, dict) else None
309
+ if not isinstance(epoch, str):
310
+ return page
311
+ self.epoch = self.epoch or epoch
312
+ return {**page, "source": self.source if epoch == self.epoch else None}
313
+
314
+
315
+ @dataclass(frozen=True)
316
+ class LedgerAccount:
317
+ """This host's account of one session's request ledger as the session leaves it: ``final``
318
+ once everything the host held of it is stored, with what its outbox shed and lost."""
319
+
320
+ final: bool
321
+ shed: int = 0
322
+ lost: int = 0
323
+
324
+ def as_json(self) -> dict:
325
+ return {"final": self.final, "shed": self.shed, "lost": self.lost}
326
+
327
+
328
+ class HostLedger(Protocol):
329
+ """What a host tells its ledger: a session's outbox is filling (``wake``), a session is leaving
330
+ (``leave``), a session is about to move to another host and its requests must be stored first
331
+ (``settle``), a dead worker left a session's unsent entries (``adopt``), the host is stopping
332
+ (``stop``, which gives ``settle`` half its time first), a grade waits until a session's requests
333
+ are stored (``barrier``)."""
334
+
335
+ def wake(self, session_id: str) -> None: ...
336
+
337
+ def leave(self, session: Session) -> None: ...
338
+
339
+ def settle(self, session: Session, until: float) -> LedgerAccount: ...
340
+
341
+ def adopt(self, session_id: str, entries: list[dict]) -> None: ...
342
+
343
+ def stop(self, deadline_s: float = ..., *, settle: Callable[[float], object] = ...) -> None: ...
344
+
345
+ def barrier(self, session: Session) -> dict: ...
346
+
347
+
348
+ def _settled(deadline_s: float) -> bool:
349
+ return True
350
+
351
+
352
+ class NoLedger:
353
+ """The ledger of a host with no platform to send to: a version worker, whose supervisor
354
+ drains its outboxes, or a dev host, whose outboxes keep to their ceilings and whose
355
+ accounts are never final."""
356
+
357
+ def wake(self, session_id: str) -> None:
358
+ return None
359
+
360
+ def leave(self, session: Session) -> None:
361
+ return None
362
+
363
+ def settle(self, session: Session, until: float) -> LedgerAccount:
364
+ return LedgerAccount(final=False)
365
+
366
+ def adopt(self, session_id: str, entries: list[dict]) -> None:
367
+ return None
368
+
369
+ def stop(self, deadline_s: float = LEDGER_STOP_S, *, settle: Callable[[float], object] = _settled) -> None:
370
+ settle(deadline_s)
371
+
372
+ def barrier(self, session: Session) -> dict:
373
+ return _unstored(NO_PLATFORM)
374
+
375
+
376
+ def _unstored(reason: str, shed: int = 0, lost: int = 0) -> dict:
377
+ return {"stored": False, "shed": shed, "lost": lost, "reason": reason}
378
+
379
+
380
+ class Delivery(enum.Enum):
381
+ """What came of one send: the platform stored it, refused it as sent, or did not answer
382
+ (no answer, a server error, a throttle, or a refusal of the host itself)."""
383
+
384
+ STORED = "stored"
385
+ REFUSED = "refused"
386
+ UNAVAILABLE = "unavailable"
387
+
388
+ @classmethod
389
+ def of(cls, status: int) -> Delivery:
390
+ if 200 <= status < 300:
391
+ return cls.STORED
392
+ refused = 400 <= status < 500 and status not in RETRIED_STATUSES
393
+ return cls.REFUSED if refused else cls.UNAVAILABLE
394
+
395
+
396
+ @dataclass(frozen=True)
397
+ class Answer:
398
+ """What came of one send, with how many of its requests the platform stored as refused (off its
399
+ contract), each session's when the platform said (``by_session``); a sink may answer a bare
400
+ :class:`Delivery`, a bool (stored or not) or the refused count."""
401
+
402
+ delivery: Delivery
403
+ refused: int = 0
404
+ by_session: Mapping[str, int] | None = None
405
+
406
+ @classmethod
407
+ def of(cls, answer: object) -> Answer:
408
+ if isinstance(answer, Answer):
409
+ return answer
410
+ if isinstance(answer, Delivery):
411
+ return cls(answer)
412
+ if answer is True:
413
+ return cls(Delivery.STORED)
414
+ stored = counted(answer) and answer is not False
415
+ return cls(Delivery.STORED, int(answer)) if stored else cls(Delivery.UNAVAILABLE)
416
+
417
+
418
+ class LedgerSink(Protocol):
419
+ """Where a flush goes, sent by ``until`` (a monotonic time) when one is given."""
420
+
421
+ def send(self, sessions: list[dict], until: float | None = None) -> Answer | Delivery | int | bool | None: ...
422
+
423
+
424
+ class _Unredirected(urllib.request.HTTPRedirectHandler):
425
+ def redirect_request(
426
+ self,
427
+ req: urllib.request.Request,
428
+ fp: IO[bytes],
429
+ code: int,
430
+ msg: str,
431
+ headers: http.client.HTTPMessage,
432
+ newurl: str,
433
+ ) -> None:
434
+ return None
435
+
436
+
437
+ class PlatformLedger:
438
+ """The platform's ledger door (:data:`LEDGER_ROUTE`), behind the control-plane secret; an answer that
439
+ redirects the flush is not followed, so it reads as unavailable, never as stored."""
440
+
441
+ _opener = urllib.request.build_opener(_Unredirected)
442
+
443
+ def __init__(self, platform_url: str, secret: str, timeout_s: float = 30.0, host_url: str | None = None) -> None:
444
+ self.url = f"{platform_url.rstrip('/')}{LEDGER_ROUTE}"
445
+ self.secret = secret
446
+ self.timeout_s = timeout_s
447
+ self.host = {"host": host_url} if host_url else {}
448
+
449
+ def send(self, sessions: list[dict], until: float | None = None) -> Answer:
450
+ flush = json.dumps(
451
+ {**self.host, "sessions": sessions}, ensure_ascii=False, separators=(",", ":"), allow_nan=False
452
+ )
453
+ body = flush.encode("utf-8", "replace")
454
+ headers = {"content-type": "application/json", "x-host-secret": self.secret}
455
+ request = urllib.request.Request(self.url, data=body, method="POST", headers=headers)
456
+ timeout = self.timeout_s if until is None else min(self.timeout_s, until - time.monotonic())
457
+ if timeout <= 0:
458
+ return Answer(Delivery.UNAVAILABLE)
459
+ try:
460
+ with self._opener.open(request, timeout=timeout) as answer:
461
+ delivery = Delivery.of(answer.status)
462
+ said = _said(answer.read(INGEST_ANSWER_BYTES)) if delivery is Delivery.STORED else {}
463
+ return Answer(delivery, _refused(said), _refused_sessions(said))
464
+ except urllib.error.HTTPError as error:
465
+ error.close()
466
+ log_line(f"request ledger: the platform answered a flush {error.code}")
467
+ return Answer(Delivery.of(error.code))
468
+ except (urllib.error.URLError, OSError, ValueError, http.client.HTTPException) as error:
469
+ log_line(f"request ledger: the platform did not take a flush: {type(error).__name__}")
470
+ return Answer(Delivery.UNAVAILABLE)
471
+
472
+
473
+ def _said(answer: bytes) -> dict:
474
+ try:
475
+ said = json.loads(answer or b"{}")
476
+ except ValueError:
477
+ return {}
478
+ return said if isinstance(said, dict) else {}
479
+
480
+
481
+ def _refused(said: dict) -> int:
482
+ refused = said.get("refused", 0)
483
+ return refused if counted(refused) else 0
484
+
485
+
486
+ def _refused_sessions(said: dict) -> dict[str, int] | None:
487
+ sessions = said.get("refusedSessions")
488
+ if not isinstance(sessions, dict):
489
+ return None
490
+ return {sid: n for sid, n in sessions.items() if isinstance(sid, str) and counted(n) and n > 0}
491
+
492
+
493
+ @dataclass
494
+ class SendBackoff:
495
+ """How long the sender waits after a round the platform took none of: ``first_s``,
496
+ doubling to ``max_s``; a round it stores ends the wait. Nothing else cuts it short."""
497
+
498
+ first_s: float = LEDGER_FLUSH_S
499
+ max_s: float = LEDGER_BACKOFF_MAX_S
500
+ delay_s: float = 0.0
501
+ until: float = 0.0
502
+
503
+ def ready(self) -> bool:
504
+ return time.monotonic() >= self.until
505
+
506
+ def failed(self) -> None:
507
+ self.delay_s = min(self.max_s, max(self.first_s, 2 * self.delay_s))
508
+ self.until = time.monotonic() + self.delay_s
509
+
510
+ def cleared(self) -> None:
511
+ self.delay_s, self.until = 0.0, 0.0
512
+
513
+
514
+ class LedgerFlusher:
515
+ """Moves the host's outboxes to its spill and the spill to a :class:`LedgerSink`, on two
516
+ threads. The spiller moves every live session's outbox to disk every ``interval_s``, and an
517
+ outbox past half its ceiling at once; it never waits on the platform. The sender sends the
518
+ spill oldest first, one piece of every session a round, batched across sessions up to
519
+ ``batch_bytes`` a POST, each piece let go of once stored. A round the platform takes none of
520
+ starts a :class:`SendBackoff`; a batch it refuses as sent is resent a session at a time, and
521
+ a session refused alone is held for ``backoff.max_s`` while the rest go on. A session
522
+ leaving the host (:meth:`leave`), a dead worker's journal (:meth:`adopt`) and every session
523
+ when the host stops (:meth:`stop`) go to the spill too, their accounts closing once stored.
524
+ A worker runs tenant code, so a leaving session's take moves at most ``take_pages`` pages and
525
+ ``take_bytes`` bytes, within ``take_s`` of the worker's first answer, and counts the rest lost
526
+ (``takes_cut`` counts the cuts). A drain answer off the contract, or any failure, is that
527
+ session's alone."""
528
+
529
+ def __init__(
530
+ self,
531
+ host: WorldHost,
532
+ sink: LedgerSink,
533
+ spill: LedgerSpill,
534
+ *,
535
+ interval_s: float = LEDGER_FLUSH_S,
536
+ batch_bytes: int = LEDGER_BATCH_BYTES,
537
+ rounds: int = LEDGER_ROUNDS,
538
+ backoff: SendBackoff | None = None,
539
+ take_pages: int = TAKE_MAX_PAGES,
540
+ take_s: float = TAKE_DEADLINE_S,
541
+ take_bytes: int | None = None,
542
+ ) -> None:
543
+ self.host = host
544
+ self.sink = sink
545
+ self.spill = spill
546
+ self.interval_s = interval_s
547
+ self.batch_bytes = batch_bytes
548
+ self.rounds = rounds
549
+ self.backoff = backoff or SendBackoff(interval_s)
550
+ self.take_pages = take_pages
551
+ self.take_s = take_s
552
+ self.take_bytes = spill.max_bytes // 2 if take_bytes is None else take_bytes
553
+ self.takes_cut = 0
554
+ self.acked: dict[str, str] = {}
555
+ self.refused: collections.Counter[str] = collections.Counter()
556
+ self._moving: collections.Counter[str] = collections.Counter()
557
+ self._held: dict[str, float] = {}
558
+ self._pressed: set[str] = set()
559
+ self._next_pass = 0.0
560
+ self._turn = 0
561
+ self._lock = threading.Lock()
562
+ self._stop = threading.Event()
563
+ self._spill_due = threading.Event()
564
+ self._send_due = threading.Event()
565
+ self._threads: list[threading.Thread] = []
566
+
567
+ def start(self) -> LedgerFlusher:
568
+ self._threads = [
569
+ threading.Thread(target=self._spilling, name="world-host-ledger-spill", daemon=True),
570
+ threading.Thread(target=self._sending, name="world-host-ledger-send", daemon=True),
571
+ ]
572
+ for thread in self._threads:
573
+ thread.start()
574
+ return self
575
+
576
+ def wake(self, session_id: str) -> None:
577
+ with self._lock:
578
+ self._pressed.add(session_id)
579
+ self._spill_due.set()
580
+
581
+ def stop(self, deadline_s: float = LEDGER_STOP_S, *, settle: Callable[[float], object] = _settled) -> None:
582
+ """Within ``deadline_s`` in all, the time the host's grace leaves after its drain: let the last
583
+ finishes run (``settle``, half of it, both threads still running), then stop both threads, send
584
+ what the spill holds, move every live outbox to it and close each account (complete when
585
+ everything was moved or counted), and send once more: no drain or send outlives it, and what is
586
+ not stored by then stays on disk for the next host on this cache."""
587
+ until = time.monotonic() + deadline_s
588
+ settle(deadline_s / 2)
589
+ self._stop.set()
590
+ self._spill_due.set()
591
+ self._send_due.set()
592
+ for thread in self._threads:
593
+ thread.join(max(0.0, until - time.monotonic()))
594
+ self._guarded(partial(self._last, until))
595
+
596
+ def flush(self, until: float | None = None) -> None:
597
+ self.spill_live(until)
598
+ self.replay(until)
599
+
600
+ def leave(self, session: Session) -> None:
601
+ """A take for a session whose state goes next: what the take could not reach is counted lost
602
+ and the account closes, complete unless the move failed."""
603
+ self._leave(session.session_id, partial(self.host.ledger_page, session))
604
+ with self._lock:
605
+ self.acked.pop(session.session_id, None)
606
+
607
+ def settle(self, session: Session, until: float) -> LedgerAccount:
608
+ """Send everything this host holds of a session about to move to another host, by ``until``:
609
+ its outbox moved to the spill and every piece of it stored. Final when all of it is stored and
610
+ no account of it closed incomplete; what the platform refused of it is counted lost, so an
611
+ account is never final over a refusal it does not count. The next host keeps its own."""
612
+ session_id = session.session_id
613
+ page, sent = self._durable(session_id, partial(self.host.ledger_page, session), until)
614
+ shed, lost = self.spill.counts(session_id)
615
+ with self._lock:
616
+ refused = self.refused[session_id]
617
+ final = page is not None and not page.more and sent and not self.spill.incomplete(session_id)
618
+ return LedgerAccount(final, shed, lost + refused)
619
+
620
+ def barrier(self, session: Session, deadline_s: float = LEDGER_BARRIER_S) -> dict:
621
+ """A grade's wait: the session's outbox moved to the spill and every piece of it sent, by
622
+ ``deadline_s``; stored only when nothing of it waits here, the platform refused none of it and
623
+ the outbox can say it holds every request it was given. What was shed or lost is counted."""
624
+ session_id = session.session_id
625
+ until = time.monotonic() + deadline_s
626
+ page, sent = self._durable(session_id, partial(self.host.requests, session_id), until)
627
+ shed, lost = self.spill.counts(session_id)
628
+ held = session_id in self._holding()
629
+ with self._lock:
630
+ refused = self.refused[session_id]
631
+ reasons = (
632
+ (page is None, "the session's outbox could not be read"),
633
+ (page is not None and page.more, "the session's outbox was not moved before the deadline"),
634
+ (held, "the platform refused the session's requests as sent; they wait on this host"),
635
+ (not sent, "the platform did not take the session's requests before the deadline"),
636
+ (refused > 0, f"the platform refused {refused} requests of a flush that carried this session's"),
637
+ (page is not None and page.inherited is not None, page.inherited if page is not None else None),
638
+ )
639
+ reason = next((said for found, said in reasons if found), None)
640
+ return _unstored(reason, shed, lost + refused) if reason else {"stored": True, "shed": shed, "lost": lost}
641
+
642
+ def _durable(self, session_id: str, drain: Callable[..., object], until: float) -> tuple[OutboxPage | None, bool]:
643
+ """The session's outbox moved to the spill and its pieces sent by ``until``: the last page (None
644
+ when it could not be read) and whether nothing of the session waits here any more."""
645
+ with self._lock:
646
+ self._moving[session_id] += 1
647
+ try:
648
+ page = self._each(session_id, partial(self._drained, session_id, drain, until))
649
+ finally:
650
+ with self._lock:
651
+ self._moving[session_id] -= 1
652
+ self._moving += collections.Counter()
653
+ moved = isinstance(page, OutboxPage)
654
+ return (page if moved else None), moved and self._sent(session_id, until)
655
+
656
+ def _drained(self, session_id: str, drain: Callable[..., object], until: float) -> OutboxPage | None:
657
+ """Until a drain answers nothing, so the outbox has let go of everything the spill holds."""
658
+ page = self._move(session_id, drain, 1, until)
659
+ while page is not None and (page.more or page.lines) and not _late(until):
660
+ page = self._move(session_id, drain, TAKE_MAX_PAGES, until)
661
+ return page
662
+
663
+ def _sent(self, session_id: str, until: float) -> bool:
664
+ while self.spill.pending(session_id):
665
+ held = session_id in self._holding()
666
+ if held or _late(until):
667
+ return False
668
+ pieces = self.spill.round(self._holding(), only=session_id)
669
+ stored = self._send_round(pieces, until) if pieces else None
670
+ if stored is False:
671
+ return False
672
+ time.sleep(0 if pieces else BARRIER_POLL_S)
673
+ return True
674
+
675
+ def adopt(self, session_id: str, entries: list[dict]) -> None:
676
+ """A session's entries a dead worker's journal kept, moved to the spill; its account closes
677
+ incomplete, since a dead worker's last entries are not known to be in its journal."""
678
+ for run in runs([ledger_line(entry) for entry in entries], self.batch_bytes):
679
+ self._each(session_id, partial(self._adopted, session_id, run))
680
+ self._each(session_id, partial(self.spill.finished, session_id, False))
681
+
682
+ def _adopted(self, session_id: str, run: list[bytes | None]) -> None:
683
+ """A run of a dead worker's entries into the spill; one the disk refuses is counted lost, since
684
+ nothing else holds it any more."""
685
+ try:
686
+ self.spill.write(session_id, run)
687
+ except OSError:
688
+ self.spill.lose(session_id, len(run))
689
+ raise
690
+
691
+ def spill_live(self, until: float | None = None, only: set[str] | None = None) -> None:
692
+ """Move each live session's outbox (or ``only`` those) to the spill, at most ``rounds`` pages each
693
+ and none past ``until``, starting one session further along each pass so a slow one holds back
694
+ no other for long, and forget what the host no longer serves and the platform has."""
695
+ self.spill.settle(set(self.host.session_ids(draining=True)))
696
+ live = self.host.session_ids()
697
+ with self._lock:
698
+ self.acked = {sid: cursor for sid, cursor in self.acked.items() if sid in live}
699
+ self.refused = collections.Counter({sid: n for sid, n in self.refused.items() if sid in live})
700
+ for session_id in self._rotated([sid for sid in live if only is None or sid in only]):
701
+ if _late(until):
702
+ break
703
+ self._each(session_id, partial(self._keep, session_id, until))
704
+
705
+ def replay(self, until: float | None = None) -> None:
706
+ """Send the spill until it is empty, a round finds the platform taking none of it (a backoff
707
+ starts), or ``until`` (unset: until the flusher stops)."""
708
+ while not self._over(until):
709
+ pieces = self.spill.round(self._holding())
710
+ if not pieces:
711
+ return
712
+ if not self._send_round(pieces, until):
713
+ return self.backoff.failed()
714
+ self.backoff.cleared()
715
+
716
+ def _spilling(self) -> None:
717
+ while not self._stop.is_set():
718
+ self._spill_due.wait(max(0.0, self._next_pass - time.monotonic()))
719
+ self._spill_due.clear()
720
+ if self._stop.is_set():
721
+ return
722
+ self._guarded(self._spill_due_outboxes)
723
+ self._send_due.set()
724
+
725
+ def _sending(self) -> None:
726
+ while not self._stop.is_set():
727
+ self._send_due.wait(self.interval_s)
728
+ self._send_due.clear()
729
+ if not self._stop.is_set() and self.backoff.ready():
730
+ self._guarded(self.replay)
731
+
732
+ def _spill_due_outboxes(self) -> None:
733
+ with self._lock:
734
+ pressed, self._pressed = self._pressed, set()
735
+ whole = time.monotonic() >= self._next_pass
736
+ self._next_pass = time.monotonic() + self.interval_s if whole else self._next_pass
737
+ self.spill_live(time.monotonic() + LEDGER_PASS_S, None if whole else pressed)
738
+
739
+ def _last(self, until: float) -> None:
740
+ self.replay(until)
741
+ for session_id in self._rotated(self.host.session_ids()):
742
+ if _late(until):
743
+ break
744
+ self._leave(session_id, partial(self.host.requests, session_id), until)
745
+ self.replay(until)
746
+
747
+ def _take(self, session_id: str, drain: Callable[..., object], until: float | None = None) -> OutboxPage | None:
748
+ """The outbox to the spill within a take's bounds and ``until``: ``take_s`` start at the worker's
749
+ first answer; answers the last page."""
750
+ first = self._move(session_id, drain, 1, until)
751
+ bounded = time.monotonic() + self.take_s
752
+ if first is None or not first.more:
753
+ return first
754
+ budget = _Budget(self.take_bytes - first.size, 1)
755
+ cut = self._move(session_id, drain, self.take_pages - 1, min(until or bounded, bounded), budget, first)
756
+ assert cut is not None
757
+ if cut.more:
758
+ self._cut(session_id, cut, budget.pages)
759
+ return cut
760
+
761
+ def _cut(self, session_id: str, page: OutboxPage, pages: int) -> None:
762
+ with self._lock:
763
+ self.takes_cut += 1
764
+ log_line(
765
+ f"request ledger: session {session_id} stopped after {pages} pages with {page.pending} entries"
766
+ " still on its outbox"
767
+ )
768
+
769
+ def _rotated(self, session_ids: list[str]) -> list[str]:
770
+ start = self._turn % len(session_ids) if session_ids else 0
771
+ self._turn += 1
772
+ return session_ids[start:] + session_ids[:start]
773
+
774
+ def _keep(self, session_id: str, until: float | None) -> None:
775
+ """A spiller pass's move of one session, left to a barrier or a settle moving it now."""
776
+ with self._lock:
777
+ moving = self._moving[session_id] > 0
778
+ if not moving:
779
+ self._move(session_id, partial(self.host.requests, session_id), self.rounds, until)
780
+
781
+ def _leave(self, session_id: str, drain: Callable[..., object], until: float | None = None) -> None:
782
+ """Move what a session leaving the host still holds, count what the move could not reach, and
783
+ close its account: complete unless the move failed. A move that failed once its outbox answered
784
+ a page counts what that page said was left; one that failed before has nothing to count."""
785
+ moved, left = self._each(session_id, partial(self._taken, session_id, drain, until)) or (False, 0)
786
+ self._each(session_id, partial(self._close, session_id, moved, left))
787
+
788
+ def _taken(self, session_id: str, drain: Callable[..., object], until: float | None) -> tuple[bool, int]:
789
+ try:
790
+ last = self._take(session_id, drain, until)
791
+ except MoveFailed as failed:
792
+ self.spill.lose(session_id, failed.left)
793
+ raise
794
+ moved = last is not None
795
+ return moved, last.pending if moved and last.more else 0
796
+
797
+ def _close(self, session_id: str, moved: bool, left: int) -> None:
798
+ self.spill.lose(session_id, left)
799
+ self.spill.finished(session_id, moved)
800
+
801
+ def _move(
802
+ self,
803
+ session_id: str,
804
+ drain: Callable[..., object],
805
+ pages: int,
806
+ until: float | None,
807
+ budget: _Budget | None = None,
808
+ page: OutboxPage | None = None,
809
+ ) -> OutboxPage | None:
810
+ """The session's outbox to the spill a page at a time, at most ``pages``, none past ``until`` and
811
+ none past ``budget``'s bytes, each page let go of by the outbox's next drain once it is on disk;
812
+ answers the last page the outbox answered, ``page`` (the one before this move) when it moved none.
813
+ A move that fails once the outbox answered a page raises :class:`MoveFailed`."""
814
+ for _ in range(pages):
815
+ if _late(until) or (budget is not None and budget.spent()):
816
+ return page
817
+ page = self._moved(session_id, drain, page, until)
818
+ if budget is not None:
819
+ budget.spend(page.size)
820
+ if not page.more:
821
+ return page
822
+ return page
823
+
824
+ def _moved(
825
+ self, session_id: str, drain: Callable[..., object], last: OutboxPage | None, until: float | None
826
+ ) -> OutboxPage:
827
+ try:
828
+ page = OutboxPage.of(drain(self.acked.get(session_id), timeout=_timeout(until)))
829
+ except Exception as error:
830
+ if last is None:
831
+ raise
832
+ raise MoveFailed(last, kept=True) from error
833
+ self._note(session_id, page)
834
+ try:
835
+ self._keep_page(session_id, page)
836
+ except OSError as error:
837
+ raise MoveFailed(page, kept=False) from error
838
+ return page
839
+
840
+ def _keep_page(self, session_id: str, page: OutboxPage) -> None:
841
+ if not page.lines:
842
+ return
843
+ self.spill.write(session_id, page.lines, page.epoch, page.source)
844
+ with self._lock:
845
+ self.acked[session_id] = page.cursor
846
+
847
+ def _note(self, session_id: str, page: OutboxPage) -> None:
848
+ if self.spill.outbox(session_id, page.epoch, page.shed, page.lost, page.source):
849
+ log_line(
850
+ f"request ledger: session {session_id} outbox shed the headers and bodies of {page.shed} requests,"
851
+ f" lost {page.lost} entries"
852
+ )
853
+
854
+ def _send_round(self, pieces: list[Piece], until: float | None) -> bool:
855
+ """One piece of every session, in batches read from disk one at a time, until a batch finds
856
+ the platform unavailable; True when the platform stored any of it."""
857
+ outcomes: list[Delivery] = []
858
+ try:
859
+ for ready in self._ready(pieces):
860
+ outcome = Delivery.UNAVAILABLE if self._over(until) else self._deliver(ready, until)
861
+ outcomes.append(outcome)
862
+ if outcome is Delivery.UNAVAILABLE:
863
+ break
864
+ finally:
865
+ self.spill.released(pieces)
866
+ return Delivery.STORED in outcomes
867
+
868
+ def _deliver(self, batch: list[tuple[Piece, dict]], until: float | None) -> Delivery:
869
+ """One batch; STORED when any of it was stored, what the platform refused of it recorded before
870
+ its pieces read as stored, so a barrier never answers stored over a refusal. A batch of several
871
+ the platform refuses as sent goes again a session at a time; a session it refuses alone is held."""
872
+ answer = self._send([item for _, item in batch], until)
873
+ outcome = answer.delivery
874
+ if outcome is Delivery.STORED:
875
+ self._refused(batch, answer)
876
+ self.spill.stored([piece for piece, _ in batch])
877
+ if outcome is Delivery.REFUSED and len(batch) == 1:
878
+ self._hold(batch[0][0])
879
+ if outcome is not Delivery.REFUSED or len(batch) == 1:
880
+ return outcome
881
+ return self._one_by_one(batch, until)
882
+
883
+ def _one_by_one(self, batch: list[tuple[Piece, dict]], until: float | None) -> Delivery:
884
+ outcomes: list[Delivery] = []
885
+ for one in batch:
886
+ outcomes.append(self._deliver([one], until))
887
+ if outcomes[-1] is Delivery.UNAVAILABLE:
888
+ return Delivery.UNAVAILABLE
889
+ return Delivery.STORED if Delivery.STORED in outcomes else Delivery.REFUSED
890
+
891
+ def _refused(self, batch: list[tuple[Piece, dict]], answer: Answer) -> None:
892
+ """What the platform refused of a stored batch, each session's own when it said, else the batch's
893
+ count against every session in it."""
894
+ if not answer.refused:
895
+ return
896
+ log_line(f"request ledger: the platform refused {answer.refused} requests of a flush")
897
+ sessions = {piece.account.session_id for piece, _ in batch}
898
+ each = answer.by_session or {sid: answer.refused for sid in sessions}
899
+ with self._lock:
900
+ for session_id in sessions:
901
+ self.refused[session_id] += each.get(session_id, 0)
902
+
903
+ def _hold(self, piece: Piece) -> None:
904
+ session_id = piece.account.session_id
905
+ log_line(f"request ledger: the platform refused session {session_id}'s entries; held {self.backoff.max_s:g} s")
906
+ with self._lock:
907
+ self._held[session_id] = time.monotonic() + self.backoff.max_s
908
+
909
+ def _holding(self) -> set[str]:
910
+ now = time.monotonic()
911
+ with self._lock:
912
+ self._held = {sid: until for sid, until in self._held.items() if until > now}
913
+ return set(self._held)
914
+
915
+ def _ready(self, pieces: list[Piece]) -> Iterator[list[tuple[Piece, dict]]]:
916
+ """The pieces in batches of at most ``batch_bytes``, each read from disk as it is reached; a piece
917
+ whose segment could not be read this round is left out."""
918
+ for batch in self._batches(pieces):
919
+ items = [(piece, self.spill.item(piece)) for piece in batch]
920
+ ready = [(piece, item) for piece, item in items if item is not None]
921
+ if ready:
922
+ yield ready
923
+
924
+ def _batches(self, pieces: list[Piece]) -> list[list[Piece]]:
925
+ batches: list[list[Piece]] = []
926
+ spent = self.batch_bytes
927
+ for piece in pieces:
928
+ if spent + piece.size > self.batch_bytes:
929
+ batches.append([])
930
+ spent = 0
931
+ batches[-1].append(piece)
932
+ spent += piece.size
933
+ return batches
934
+
935
+ def _send(self, items: list[dict], until: float | None) -> Answer:
936
+ try:
937
+ return Answer.of(self.sink.send(items, until))
938
+ except Exception as error: # noqa: BLE001 - a sink that fails is a send not taken, retried
939
+ log_line(f"request ledger: a flush failed: {type(error).__name__}")
940
+ return Answer(Delivery.UNAVAILABLE)
941
+
942
+ def _over(self, until: float | None) -> bool:
943
+ return _late(until) if until is not None else self._stop.is_set()
944
+
945
+ @staticmethod
946
+ def _each(session_id: str, work: Callable[[], object]) -> object:
947
+ try:
948
+ return work()
949
+ except Exception as error: # noqa: BLE001 - one session's ledger never stops another's
950
+ log_line(f"request ledger: session {session_id} not spilled: {type(error.__cause__ or error).__name__}")
951
+ return None
952
+
953
+ @staticmethod
954
+ def _guarded(work: Callable[[], object]) -> None:
955
+ try:
956
+ work()
957
+ except Exception as error: # noqa: BLE001 - the flusher outlives any one pass
958
+ log_line(f"request ledger: a flush failed: {type(error).__name__}")