@withgateway/sdk 0.3.1 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. package/README.md +8 -6
  2. package/dist/cli.js +4372 -2265
  3. package/dist/cli.js.map +1 -1
  4. package/dist/cli.mjs +4362 -2256
  5. package/dist/cli.mjs.map +1 -1
  6. package/dist/files/index.d.mts +1 -1
  7. package/dist/files/index.d.ts +1 -1
  8. package/dist/files/index.js +259 -58
  9. package/dist/files/index.js.map +1 -1
  10. package/dist/files/index.mjs +249 -50
  11. package/dist/files/index.mjs.map +1 -1
  12. package/dist/hooks/index.d.mts +1 -1
  13. package/dist/hooks/index.d.ts +1 -1
  14. package/dist/hooks/index.js +124 -11
  15. package/dist/hooks/index.js.map +1 -1
  16. package/dist/hooks/index.mjs +124 -11
  17. package/dist/hooks/index.mjs.map +1 -1
  18. package/dist/{index-C3UzD0G3.d.mts → index-CDPQBsU5.d.mts} +19 -3
  19. package/dist/{index-C3UzD0G3.d.ts → index-CDPQBsU5.d.ts} +19 -3
  20. package/dist/{index-CBqbB_0J.d.ts → index-DUHmS74p.d.ts} +1 -1
  21. package/dist/{index-D-AMnAMa.d.mts → index-mJhG3vZU.d.mts} +1 -1
  22. package/dist/index.d.mts +5 -5
  23. package/dist/index.d.ts +5 -5
  24. package/dist/index.js +2240 -789
  25. package/dist/index.js.map +1 -1
  26. package/dist/index.mjs +2150 -739
  27. package/dist/index.mjs.map +1 -1
  28. package/dist/{media-CbcAfQvd.d.mts → media-BIApiPzS.d.mts} +21 -2
  29. package/dist/{media-CbcAfQvd.d.ts → media-BIApiPzS.d.ts} +21 -2
  30. package/dist/traces/index.d.mts +1 -1
  31. package/dist/traces/index.d.ts +1 -1
  32. package/dist/traces/index.js +253 -52
  33. package/dist/traces/index.js.map +1 -1
  34. package/dist/traces/index.mjs +243 -44
  35. package/dist/traces/index.mjs.map +1 -1
  36. package/dist/tracing/index.d.mts +2 -2
  37. package/dist/tracing/index.d.ts +2 -2
  38. package/dist/tracing/index.js +75 -13
  39. package/dist/tracing/index.js.map +1 -1
  40. package/dist/tracing/index.mjs +75 -13
  41. package/dist/tracing/index.mjs.map +1 -1
  42. package/dist/worlds/index.d.mts +893 -482
  43. package/dist/worlds/index.d.ts +893 -482
  44. package/dist/worlds/index.js +1968 -669
  45. package/dist/worlds/index.js.map +1 -1
  46. package/dist/worlds/index.mjs +1960 -662
  47. package/dist/worlds/index.mjs.map +1 -1
  48. package/package.json +1 -1
  49. package/runtime/gatewaysdk/environment/_bundle.py +2 -27
  50. package/runtime/gatewaysdk/environment/_materialize.py +48 -88
  51. package/runtime/gatewaysdk/environment/_world.py +155 -237
  52. package/runtime/gatewaysdk/environment/backing.py +133 -0
  53. package/runtime/gatewaysdk/environment/core.py +2 -17
  54. package/runtime/gatewaysdk/environment/legacy_sqlite.py +288 -0
  55. package/runtime/gatewaysdk/environment/reward_keys.py +25 -0
  56. package/runtime/gatewaysdk/environment/runtime.py +8 -13
  57. package/runtime/gatewaysdk/environment/schema/__init__.py +4 -4
  58. package/runtime/gatewaysdk/environment/schema/admin_routes.py +86 -0
  59. package/runtime/gatewaysdk/environment/schema/admission.py +174 -0
  60. package/runtime/gatewaysdk/environment/schema/api.py +347 -892
  61. package/runtime/gatewaysdk/environment/schema/assertions.py +458 -0
  62. package/runtime/gatewaysdk/environment/schema/batch.py +44 -39
  63. package/runtime/gatewaysdk/environment/schema/bundle_runtime.py +17 -5
  64. package/runtime/gatewaysdk/environment/schema/cache_layout.py +5 -0
  65. package/runtime/gatewaysdk/environment/schema/call_capture.py +309 -0
  66. package/runtime/gatewaysdk/environment/schema/call_log.py +503 -0
  67. package/runtime/gatewaysdk/environment/schema/call_redaction.py +526 -0
  68. package/runtime/gatewaysdk/environment/schema/cast.py +15 -35
  69. package/runtime/gatewaysdk/environment/schema/compiler.py +334 -158
  70. package/runtime/gatewaysdk/environment/schema/conform.py +14 -39
  71. package/runtime/gatewaysdk/environment/schema/connector.py +113 -211
  72. package/runtime/gatewaysdk/environment/schema/connectors/confluence/connector.toml +1 -12
  73. package/runtime/gatewaysdk/environment/schema/connectors/github/connector.toml +1 -0
  74. package/runtime/gatewaysdk/environment/schema/connectors/servicenow/connector.toml +2 -5
  75. package/runtime/gatewaysdk/environment/schema/cors.py +54 -0
  76. package/runtime/gatewaysdk/environment/schema/credentials.py +313 -8
  77. package/runtime/gatewaysdk/environment/schema/data.py +60 -216
  78. package/runtime/gatewaysdk/environment/schema/database/__init__.py +24 -0
  79. package/runtime/gatewaysdk/environment/schema/database/audit.py +121 -0
  80. package/runtime/gatewaysdk/environment/schema/database/base.py +936 -0
  81. package/runtime/gatewaysdk/environment/schema/database/budget.py +33 -0
  82. package/runtime/gatewaysdk/environment/schema/database/connections.py +101 -0
  83. package/runtime/gatewaysdk/environment/schema/database/engines.py +8 -0
  84. package/runtime/gatewaysdk/environment/schema/database/guard.py +215 -0
  85. package/runtime/gatewaysdk/environment/schema/database/ingest.py +154 -0
  86. package/runtime/gatewaysdk/environment/schema/database/matching.py +58 -0
  87. package/runtime/gatewaysdk/environment/schema/database/provenance.py +30 -0
  88. package/runtime/gatewaysdk/environment/schema/database/reads.py +169 -0
  89. package/runtime/gatewaysdk/environment/schema/database/rows.py +53 -0
  90. package/runtime/gatewaysdk/environment/schema/database/runtime_rows.py +39 -0
  91. package/runtime/gatewaysdk/environment/schema/database/sqlite.py +393 -0
  92. package/runtime/gatewaysdk/environment/schema/declared_routes.py +210 -0
  93. package/runtime/gatewaysdk/environment/schema/domain_migration.py +236 -0
  94. package/runtime/gatewaysdk/environment/schema/door.py +899 -0
  95. package/runtime/gatewaysdk/environment/schema/envelope.py +227 -0
  96. package/runtime/gatewaysdk/environment/schema/extract.py +6 -13
  97. package/runtime/gatewaysdk/environment/schema/grading.py +90 -103
  98. package/runtime/gatewaysdk/environment/schema/handler_errors.py +163 -12
  99. package/runtime/gatewaysdk/environment/schema/host.py +237 -1795
  100. package/runtime/gatewaysdk/environment/schema/host_control.py +144 -0
  101. package/runtime/gatewaysdk/environment/schema/host_handoff.py +370 -0
  102. package/runtime/gatewaysdk/environment/schema/host_loops.py +97 -0
  103. package/runtime/gatewaysdk/environment/schema/host_owners.py +121 -0
  104. package/runtime/gatewaysdk/environment/schema/host_proxy.py +111 -0
  105. package/runtime/gatewaysdk/environment/schema/host_requests.py +6 -0
  106. package/runtime/gatewaysdk/environment/schema/host_session.py +126 -0
  107. package/runtime/gatewaysdk/environment/schema/host_surfaces.py +79 -21
  108. package/runtime/gatewaysdk/environment/schema/host_transfers.py +127 -0
  109. package/runtime/gatewaysdk/environment/schema/host_versions.py +242 -0
  110. package/runtime/gatewaysdk/environment/schema/host_workers.py +577 -0
  111. package/runtime/gatewaysdk/environment/schema/journal.py +437 -0
  112. package/runtime/gatewaysdk/environment/schema/lanes.py +53 -0
  113. package/runtime/gatewaysdk/environment/schema/ledger_spill.py +828 -0
  114. package/runtime/gatewaysdk/environment/schema/local.py +58 -170
  115. package/runtime/gatewaysdk/environment/schema/mapping.py +5 -25
  116. package/runtime/gatewaysdk/environment/schema/openapi_import.py +112 -73
  117. package/runtime/gatewaysdk/environment/schema/openapi_paging.py +147 -0
  118. package/runtime/gatewaysdk/environment/schema/openapi_payloads.py +288 -0
  119. package/runtime/gatewaysdk/environment/schema/openapi_routes.py +97 -102
  120. package/runtime/gatewaysdk/environment/schema/openapi_schemas.py +113 -30
  121. package/runtime/gatewaysdk/environment/schema/platform.py +137 -187
  122. package/runtime/gatewaysdk/environment/schema/platform_import.py +2 -3
  123. package/runtime/gatewaysdk/environment/schema/platform_query.py +61 -5
  124. package/runtime/gatewaysdk/environment/schema/primary_key.py +61 -0
  125. package/runtime/gatewaysdk/environment/schema/proof.py +19 -12
  126. package/runtime/gatewaysdk/environment/schema/request_body.py +238 -0
  127. package/runtime/gatewaysdk/environment/schema/request_ledger.py +958 -0
  128. package/runtime/gatewaysdk/environment/schema/route_bindings.py +25 -3
  129. package/runtime/gatewaysdk/environment/schema/route_miss.py +54 -0
  130. package/runtime/gatewaysdk/environment/schema/runtime.py +178 -308
  131. package/runtime/gatewaysdk/environment/schema/served_names.py +58 -0
  132. package/runtime/gatewaysdk/environment/schema/session_files.py +275 -0
  133. package/runtime/gatewaysdk/environment/schema/session_move.py +275 -0
  134. package/runtime/gatewaysdk/environment/schema/skills/agents/connector-template-author.md +5 -1
  135. package/runtime/gatewaysdk/environment/schema/skills/benchmarks-getting-started/SKILL.md +10 -11
  136. package/runtime/gatewaysdk/environment/schema/skills/connector-schema-authoring/SKILL.md +49 -16
  137. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/ROWS.md +39 -9
  138. package/runtime/gatewaysdk/environment/schema/skills/world-data-ingestion/SKILL.md +27 -27
  139. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/SKILL.md +33 -33
  140. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
  141. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/gotchas.md +49 -1
  142. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/journal-checks.md +182 -0
  143. package/runtime/gatewaysdk/environment/schema/skills/worlds-getting-started/references/openapi-import.md +50 -21
  144. package/runtime/gatewaysdk/environment/schema/slack.py +4 -8
  145. package/runtime/gatewaysdk/environment/schema/snapshot.py +516 -140
  146. package/runtime/gatewaysdk/environment/schema/spill_files.py +228 -0
  147. package/runtime/gatewaysdk/environment/schema/store_contract.py +8 -14
  148. package/runtime/gatewaysdk/environment/schema/tasks.py +267 -276
  149. package/runtime/gatewaysdk/environment/schema/tools_world.py +146 -242
  150. package/runtime/gatewaysdk/environment/schema/ui.py +24 -5
  151. package/runtime/gatewaysdk/environment/schema/ui_static/app.js +100 -130
  152. package/runtime/gatewaysdk/environment/schema/ui_static/index.html +1 -8
  153. package/runtime/gatewaysdk/environment/schema/validation.py +235 -41
  154. package/runtime/gatewaysdk/environment/schema/verbs.py +32 -36
  155. package/runtime/gatewaysdk/environment/schema/worker.py +6 -4
  156. package/runtime/gatewaysdk/environment/schema/workers.py +345 -96
  157. package/runtime/gatewaysdk/environment/schema/world_host.py +953 -0
  158. package/runtime/gatewaysdk/environment/schema/world_sql.py +81 -0
  159. package/runtime/gatewaysdk/environment/schema/world_tests.py +80 -50
  160. package/runtime/gatewaysdk/environment/score.py +57 -0
  161. package/runtime/gatewaysdk/environment/secret_values.py +36 -0
  162. package/runtime/gatewaysdk/environment/world.py +43 -4
  163. package/runtime/gatewaysdk/environment/world_session_contract.py +46 -5
  164. package/skills/agents/connector-template-author.md +5 -1
  165. package/skills/benchmarks-getting-started/SKILL.md +10 -11
  166. package/skills/connector-schema-authoring/SKILL.md +49 -16
  167. package/skills/connector-schema-authoring/connector-worlds.md +142 -11
  168. package/skills/connector-schema-authoring/world-schema.md +103 -51
  169. package/skills/world-data-ingestion/ROWS.md +39 -9
  170. package/skills/world-data-ingestion/SKILL.md +27 -27
  171. package/skills/worlds-getting-started/SKILL.md +33 -33
  172. package/skills/worlds-getting-started/references/cli-mcp-map.md +13 -7
  173. package/skills/worlds-getting-started/references/gotchas.md +49 -1
  174. package/skills/worlds-getting-started/references/journal-checks.md +182 -0
  175. package/skills/worlds-getting-started/references/openapi-import.md +50 -21
  176. package/runtime/gatewaysdk/environment/schema/store.py +0 -1013
@@ -0,0 +1,828 @@
1
+ """The World Host's disk for its request ledger: what the flusher drained from a session's outbox waits
2
+ here, in the session's order, until the platform has stored it, beside the session's account on this
3
+ host (what it lost, and whether everything it held was accounted for).
4
+
5
+ Each account is a directory named by its random ``source``, holding its record (``account.jsonl``: a
6
+ snapshot of the account and the segments it holds, then a line per change, compacted as it grows) and its
7
+ entries as numbered segments, one entry a line. Every line is sealed (:class:`~.spill_files.SpillSeal`) and
8
+ bound to its account and place: a segment line to its segment and ordinal, a record line to the one before
9
+ it. So a line a version worker wrote, copied, moved or changed is never sent; what a segment the record
10
+ lists no longer holds in place, whether it is gone, unreadable, short or reordered, is counted lost when the
11
+ platform stores the rest of it, or when a restarted host takes the account; and a restarted host never takes
12
+ an account as complete whose record is broken or whose directory holds a segment the record does not list.
13
+ An account belongs to one outbox (``epoch``), so whatever reaches the spill after a session's account closed
14
+ joins that account while the host holds it; a page that lands after the account was settled opens another,
15
+ never final. Past ``max_bytes`` the host's oldest segments go, counted lost. Every file goes through
16
+ :class:`~.spill_files.SpillDirectory`, keeping only entries the platform parses (:func:`ledger_line`)."""
17
+
18
+ from __future__ import annotations
19
+
20
+ import collections
21
+ import errno
22
+ import json
23
+ import os
24
+ import re
25
+ import stat
26
+ import threading
27
+ import uuid
28
+ from collections.abc import Iterator
29
+ from dataclasses import dataclass, field
30
+ from functools import partial
31
+ from pathlib import Path
32
+ from typing import Callable, Optional, TypeVar
33
+
34
+ from .cache_layout import SESSIONS_DIR, VERSIONS_DIR, WORKERS_DIR
35
+ from .call_log import _children
36
+ from .handler_errors import log_line
37
+ from .spill_files import TOMBSTONE, SpillDirectory, SpillSeal
38
+
39
+ SPILL_DIR = "ledger"
40
+ DEFAULT_SPILL_MB = 4096
41
+ HOST_DIRS = (SESSIONS_DIR, VERSIONS_DIR, WORKERS_DIR)
42
+ LEDGER_BATCH_BYTES = 4 << 20
43
+ SEGMENT_READ_BYTES = LEDGER_BATCH_BYTES + (1 << 20)
44
+ SEGMENT_TARGET_BYTES = 256 << 10
45
+ SEGMENTS_IN_CAP = 64
46
+ SEGMENT_MISSES = 3
47
+ ACCOUNT_BYTES = 256
48
+ ACCOUNT_FILE = "account.jsonl"
49
+ ACCOUNT_READ_BYTES = 8 << 20
50
+ RECORD_LINES_MIN = 64
51
+ SESSION_ID_CHARS = 512
52
+ OUTBOXES_MAX = 64
53
+ COUNT_MAX = (1 << 53) - 1
54
+ ENTRY_MAX_DEPTH = 64
55
+ HEX32 = re.compile(r"[0-9a-f]{32}")
56
+ SEGMENT = re.compile(r"[0-9]{12}\.jsonl")
57
+ WHOLE = re.compile(r"[0-9]{1,9}")
58
+
59
+
60
+ def counted(value: object) -> bool:
61
+ return type(value) is int and 0 <= value <= COUNT_MAX
62
+
63
+
64
+ def _deeper_than(entry: dict, levels: int) -> bool:
65
+ """Whether ``entry`` holds a value more than ``levels`` below itself, found without recursing:
66
+ the encoder's own depth limit moves with the frame it runs in, so an entry near it encodes in
67
+ one place and raises in the next."""
68
+ layer = [entry]
69
+ for _ in range(levels + 1):
70
+ layer = [child for node in layer for child in _children(node)]
71
+ return bool(layer)
72
+
73
+
74
+ def ledger_line(entry: object) -> bytes | None:
75
+ """``entry`` as the JSON the spill keeps, or None when the platform would not parse it: one it cannot
76
+ parse (a non-finite number from a worker's tenant code) refuses every session sent beside it, and one
77
+ nested past :data:`ENTRY_MAX_DEPTH` would fail the flush that encodes it."""
78
+ if not (isinstance(entry, dict) and isinstance(entry.get("id"), str) and HEX32.fullmatch(entry["id"])):
79
+ return None
80
+ if _deeper_than(entry, ENTRY_MAX_DEPTH):
81
+ return None
82
+ try:
83
+ text = json.dumps(entry, ensure_ascii=False, separators=(",", ":"), allow_nan=False)
84
+ except (TypeError, ValueError, RecursionError):
85
+ return None
86
+ data = text.encode("utf-8", "replace")
87
+ return data if len(data) < LEDGER_BATCH_BYTES else None
88
+
89
+
90
+ Line = TypeVar("Line", bytes, Optional[bytes])
91
+
92
+
93
+ def runs(lines: list[Line], max_bytes: int, per_line: int = 0) -> Iterator[list[Line]]:
94
+ """``lines`` in order, in runs of at most ``max_bytes``, each line ``per_line`` more (always one)."""
95
+ run: list[Line] = []
96
+ spent = 0
97
+ for line in lines:
98
+ size = len(line or b"") + per_line
99
+ if run and spent + size > max_bytes:
100
+ yield run
101
+ run, spent = [], 0
102
+ run.append(line)
103
+ spent += size
104
+ if run:
105
+ yield run
106
+
107
+
108
+ def _not_json(constant: str) -> None:
109
+ raise ValueError(constant)
110
+
111
+
112
+ def _json(raw: bytes | None) -> object:
113
+ try:
114
+ return json.loads(raw or b"", parse_constant=_not_json)
115
+ except (ValueError, RecursionError):
116
+ return None
117
+
118
+
119
+ def _entry(data: bytes | None) -> dict | None:
120
+ entry = _json(data)
121
+ return entry if ledger_line(entry) is not None else None
122
+
123
+
124
+ def _lines(raw: bytes | None) -> list[bytes]:
125
+ return [line for line in (raw or b"").split(b"\n") if line]
126
+
127
+
128
+ def _compact(value: object) -> bytes:
129
+ return json.dumps(value, separators=(",", ":")).encode("utf-8")
130
+
131
+
132
+ class SpillRoot:
133
+ """Where a host spills: a directory of its own under the cache root, the path read as written (never
134
+ the root itself, nor a directory the host clears or fills on its own) and opened from the cache root
135
+ held open, each step through no link and made the host's own; a link, a file or a directory it cannot
136
+ open or make its own on the way is set aside and made again."""
137
+
138
+ def __init__(self, cache: str, parts: tuple[str, ...]) -> None:
139
+ self.cache = cache
140
+ self.parts = parts
141
+
142
+ @classmethod
143
+ def at(cls, cache_root: Path | str, given: str | None) -> SpillRoot:
144
+ cache = os.path.normpath(os.path.abspath(cache_root))
145
+ relative = os.path.relpath(os.path.normpath(os.path.join(cache, given or SPILL_DIR)), cache)
146
+ parts = Path(relative).parts
147
+ if relative == "." or parts[0] == ".." or parts[0] in HOST_DIRS:
148
+ raise ValueError(
149
+ f"WORLDS_HOST_LEDGER_SPILL_DIR must be a directory of its own under the cache root {cache}, "
150
+ f"not {given}"
151
+ )
152
+ return cls(cache, parts)
153
+
154
+ @property
155
+ def path(self) -> Path:
156
+ return Path(self.cache).joinpath(*self.parts)
157
+
158
+ def open(self) -> SpillDirectory:
159
+ os.makedirs(self.cache, exist_ok=True)
160
+ current = SpillDirectory(os.open(self.cache, os.O_RDONLY | os.O_DIRECTORY))
161
+ for part in self.parts:
162
+ current = self._step(current, part)
163
+ return current
164
+
165
+ @staticmethod
166
+ def _step(parent: SpillDirectory, part: str) -> SpillDirectory:
167
+ try:
168
+ parent.ensure(part)
169
+ return SpillRoot._into(parent, part)
170
+ finally:
171
+ parent.close()
172
+
173
+ @staticmethod
174
+ def _into(parent: SpillDirectory, part: str) -> SpillDirectory:
175
+ try:
176
+ return SpillDirectory.owned(part, parent.descriptor)
177
+ except OSError:
178
+ parent.discard(part)
179
+ parent.make(part)
180
+ parent.sweep()
181
+ return SpillDirectory.owned(part, parent.descriptor)
182
+
183
+
184
+ @dataclass(eq=False)
185
+ class Segment:
186
+ seq: int
187
+ name: str
188
+ size: int
189
+ entries: int
190
+ sending: bool = False
191
+ misses: int = 0
192
+ full: bool = False
193
+
194
+
195
+ @dataclass(eq=False)
196
+ class Account:
197
+ """One host's account of one session's outbox (``epoch``; None for one the host only heard of from a
198
+ dead worker or closed before it drained anything): its spilled entries, oldest first, what it lost,
199
+ under the random ``source`` that names its directory, and, once ``closed``, whether everything the
200
+ host held of it was moved or counted (``complete``); final once complete with nothing left on disk.
201
+ ``home`` says its directory and record exist; one the disk refused keeps its counts in memory."""
202
+
203
+ source: str
204
+ session_id: str
205
+ epoch: str | None = None
206
+ shed: int = 0
207
+ lost: int = 0
208
+ closed: bool = False
209
+ complete: bool = False
210
+ home: bool = False
211
+ refused: bool = False
212
+ told: tuple[int, int, bool] | None = None
213
+ outboxes: dict[str, tuple[int, int]] = field(default_factory=dict)
214
+ segments: collections.deque[Segment] = field(default_factory=collections.deque)
215
+ sending: bool = False
216
+ records: int = 0
217
+ chain: bytes = b""
218
+
219
+ def account(self) -> tuple[int, int, bool]:
220
+ return (self.shed, self.lost, self.closed and self.complete and not self.segments)
221
+
222
+ def counts(self) -> dict:
223
+ return {"shed": self.shed, "lost": self.lost, "closed": self.closed, "complete": self.complete}
224
+
225
+ def snapshot(self) -> dict:
226
+ return {"session": self.session_id, **self.counts(), "segments": {s.name: s.entries for s in self.segments}}
227
+
228
+ def lose(self, entries: int) -> None:
229
+ self.lost = min(COUNT_MAX, self.lost + entries)
230
+
231
+ def outbox(self, epoch: str, shed: int, lost: int) -> bool:
232
+ """An outbox's own running counts (``epoch`` names the outbox): what they grew by since they were last
233
+ seen joins the account. True when the account grew."""
234
+ was_shed, was_lost = self.outboxes.pop(epoch, (0, 0))
235
+ self.outboxes[epoch] = (max(was_shed, shed), max(was_lost, lost))
236
+ if len(self.outboxes) > OUTBOXES_MAX:
237
+ self.outboxes.pop(next(iter(self.outboxes)))
238
+ grew_shed, grew_lost = max(0, shed - was_shed), max(0, lost - was_lost)
239
+ self.shed = min(COUNT_MAX, self.shed + grew_shed)
240
+ self.lost = min(COUNT_MAX, self.lost + grew_lost)
241
+ return bool(grew_shed or grew_lost)
242
+
243
+
244
+ class AccountRecord:
245
+ """An account's record as its sealed lines replay it: the newest snapshot (``session``, the counts and
246
+ the segments it held), then each change after it (``segment``: a segment's entries, ``gone``: a segment
247
+ stored or let go, ``counts``); a line before the snapshot, or off its shape, changes nothing. Each line
248
+ is sealed over the one before it, so a record missing, repeating or reordering a line is ``broken``."""
249
+
250
+ def __init__(self) -> None:
251
+ self.broken = False
252
+ self.session_id: str | None = None
253
+ self.counts = {"shed": 0, "lost": 0, "closed": False, "complete": False}
254
+ self.segments: dict[str, int] = {}
255
+ self._kinds: dict[str, Callable[[dict], None]] = {
256
+ "session": self._snapshot,
257
+ "segment": self._segment,
258
+ "gone": self._gone,
259
+ "counts": self._counted,
260
+ }
261
+
262
+ def apply(self, value: object) -> None:
263
+ line = value if isinstance(value, dict) else {}
264
+ kind = next((name for name in self._kinds if name in line), None)
265
+ known = kind == "session" or self.session_id is not None
266
+ if kind is not None and known:
267
+ self._kinds[kind](line)
268
+
269
+ def account(self, source: str, sizes: dict[str, int | None], unlisted: bool) -> Account:
270
+ """The account a restarted host takes: closed, complete only if its host closed it complete and its
271
+ record and segments are as that host left them, and what a listed segment no longer holds on disk
272
+ counted lost."""
273
+ assert self.session_id is not None
274
+ missing = sum(entries for name, entries in self.segments.items() if sizes.get(name) is None)
275
+ intact = not (self.broken or unlisted)
276
+ account = Account(
277
+ source,
278
+ self.session_id,
279
+ shed=self.counts["shed"],
280
+ lost=self.counts["lost"],
281
+ closed=True,
282
+ complete=self.counts["closed"] and self.counts["complete"] and intact,
283
+ home=True,
284
+ )
285
+ account.lose(missing)
286
+ account.segments.extend(
287
+ Segment(int(name[:12]), name, sizes[name] or 0, entries)
288
+ for name, entries in sorted(self.segments.items())
289
+ if sizes.get(name) is not None
290
+ )
291
+ return account
292
+
293
+ def _snapshot(self, line: dict) -> None:
294
+ session_id, segments = line.get("session"), line.get("segments")
295
+ named = isinstance(session_id, str) and 0 < len(session_id) <= SESSION_ID_CHARS
296
+ if named and self._valid_counts(line) and isinstance(segments, dict):
297
+ self.session_id = session_id
298
+ self.counts = {key: line[key] for key in self.counts}
299
+ self.segments = {name: n for name, n in segments.items() if SEGMENT.fullmatch(name) and counted(n)}
300
+
301
+ def _segment(self, line: dict) -> None:
302
+ name, entries = line.get("segment"), line.get("entries")
303
+ if isinstance(name, str) and SEGMENT.fullmatch(name) and counted(entries):
304
+ self.segments[name] = entries
305
+
306
+ def _gone(self, line: dict) -> None:
307
+ self.segments.pop(str(line.get("gone")), None)
308
+
309
+ def _counted(self, line: dict) -> None:
310
+ counts = line.get("counts")
311
+ if isinstance(counts, dict) and self._valid_counts(counts):
312
+ self.counts = {key: counts[key] for key in self.counts}
313
+
314
+ @staticmethod
315
+ def _valid_counts(value: dict) -> bool:
316
+ whole = counted(value.get("shed")) and counted(value.get("lost"))
317
+ return whole and isinstance(value.get("closed"), bool) and isinstance(value.get("complete"), bool)
318
+
319
+
320
+ @dataclass(eq=False)
321
+ class Piece:
322
+ """What one round sends of an account: its oldest segment, or its account alone, what of the segment did
323
+ not come back (``loss``, counted when the platform stores the piece) and the account as it was sent."""
324
+
325
+ account: Account
326
+ segment: Segment | None
327
+ loss: int = 0
328
+ sent: tuple[int, int, bool] = (0, 0, False)
329
+
330
+ @property
331
+ def size(self) -> int:
332
+ return ACCOUNT_BYTES + (self.segment.size if self.segment is not None else 0)
333
+
334
+
335
+ class LedgerSpill:
336
+ """The entries the host drained and the platform has not stored, on disk at ``root``, at most ``max_bytes``
337
+ of segments: past it the host's oldest go, counted lost in their accounts. A session's entries go to the
338
+ account of the outbox that answered them; :meth:`finished` closes the session's newest account, and only a
339
+ new outbox, or a page that lands after its account was settled, opens another. :meth:`round` hands out one
340
+ piece of every session at a time, oldest account first, so a session's entries go in order and no session
341
+ waits behind another; a piece is :meth:`stored` or :meth:`released`."""
342
+
343
+ def __init__(self, root: SpillRoot, max_bytes: int, seal: SpillSeal) -> None:
344
+ if max_bytes <= 0:
345
+ raise ValueError(f"the ledger spill's cap (WORLDS_HOST_LEDGER_SPILL_MB) must be positive, not {max_bytes}")
346
+ self.root = root
347
+ self.max_bytes = max_bytes
348
+ self.segment_bytes = min(SEGMENT_TARGET_BYTES, max_bytes // SEGMENTS_IN_CAP)
349
+ self.seal = seal
350
+ self.bytes = 0
351
+ self.accounts: dict[str, Account] = {}
352
+ self.latest: dict[str, Account] = {}
353
+ self.epochs: dict[tuple[str, str | None], Account] = {}
354
+ self._oldest: dict[Segment, Account] = {}
355
+ self._next = 1
356
+ self._directory: SpillDirectory | None = None
357
+ self._lock = threading.Lock()
358
+ self._sweeping = threading.Lock()
359
+ self._sweep_asked = threading.Event()
360
+ self._incomplete: set[str] = set()
361
+
362
+ @classmethod
363
+ def at(cls, cache_root: Path | str, given_dir: str | None, given_mb: str | None, secret: str) -> LedgerSpill:
364
+ """The spill a deployed host keeps (``WORLDS_HOST_LEDGER_SPILL_DIR``, ``..._MB``), not opened yet."""
365
+ megabytes = given_mb or str(DEFAULT_SPILL_MB)
366
+ if not WHOLE.fullmatch(megabytes):
367
+ raise ValueError(f"WORLDS_HOST_LEDGER_SPILL_MB must be a whole number of MiB, not {megabytes!r}")
368
+ return cls(SpillRoot.at(cache_root, given_dir), int(megabytes) << 20, SpillSeal(secret))
369
+
370
+ def open(self) -> LedgerSpill:
371
+ """Open the spill's directory (made if missing) and take every account a previous host left in it:
372
+ each is closed, final only if that host had closed it complete, and what a segment its record lists
373
+ no longer holds is counted lost; anything else in the directory is set aside."""
374
+ self._directory = self.root.open()
375
+ found = [self._adopt(name) for name in self.directory.names() if not name.startswith(TOMBSTONE)]
376
+ adopted = [account for account in found if account is not None]
377
+ segments = sorted(((segment, account) for account in adopted for segment in account.segments), key=_seq)
378
+ with self._lock:
379
+ self._oldest = dict(segments)
380
+ self.bytes = sum(segment.size for segment in self._oldest)
381
+ self._next = 1 + max((segment.seq for segment in self._oldest), default=0)
382
+ dropped = self._bound()
383
+ self._dropped(dropped)
384
+ self._sweep_later()
385
+ return self
386
+
387
+ @property
388
+ def directory(self) -> SpillDirectory:
389
+ if self._directory is None:
390
+ raise RuntimeError("the ledger spill is not open")
391
+ return self._directory
392
+
393
+ def write(
394
+ self, session_id: str, lines: list[bytes | None], epoch: str | None = None, source: str | None = None
395
+ ) -> None:
396
+ """``lines`` into the account of the outbox ``epoch`` (the session's newest when None; a new one is
397
+ kept under ``source``, the platform's account of the life, when given), an entry the platform would
398
+ not parse counted lost; past the cap the host's oldest segments go. Raises OSError, writing nothing
399
+ it can be sure of, when the disk refuses it: the outbox still holds the entries."""
400
+ kept = [line for line in lines if line is not None]
401
+ with self._lock:
402
+ account = self._account(session_id, epoch, source)
403
+ self._housed(account)
404
+ self._append(account, kept)
405
+ self._lose(account, len(lines) - len(kept))
406
+ dropped = self._bound()
407
+ self._dropped(dropped)
408
+ self._unsendable(session_id, len(lines) - len(kept))
409
+
410
+ def outbox(self, session_id: str, epoch: str, shed: int, lost: int, source: str | None = None) -> bool:
411
+ """The counts the outbox ``epoch`` reported in a drain answer; True when its account grew."""
412
+ with self._lock:
413
+ account = self._account(session_id, epoch, source)
414
+ grew = account.outbox(epoch, shed, lost)
415
+ if grew:
416
+ self._remember(account)
417
+ return grew
418
+
419
+ def lose(self, session_id: str, entries: int) -> None:
420
+ with self._lock:
421
+ self._lose(self._account(session_id, None), entries)
422
+
423
+ def finished(self, session_id: str, complete: bool) -> None:
424
+ """The host is done with the session: its newest account closes, complete when everything the host
425
+ held of it was moved here or counted; an account closes once, the first verdict standing, except that
426
+ an incomplete verdict after a complete one closes a new account, so it is never lost. The session
427
+ stays :meth:`incomplete` for as long as the host serves it."""
428
+ with self._lock:
429
+ if not complete:
430
+ self._incomplete.add(session_id)
431
+ account = self._account(session_id, None)
432
+ if account.closed and (complete or not account.complete):
433
+ return
434
+ if account.closed:
435
+ account = self._open_account(session_id, None)
436
+ account.closed, account.complete = True, complete
437
+ self._remember(account)
438
+
439
+ def round(self, held: set[str], only: str | None = None) -> list[Piece]:
440
+ """The next piece of every session not ``held`` (or of ``only``): its oldest account's oldest
441
+ segment, or that account alone when it has only its counts left to tell."""
442
+ with self._lock:
443
+ taken: set[str] = set(held)
444
+ accounts = [account for account in self.accounts.values() if only in (None, account.session_id)]
445
+ pieces = [self._piece(account, taken) for account in accounts]
446
+ return [piece for piece in pieces if piece is not None]
447
+
448
+ def pending(self, session_id: str) -> bool:
449
+ """Whether entries of the session wait here unsent, or are on their way in a send."""
450
+ with self._lock:
451
+ return any(account.segments or account.sending for account in self._of(session_id))
452
+
453
+ def counts(self, session_id: str) -> tuple[int, int]:
454
+ """What the session's accounts here shed and lost."""
455
+ with self._lock:
456
+ accounts = self._of(session_id)
457
+ return sum(a.shed for a in accounts), sum(a.lost for a in accounts)
458
+
459
+ def incomplete(self, session_id: str) -> bool:
460
+ """Whether an account of the session closed incomplete while the host served it, or one here did."""
461
+ with self._lock:
462
+ closed = (account.closed and not account.complete for account in self._of(session_id))
463
+ return session_id in self._incomplete or any(closed)
464
+
465
+ def _of(self, session_id: str) -> list[Account]:
466
+ return [account for account in self.accounts.values() if account.session_id == session_id]
467
+
468
+ def item(self, piece: Piece) -> dict | None:
469
+ """The flush item a piece sends: its segment's entries the platform parses, with its account as it
470
+ stands once what the segment did not give back is counted, final when the account is complete and
471
+ this is all it holds. None while its segment could not be read and has reads left; one read in none
472
+ of ``SEGMENT_MISSES`` tries is sent empty. Nothing is counted until :meth:`stored`."""
473
+ requests = self._read(piece)
474
+ if requests is None:
475
+ return None
476
+ with self._lock:
477
+ account = piece.account
478
+ expected = piece.segment.entries if piece.segment is not None else 0
479
+ piece.loss = max(0, expected - len(requests))
480
+ last = all(segment is piece.segment for segment in account.segments)
481
+ piece.sent = (
482
+ account.shed,
483
+ min(COUNT_MAX, account.lost + piece.loss),
484
+ account.closed and account.complete and last,
485
+ )
486
+ shed, lost, final = piece.sent
487
+ return {
488
+ "sessionId": account.session_id,
489
+ "requests": requests,
490
+ "source": account.source,
491
+ "shed": shed,
492
+ "lost": lost,
493
+ "final": final,
494
+ }
495
+
496
+ def stored(self, pieces: list[Piece]) -> None:
497
+ """Pieces the platform stored: their segments go, what they did not give back is counted, and their
498
+ accounts were told."""
499
+ with self._lock:
500
+ for piece in pieces:
501
+ self._drop(piece.segment)
502
+ self._lose(piece.account, piece.loss)
503
+ piece.account.told = piece.sent
504
+
505
+ def released(self, pieces: list[Piece]) -> None:
506
+ """Pieces sent or not: each may be handed out again."""
507
+ with self._lock:
508
+ for piece in pieces:
509
+ piece.account.sending = False
510
+ _mark(piece.segment, False)
511
+
512
+ def settle(self, served: set[str]) -> None:
513
+ """Forget every account whose entries were stored and whose counts were told, unless it is still open
514
+ for a session the host serves; their directories are set aside at once and deleted after, off the
515
+ caller's thread."""
516
+ with self._lock:
517
+ self._incomplete &= served
518
+ done = [account for account in self.accounts.values() if self._settled(account, served)]
519
+ for account in done:
520
+ self._forget(account)
521
+ if done:
522
+ self._sweep_later()
523
+
524
+ def _settled(self, account: Account, served: set[str]) -> bool:
525
+ idle = not (account.segments or account.sending)
526
+ open_here = not account.closed and account.session_id in served
527
+ return idle and not open_here and account.told == account.account()
528
+
529
+ def _forget(self, account: Account) -> None:
530
+ del self.accounts[account.source]
531
+ if self.latest.get(account.session_id) is account:
532
+ del self.latest[account.session_id]
533
+ self.epochs.pop((account.session_id, account.epoch), None)
534
+ if account.home:
535
+ self._discard(account.source)
536
+
537
+ def _discard(self, name: str) -> None:
538
+ try:
539
+ self.directory.discard(name)
540
+ except OSError as error:
541
+ log_line(f"request ledger spill: {name} not set aside: {type(error).__name__}")
542
+
543
+ def _account(self, session_id: str, epoch: str | None, source: str | None = None) -> Account:
544
+ found = self.epochs.get((session_id, epoch)) if epoch is not None else self.latest.get(session_id)
545
+ return found if found is not None else self._open_account(session_id, epoch, source)
546
+
547
+ def _open_account(self, session_id: str, epoch: str | None, source: str | None = None) -> Account:
548
+ fresh = source is None or source in self.accounts
549
+ account = Account(uuid.uuid4().hex if fresh else source, session_id, epoch)
550
+ self.accounts[account.source] = self.latest[session_id] = self.epochs[(session_id, epoch)] = account
551
+ self._house(account)
552
+ return account
553
+
554
+ def _house(self, account: Account) -> bool:
555
+ """The account's directory and record made, when they are not yet; False while the disk refuses them,
556
+ the account kept in memory meanwhile."""
557
+ if account.home:
558
+ return True
559
+ try:
560
+ self.directory.reowned(partial(self.directory.ensure, account.source))
561
+ self._compact(account)
562
+ except OSError as error:
563
+ self._refused(account, error)
564
+ return False
565
+ account.home = True
566
+ return True
567
+
568
+ def _housed(self, account: Account) -> None:
569
+ if not self._house(account):
570
+ raise OSError(errno.EIO, f"the spill cannot hold session {account.session_id}'s account")
571
+
572
+ def _refused(self, account: Account, error: OSError) -> None:
573
+ if not account.refused:
574
+ log_line(
575
+ f"request ledger spill: session {account.session_id} has no account on disk: {type(error).__name__}"
576
+ )
577
+ account.refused = True
578
+
579
+ def _adopt(self, name: str) -> Account | None:
580
+ try:
581
+ return self._found(name)
582
+ except Exception as error: # noqa: BLE001 - one account a previous host left never stops the others
583
+ log_line(f"request ledger spill: {name} not taken: {type(error).__name__}")
584
+ return None
585
+
586
+ def _found(self, name: str) -> Account | None:
587
+ mode = self.directory.mode(name)
588
+ if not (HEX32.fullmatch(name) and mode is not None and stat.S_ISDIR(mode)):
589
+ return self._unknown(name)
590
+ with self.directory.child(name) as directory:
591
+ record = self._replayed(name, directory.tail(ACCOUNT_FILE, ACCOUNT_READ_BYTES))
592
+ sizes = {segment: directory.size(segment) for segment in record.segments}
593
+ unlisted = any(SEGMENT.fullmatch(file) and file not in sizes for file in directory.names())
594
+ if record.session_id is None:
595
+ return self._unknown(name)
596
+ account = record.account(name, sizes, unlisted)
597
+ self.accounts[name] = account
598
+ self._rewrite(account)
599
+ return account
600
+
601
+ def _replayed(self, source: str, raw: bytes | None) -> AccountRecord:
602
+ record = AccountRecord()
603
+ place = b""
604
+ for line in _lines(raw):
605
+ data = self.seal.opened(source, line, place)
606
+ record.broken = data is None
607
+ if record.broken:
608
+ break
609
+ record.apply(_json(data))
610
+ place = SpillSeal.tag(line)
611
+ return record
612
+
613
+ def _unknown(self, name: str) -> None:
614
+ log_line(f"request ledger spill: {name} holds no account this host sealed; removed")
615
+ self._discard(name)
616
+
617
+ @staticmethod
618
+ def _piece(account: Account, taken: set[str]) -> Piece | None:
619
+ if account.session_id in taken:
620
+ return None
621
+ if account.sending:
622
+ taken.add(account.session_id)
623
+ return None
624
+ segment = account.segments[0] if account.segments else None
625
+ if segment is None and account.told == account.account():
626
+ return None
627
+ taken.add(account.session_id)
628
+ account.sending = True
629
+ return Piece(account, _mark(segment, True))
630
+
631
+ def _read(self, piece: Piece) -> list[dict] | None:
632
+ segment = piece.segment
633
+ if segment is None:
634
+ return []
635
+ raw = self._raw(piece.account, segment)
636
+ if raw is None and self._missed(segment):
637
+ return None
638
+ source = piece.account.source
639
+ opened = (self.seal.opened(source, line, _place(segment, n)) for n, line in enumerate(_lines(raw)))
640
+ return [entry for entry in map(_entry, opened) if entry is not None]
641
+
642
+ def _missed(self, segment: Segment) -> bool:
643
+ with self._lock:
644
+ segment.misses += 1
645
+ return segment.misses < SEGMENT_MISSES
646
+
647
+ def _raw(self, account: Account, segment: Segment) -> bytes | None:
648
+ try:
649
+ with self.directory.child(account.source) as directory:
650
+ return directory.tail(segment.name, SEGMENT_READ_BYTES)
651
+ except OSError:
652
+ return None
653
+
654
+ def _append(self, account: Account, lines: list[bytes]) -> None:
655
+ for run in runs(lines, LEDGER_BATCH_BYTES, SpillSeal.OVERHEAD):
656
+ self._add(account, run)
657
+
658
+ def _add(self, account: Account, lines: list[bytes]) -> None:
659
+ """A run of lines on disk, each sealed to its segment and place in it, the record told first: a
660
+ segment the record lists and the disk does not hold is counted lost by a restarted host, never one
661
+ it holds and the record misses."""
662
+ size = sum(len(line) + SpillSeal.OVERHEAD for line in lines)
663
+ tail = account.segments[-1] if account.segments else None
664
+ if tail is not None and self._extends(tail, size):
665
+ return self._extend(account, tail, self._sealed(account, tail.name, tail.entries, lines), len(lines))
666
+ seq, self._next = self._next, self._next + 1
667
+ name = f"{seq:012d}.jsonl"
668
+ data = self._sealed(account, name, 0, lines)
669
+ entries = len(lines)
670
+ segment = Segment(seq, name, len(data), entries)
671
+ account.segments.append(segment)
672
+ try:
673
+ self._write_record(account, {"segment": segment.name, "entries": entries})
674
+ with self.directory.child(account.source) as directory:
675
+ directory.create(segment.name, data)
676
+ except OSError:
677
+ account.segments.remove(segment)
678
+ self._record(account, {"gone": segment.name})
679
+ raise
680
+ self._oldest[segment] = account
681
+ self.bytes += segment.size
682
+
683
+ def _sealed(self, account: Account, segment: str, first: int, lines: list[bytes]) -> bytes:
684
+ places = (_place_of(segment, first + n) for n in range(len(lines)))
685
+ return b"".join(self.seal.seal(account.source, line, place) for line, place in zip(lines, places))
686
+
687
+ def _extends(self, tail: Segment, size: int) -> bool:
688
+ return not (tail.sending or tail.full) and tail.size + size <= self.segment_bytes
689
+
690
+ def _extend(self, account: Account, segment: Segment, data: bytes, entries: int) -> None:
691
+ segment.entries += entries
692
+ try:
693
+ self._write_record(account, {"segment": segment.name, "entries": segment.entries})
694
+ except OSError:
695
+ segment.entries -= entries
696
+ raise
697
+ try:
698
+ with self.directory.child(account.source) as directory:
699
+ directory.add(segment.name, data)
700
+ except OSError:
701
+ segment.entries -= entries
702
+ segment.full = True
703
+ self._record(account, {"segment": segment.name, "entries": segment.entries})
704
+ raise
705
+ segment.size += len(data)
706
+ self.bytes += len(data)
707
+
708
+ def _bound(self) -> int:
709
+ dropped = 0
710
+ while self.bytes > self.max_bytes:
711
+ victim = next((segment for segment in self._oldest if not segment.sending), None)
712
+ if victim is None:
713
+ break
714
+ account = self._oldest[victim]
715
+ self._lose(account, victim.entries)
716
+ self._drop(victim)
717
+ dropped += victim.entries
718
+ return dropped
719
+
720
+ def _drop(self, segment: Segment | None) -> None:
721
+ """A segment stored or let go: deleted only once the record says so, so a restarted host never counts
722
+ it lost for having been deleted (one the record could not drop stays until its account goes)."""
723
+ account = self._oldest.pop(segment, None) if segment is not None else None
724
+ if account is None:
725
+ return
726
+ account.segments.remove(segment)
727
+ self.bytes -= segment.size
728
+ try:
729
+ self._write_record(account, {"gone": segment.name})
730
+ with self.directory.child(account.source) as directory:
731
+ directory.unlink(segment.name)
732
+ except OSError as error:
733
+ log_line(f"request ledger spill: segment {segment.seq} not removed: {type(error).__name__}")
734
+
735
+ def _lose(self, account: Account, entries: int) -> None:
736
+ if entries <= 0:
737
+ return
738
+ account.lose(entries)
739
+ self._remember(account)
740
+
741
+ def _remember(self, account: Account) -> None:
742
+ self._record(account, {"counts": account.counts()})
743
+
744
+ def _record(self, account: Account, change: dict) -> None:
745
+ """One change to the account's record, or the record rewritten as a snapshot once its changes
746
+ outnumber its segments; a record the disk refuses is logged, its counts kept in memory."""
747
+ if not account.home:
748
+ return
749
+ try:
750
+ self._write_record(account, change)
751
+ except OSError as error:
752
+ log_line(f"request ledger spill: session {account.session_id} record not written: {type(error).__name__}")
753
+
754
+ def _write_record(self, account: Account, change: dict) -> None:
755
+ if account.records >= max(RECORD_LINES_MIN, 4 * len(account.segments)):
756
+ return self._compact(account)
757
+ line = self.seal.seal(account.source, _compact(change), account.chain)
758
+ with self.directory.child(account.source) as directory:
759
+ directory.add(ACCOUNT_FILE, line)
760
+ account.records += 1
761
+ account.chain = SpillSeal.tag(line)
762
+
763
+ def _compact(self, account: Account) -> None:
764
+ line = self.seal.seal(account.source, _compact(account.snapshot()))
765
+ with self.directory.child(account.source) as directory:
766
+ directory.replace(ACCOUNT_FILE, line)
767
+ account.records = 1
768
+ account.chain = SpillSeal.tag(line)
769
+
770
+ def _rewrite(self, account: Account) -> None:
771
+ try:
772
+ self._compact(account)
773
+ except OSError as error:
774
+ log_line(f"request ledger spill: session {account.session_id} record not rewritten: {type(error).__name__}")
775
+
776
+ def _sweep_later(self) -> None:
777
+ """Delete what was set aside on a thread of its own, once more after any sweep asked for while one
778
+ ran: a tree tenant code planted never holds up a pass."""
779
+ self._sweep_asked.set()
780
+ if self._sweeping.acquire(blocking=False):
781
+ threading.Thread(target=self._sweep, name="world-host-ledger-sweep", daemon=True).start()
782
+
783
+ def _sweep(self) -> None:
784
+ try:
785
+ self._sweep_while_asked()
786
+ finally:
787
+ self._sweeping.release()
788
+ if self._sweep_asked.is_set():
789
+ self._sweep_later()
790
+
791
+ def _sweep_while_asked(self) -> None:
792
+ while self._sweep_asked.is_set():
793
+ self._sweep_asked.clear()
794
+ self._sweep_once()
795
+
796
+ def _sweep_once(self) -> None:
797
+ try:
798
+ self.directory.sweep()
799
+ except OSError as error:
800
+ log_line(f"request ledger spill: sweep failed: {type(error).__name__}")
801
+
802
+ @staticmethod
803
+ def _dropped(entries: int) -> None:
804
+ if entries:
805
+ log_line(f"request ledger spill: let go of {entries} unsent entries past its cap")
806
+
807
+ @staticmethod
808
+ def _unsendable(session_id: str, entries: int) -> None:
809
+ if entries:
810
+ log_line(f"request ledger spill: session {session_id} let go of {entries} unsendable entries, counted lost")
811
+
812
+
813
+ def _mark(segment: Segment | None, sending: bool) -> Segment | None:
814
+ if segment is not None:
815
+ segment.sending = sending
816
+ return segment
817
+
818
+
819
+ def _seq(pair: tuple[Segment, Account]) -> int:
820
+ return pair[0].seq
821
+
822
+
823
+ def _place_of(segment: str, ordinal: int) -> bytes:
824
+ return f"{segment}:{ordinal}".encode("ascii")
825
+
826
+
827
+ def _place(segment: Segment, ordinal: int) -> bytes:
828
+ return _place_of(segment.name, ordinal)